diff --git a/.gitignore b/.gitignore index ce377c67..76c6a42f 100644 --- a/.gitignore +++ b/.gitignore @@ -3,13 +3,19 @@ semver.mk vars.mk Dockerfile-debug +go-code-tester* +csm-common.mk +*coverage*.txt # test directories and artifacts service/c.out service/test/ +vendor +**/test-data* # IDE .idea +.vscode # gosec gosec.log @@ -26,3 +32,5 @@ gosecresults.csv # logs *.log +vendor/ + diff --git a/.golangci.yaml b/.golangci.yaml deleted file mode 100644 index 56f53324..00000000 --- a/.golangci.yaml +++ /dev/null @@ -1,30 +0,0 @@ -run: - timeout: 20m - issue-exit-code: 0 # we will change this later - tests: true - skip-dirs-use-default: true - modules-download-mode: readonly - -issues: - max-issues-per-linter: 0 - max-same-issues: 0 - new: false - -output: - print-linter-name: true - sort-results: true - uniq-by-line: false - print-issued-lines: true - -linters: - disable-all: true - fast: false - enable: - # A stricter replacement for gofmt. - - gofumpt - # Inspects source code for security problems. - - gosec - # Check for correctness of programs. - - govet - # Drop-in replacement of golint. - - revive diff --git a/.mockery.yaml b/.mockery.yaml index f9401e64..8be4c164 100644 --- a/.mockery.yaml +++ b/.mockery.yaml @@ -14,8 +14,8 @@ # # -quiet: False -all: True -inpackage: False +quiet: false +all: true +inpackage: false dir: ./pkg -disable-version-string: True \ No newline at end of file +disable-version-string: true diff --git a/docker-files/Dockerfile.ubi.micro b/Dockerfile similarity index 56% rename from docker-files/Dockerfile.ubi.micro rename to Dockerfile index 95d01fbd..c3b39be1 100644 --- a/docker-files/Dockerfile.ubi.micro +++ b/Dockerfile @@ -1,6 +1,4 @@ -# -# -# Copyright © 2023 Dell Inc. or its subsidiaries. All Rights Reserved. +# Copyright © 2023-2025 Dell Inc. or its subsidiaries. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,30 +9,37 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# -# Dockerfile to build PowerStore CSI Driver -# based on UBI-micro image -# Requires: RHEL host with subscription -# UBI Image: ubi9/ubi-micro:9.2-13 +# some arguments that must be supplied +ARG GOIMAGE ARG BASEIMAGE +ARG VERSION="2.16.0" -FROM $BASEIMAGE +# Stage to build the driver +FROM $GOIMAGE as builder +ARG VERSION -LABEL vendor="Dell Inc." \ +RUN mkdir -p /go/src/csi-powerstore +COPY ./ /go/src/csi-powerstore + +WORKDIR /go/src/csi-powerstore +RUN make build IMAGE_VERSION=$VERSION && \ + rm -rf /go/src/csi-powerstore/vendor + +# Stage to build the driver image +FROM $BASEIMAGE +ARG VERSION +WORKDIR / +LABEL vendor="Dell Technologies" \ + maintainer="Dell Technologies" \ name="csi-powerstore" \ summary="CSI Driver for Dell EMC PowerStore" \ description="CSI Driver for provisioning persistent storage from Dell EMC PowerStore" \ - version="2.8.0" \ + release="1.16.0" \ + version=$VERSION \ license="Apache-2.0" - COPY licenses /licenses -# validate some cli utilities are found -RUN which mkfs.ext4 -RUN which mkfs.xfs -RUN echo "export PATH=$PATH:/sbin:/bin" > /etc/profile.d/ubuntu_path.sh +COPY --from=builder /go/src/csi-powerstore/csi-powerstore /csi-powerstore -COPY "csi-powerstore" . ENTRYPOINT ["/csi-powerstore"] diff --git a/Makefile b/Makefile index 42ebd712..abae4a27 100644 --- a/Makefile +++ b/Makefile @@ -1,94 +1,54 @@ +# Copyright © 2026 Dell Inc. or its subsidiaries. All Rights Reserved. # -# -# Copyright © 2020-2023 Dell Inc. or its subsidiaries. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# - -# for variables override --include vars.mk - -all: clean build - -# Dockerfile defines which base image to use [Dockerfile.centos, Dockerfile.ubi, Dockerfile.ubi.min, Dockerfile.ubi.alt] -# e.g.:$ make docker DOCKER_FILE=Dockerfile.ubi.alt -ifndef DOCKER_FILE - DOCKER_FILE = Dockerfile.ubi.micro -endif - -# Tag parameters -ifndef MAJOR - MAJOR=2 -endif -ifndef MINOR - MINOR=8 -endif -ifndef PATCH - PATCH=0 -endif -ifndef NOTES - NOTES= -endif -ifndef TAGMSG - TAGMSG="CSI Spec 1.6" -endif +# Dell Technologies, Dell and other trademarks are trademarks of Dell Inc. +# or its subsidiaries. Other trademarks may be trademarks of their respective +# owners. + +include images.mk + +all: help + +# This will be overridden during image build. +IMAGE_VERSION ?= 0.0.0 +LDFLAGS = "-X main.ManifestSemver=$(IMAGE_VERSION)" + +# Help target, prints usefule information +help: + @echo + @echo "The following targets are commonly used:" + @echo + @echo "build - Builds the code locally" + @echo "check - Runs the suite of code checking tools: lint, format, etc" + @echo "clean - Cleans the local build" + @echo "images - Builds the code within a golang container and then creates the driver image" + @echo "push - Pushes the built container to a target registry" + @echo "unit-test - Runs the unit tests" + @echo "vendor - Downloads a vendor list (local copy) of repositories required to compile the repo." clean: - rm -f core/core_generated.go - rm -f semver.mk - go clean + rm -f semver.mk core/core_generated.go + rm -rf vendor + rm -f csi-powerstore + go clean -cache -build: - go generate ./cmd/csi-powerstore - GOOS=linux CGO_ENABLED=0 go build ./cmd/csi-powerstore - -install: - go generate ./cmd/csi-powerstore - GOOS=linux CGO_ENABLED=0 go install ./cmd/csi-powerstore - -# Tags the release with the Tag parameters set above -tag: - -git tag -d v$(MAJOR).$(MINOR).$(PATCH)$(NOTES) - git tag -a -m $(TAGMSG) v$(MAJOR).$(MINOR).$(PATCH)$(NOTES) - -# Generates the docker container (but does not push) -docker: - go generate ./cmd/csi-powerstore - go run core/semver/semver.go -f mk >semver.mk - make -f docker.mk DOCKER_FILE=docker-files/$(DOCKER_FILE) docker - -# Same as `docker` but without cached layers and will pull latest version of base image -docker-no-cache: - go generate ./cmd/csi-powerstore - go run core/semver/semver.go -f mk >semver.mk - make -f docker.mk DOCKER_FILE=docker-files/$(DOCKER_FILE) docker-no-cache - -# Pushes container to the repository -push: docker - make -f docker.mk push - -check: gosec - gofmt -w ./. - golint -set_exit_status ./. - go vet ./... +build: generate vendor + GOOS=linux CGO_ENABLED=0 go build -mod=vendor -ldflags $(LDFLAGS) ./cmd/csi-powerstore mocks: mockery +unit-test: go-code-tester + GITHUB_OUTPUT=/dev/null \ + ./go-code-tester 90 "." "" "true" "" "" "./mocks|./v2/core|./tests|./replace" + test: - go clean -cache; cd ./pkg; go test -race -cover -coverprofile=coverage.out -coverpkg ./... ./... + cd ./pkg; go test -race -cover -coverprofile=coverage.out ./... coverage: cd ./pkg; go tool cover -html=coverage.out -o coverage.html -gosec: - gosec -quiet -log gosec.log -out=gosecresults.csv -fmt=csv ./... +go-code-tester: + git clone --depth 1 git@github.com:CSM/actions.git temp-repo + cp temp-repo/go-code-tester/entrypoint.sh ./go-code-tester + chmod +x go-code-tester + rm -rf temp-repo diff --git a/README.md b/README.md index 5d380b9e..2129adbd 100644 --- a/README.md +++ b/README.md @@ -1,66 +1,74 @@ -# CSI Driver for Dell PowerStore - -[![Go Report Card](https://goreportcard.com/badge/github.com/dell/csi-powerstore?style=flat-square)](https://goreportcard.com/report/github.com/dell/csi-powerstore) -[![License](https://img.shields.io/github/license/dell/csi-powerstore?style=flat-square&color=blue&label=License)](https://github.com/dell/csi-powerstore/blob/master/LICENSE) -[![Docker](https://img.shields.io/docker/pulls/dellemc/csi-powerstore.svg?logo=docker&style=flat-square&label=Pulls)](https://hub.docker.com/r/dellemc/csi-powerstore) -[![Last Release](https://img.shields.io/github/v/release/dell/csi-powerstore?label=Latest&style=flat-square&logo=go)](https://github.com/dell/csi-powerstore/releases) - -**Repository for CSI Driver for Dell PowerStore** - -## Description -CSI Driver for PowerStore is part of the [CSM (Container Storage Modules)](https://github.com/dell/csm) open-source suite of Kubernetes storage enablers for Dell products. CSI Driver for PowerStore is a Container Storage Interface (CSI) driver that provides support for provisioning persistent storage using Dell PowerStore storage array. - -It supports CSI specification version 1.6. - -This project may be compiled as a stand-alone binary using Golang that, when run, provides a valid CSI endpoint. It also can be used as a precompiled container image. - -## Table of Contents - -* [Code of Conduct](https://github.com/dell/csm/blob/main/docs/CODE_OF_CONDUCT.md) -* [Maintainer Guide](https://github.com/dell/csm/blob/main/docs/MAINTAINER_GUIDE.md) -* [Committer Guide](https://github.com/dell/csm/blob/main/docs/COMMITTER_GUIDE.md) -* [Contributing Guide](https://github.com/dell/csm/blob/main/docs/CONTRIBUTING.md) -* [List of Adopters](https://github.com/dell/csm/blob/main/docs/ADOPTERS.md) -* [Support](#support) -* [Security](https://github.com/dell/csm/blob/main/docs/SECURITY.md) -* [Building](#building) -* [Runtime Dependecies](#runtime-dependencies) -* [Driver Installation](#driver-installation) -* [Using Driver](#using-driver) -* [Documentation](#documentation) - -## Support -For any CSI driver issues, questions or feedback, please follow our [support process](https://github.com/dell/csm/blob/main/docs/SUPPORT.md) - -## Building -This project is a Go module (see golang.org Module information for explanation). -The dependencies for this project are listed in the go.mod file. - -To build the source, execute `make clean build`. - -To run unit tests, execute `make test`. - -To build an image, execute `make docker`. - -## Runtime Dependencies - -Both the Controller and the Node portions of the driver can only be run on nodes with network connectivity to a Dell PowerStore server (which is used by the driver). - -If you want to use iSCSI as a transport protocol be sure that `iscsi-initiator-utils` package is installed on your node. - -If you want to use FC be sure that zoning of Host Bus Adapters to the FC port directors was done. - -If you want to use NFS be sure to enable it in `myvalues.yaml` or in your storage classes, and configure corresponding NAS servers on PowerStore. - -If you want to use NVMe/TCP be sure that the `nvme-cli` package is installed on your node. - -If you want to use NVMe/FC be sure that the NVMeFC zoning of the Host Bus Adapters to the Fibre Channel port is done. - -## Driver Installation -Please consult the [Installation Guide](https://dell.github.io/csm-docs/docs/csidriver/installation) - -## Using Driver -Please refer to the section `Testing Drivers` in the [Documentation](https://dell.github.io/csm-docs/docs/csidriver/installation/test/) for more info. - -## Documentation -For more detailed information on the driver, please refer to [Container Storage Modules documentation](https://dell.github.io/csm-docs/). +# :lock: **Important Notice** +Starting with the release of **Container Storage Modules v1.16.0**, this repository will no longer be maintained as an open source project. Future development will continue under a closed source model. This change reflects our commitment to delivering even greater value to our customers by enabling faster innovation and more deeply integrated features with the Dell storage portfolio.
+For existing customers using Dell’s Container Storage Modules, you will continue to receive: +* **Ongoing Support & Community Engagement**
+ You will continue to receive high-quality support through Dell Support and our community channels. Your experience of engaging with the Dell community remains unchanged. +* **Streamlined Deployment & Updates**
+ Deployment and update processes will remain consistent, ensuring a smooth and familiar experience. +* **Access to Documentation & Resources**
+ All documentation and related materials will remain publicly accessible, providing transparency and technical guidance. +* **Continued Access to Current Open Source Version**
+ The current open-source version will remain available under its existing license for those who rely on it. + +Moving to a closed source model allows Dell’s development team to accelerate feature delivery and enhance integration across our Enterprise Kubernetes Storage solutions ultimately providing a more seamless and robust experience.
+We deeply appreciate the contributions of the open source community and remain committed to supporting our customers through this transition.
+ +For questions or access requests, please contact the maintainers via [Dell Support](https://www.dell.com/support/kbdoc/en-in/000188046/container-storage-interface-csi-drivers-and-container-storage-modules-csm-how-to-get-support). + +# CSI Driver for Dell PowerStore + +[![Go Report Card](https://goreportcard.com/badge/github.com/dell/csi-powerstore?style=flat-square)](https://goreportcard.com/report/github.com/dell/csi-powerstore) +[![License](https://img.shields.io/github/license/dell/csi-powerstore?style=flat-square&color=blue&label=License)](https://github.com/dell/csi-powerstore/blob/master/LICENSE) +[![Docker](https://img.shields.io/docker/pulls/dellemc/csi-powerstore.svg?logo=docker&style=flat-square&label=Pulls)](https://hub.docker.com/r/dellemc/csi-powerstore) +[![Last Release](https://img.shields.io/github/v/release/dell/csi-powerstore?label=Latest&style=flat-square&logo=go)](https://github.com/dell/csi-powerstore/releases) + +**Repository for CSI Driver for Dell PowerStore** + +## Description +CSI Driver for PowerStore is part of the [CSM (Container Storage Modules)](https://github.com/dell/csm) open-source suite of Kubernetes storage enablers for Dell products. CSI Driver for PowerStore is a Container Storage Interface (CSI) driver that provides support for provisioning persistent storage using Dell PowerStore storage array. + +This project may be compiled as a stand-alone binary using Golang that, when run, provides a valid CSI endpoint. It also can be used as a precompiled container image. + +## Table of Contents + +* [Code of Conduct](https://github.com/dell/csm/blob/main/docs/CODE_OF_CONDUCT.md) +* [Maintainer Guide](https://github.com/dell/csm/blob/main/docs/MAINTAINER_GUIDE.md) +* [Committer Guide](https://github.com/dell/csm/blob/main/docs/COMMITTER_GUIDE.md) +* [Contributing Guide](https://github.com/dell/csm/blob/main/docs/CONTRIBUTING.md) +* [List of Adopters](https://github.com/dell/csm/blob/main/docs/ADOPTERS.md) +* [Support](#support) +* [Security](https://github.com/dell/csm/blob/main/docs/SECURITY.md) +* [Building](#building) +* [Runtime Dependecies](#runtime-dependencies) +* [Documentation](#documentation) + +## Support +For any issues, questions or feedback, please contact [Dell support](https://www.dell.com/support/incidents-online/en-us/contactus/product/container-storage-modules). + +## Building +This project is a Go module (see golang.org Module information for explanation). +The dependencies for this project are listed in the go.mod file. + +To build the source, execute `make clean build`. + +To run unit tests, execute `make test`. + +To build an image, execute `make docker`. + +## Runtime Dependencies + +Both the Controller and the Node portions of the driver can only be run on nodes with network connectivity to a Dell PowerStore server (which is used by the driver). + +If you want to use iSCSI as a transport protocol be sure that `iscsi-initiator-utils` package is installed on your node. + +If you want to use FC be sure that zoning of Host Bus Adapters to the FC port directors was done. + +If you want to use NFS be sure to enable it in `myvalues.yaml` or in your storage classes, and configure corresponding NAS servers on PowerStore. + +If you want to use NVMe/TCP be sure that the `nvme-cli` package is installed on your node. + +If you want to use NVMe/FC be sure that the NVMeFC zoning of the Host Bus Adapters to the Fibre Channel port is done. + +## Documentation +For more detailed information on the driver, please refer to [Container Storage Modules documentation](https://dell.github.io/csm-docs/). + diff --git a/buildubimicro.sh b/buildubimicro.sh deleted file mode 100755 index 72c4e351..00000000 --- a/buildubimicro.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -# -# Copyright © 2023 Dell Inc., or its subsidiaries. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -microcontainer=$(buildah from $1) -micromount=$(buildah mount $microcontainer) -dnf install --installroot $micromount --releasever=9 --nodocs --setopt install_weak_deps=false --setopt=reposdir=/etc/yum.repos.d/ e2fsprogs xfsprogs nfs-utils nfs4-acl-tools acl which device-mapper-multipath util-linux -y -dnf clean all --installroot $micromount -buildah umount $microcontainer -buildah commit $microcontainer csipowerstore-ubimicro -echo "$(buildah images | grep csipowerstore-ubimicro)" \ No newline at end of file diff --git a/cmd/csi-powerstore/main.go b/cmd/csi-powerstore/main.go index 9efa395a..cff34461 100644 --- a/cmd/csi-powerstore/main.go +++ b/cmd/csi-powerstore/main.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2021-2023 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2021-2026 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,45 +20,63 @@ import ( "context" "fmt" "os" + "strconv" "strings" "time" - "github.com/dell/csi-powerstore/v2/core" - "github.com/dell/csi-powerstore/v2/pkg/common" - "github.com/dell/csi-powerstore/v2/pkg/common/fs" + "github.com/dell/csi-powerstore/v2/pkg/array" "github.com/dell/csi-powerstore/v2/pkg/controller" + "github.com/dell/csi-powerstore/v2/pkg/identifiers" + "github.com/dell/csi-powerstore/v2/pkg/identifiers/fs" "github.com/dell/csi-powerstore/v2/pkg/identity" "github.com/dell/csi-powerstore/v2/pkg/interceptors" + "github.com/dell/csi-powerstore/v2/pkg/monitor" "github.com/dell/csi-powerstore/v2/pkg/node" "github.com/dell/csi-powerstore/v2/pkg/tracer" + drController "github.com/dell/csm-dr/pkg/controller" + "github.com/dell/csmlog" "github.com/dell/gocsi" csictx "github.com/dell/gocsi/context" "github.com/dell/gofsutil" "github.com/fsnotify/fsnotify" grpc_opentracing "github.com/grpc-ecosystem/go-grpc-middleware/tracing/opentracing" "github.com/opentracing/opentracing-go" - log "github.com/sirupsen/logrus" + "github.com/sirupsen/logrus" "github.com/spf13/viper" "github.com/uber/jaeger-client-go/config" "google.golang.org/grpc" ) +var log = csmlog.GetLogger() + //go:generate go generate ../../core func init() { // We set X_CSI_DEBUG to false, because we don't want gocsi to override our logging level - _ = os.Setenv(common.EnvGOCSIDebug, "false") + _ = os.Setenv(identifiers.EnvGOCSIDebug, "false") // Enable X_CSI_REQ_LOGGING and X_CSI_REP_LOGGING to see gRPC request information _ = os.Setenv(gocsi.EnvVarReqLogging, "true") _ = os.Setenv(gocsi.EnvVarRepLogging, "true") - paramsPath, ok := csictx.LookupEnv(context.Background(), common.EnvConfigParamsFilePath) - if !ok { - log.Warnf("config path X_CSI_POWERSTORE_CONFIG_PARAMS_PATH is not specified") + updateDriverName() + + initilizeDriverConfigParams() + + // If we don't set this env gocsi will overwrite log level with default Info level + _ = os.Setenv(gocsi.EnvVarLogLevel, csmlog.GetLevel().String()) +} + +func updateDriverName() { + if name, ok := csictx.LookupEnv(context.Background(), identifiers.EnvDriverName); ok { + identifiers.Name = name } +} - if name, ok := csictx.LookupEnv(context.Background(), common.EnvDriverName); ok { - common.Name = name +func initilizeDriverConfigParams() { + log.SetLevel(csmlog.InfoLevel) + paramsPath, ok := csictx.LookupEnv(context.Background(), identifiers.EnvConfigParamsFilePath) + if !ok { + log.Warn("config path X_CSI_POWERSTORE_CONFIG_PARAMS_PATH is not specified") } paramsViper := viper.New() @@ -68,7 +86,7 @@ func init() { err := paramsViper.ReadInConfig() // if unable to read configuration file, default values will be used in updateDriverConfigParams if err != nil { - log.WithError(err).Error("unable to read config file, using default values") + log.Warnf("unable to read config file, using default values %s ", err.Error()) } paramsViper.WatchConfig() paramsViper.OnConfigChange(func(e fsnotify.Event) { @@ -77,90 +95,88 @@ func init() { }) updateDriverConfigParams(paramsViper) - - // If we don't set this env gocsi will overwrite log level with default Info level - err = os.Setenv(gocsi.EnvVarLogLevel, log.GetLevel().String()) - if err != nil { - log.WithError(err).Errorf("unable to set env variable %s", gocsi.EnvVarDebug) - } } +var ManifestSemver string + func main() { + log.SetLevel(csmlog.InfoLevel) f := &fs.Fs{Util: &gofsutil.FS{}} - common.RmSockFile(f) + identifiers.RmSockFile(f) - identityService := identity.NewIdentityService(common.Name, core.SemVer, common.Manifest) + if ManifestSemver != "" { + log.Info("ManifestVersion isn't empty, setting it") + identifiers.ManifestSemver = ManifestSemver + identifiers.Manifest["semver"] = ManifestSemver + } + + identityService := identity.NewIdentityService(identifiers.Name, ManifestSemver, identifiers.Manifest) var controllerService *controller.Service var nodeService *node.Service mode := csictx.Getenv(context.Background(), gocsi.EnvVarMode) - configPath, ok := csictx.LookupEnv(context.Background(), common.EnvArrayConfigFilePath) + configPath, ok := csictx.LookupEnv(context.Background(), identifiers.EnvArrayConfigFilePath) if !ok { log.Fatalf("config path X_CSI_POWERSTORE_CONFIG_PATH is not specified") } - if name, ok := csictx.LookupEnv(context.Background(), common.EnvDriverName); ok { - common.Name = name + if name, ok := csictx.LookupEnv(context.Background(), identifiers.EnvDriverName); ok { + identifiers.Name = name } - common.SetAPIPort(context.Background()) - if strings.EqualFold(mode, "controller") { - cs := &controller.Service{ - Fs: f, - } + identifiers.SetAPIPort(context.Background()) - err := cs.UpdateArrays(configPath, f) - if err != nil { - log.Fatalf("couldn't initialize arrays in controller service: %s", err.Error()) - } + var nodeName string + var arrayLocker *array.Locker + + isCSMDREnabled, err := strconv.ParseBool(os.Getenv(identifiers.EnvCSMDREnabled)) + if err != nil { + log.Infof("Error parsing %s: %s. Defaulting to true", identifiers.EnvCSMDREnabled, err.Error()) + isCSMDREnabled = true + } + + if strings.EqualFold(mode, "controller") { - err = cs.Init() + var err error + controllerService, err = initControllerService(f, configPath) if err != nil { - log.Fatalf("couldn't create controller service: %s", err.Error()) + log.Fatalf("couldn't initialize controller service: %s", err.Error()) } - if cs.K8sVisibilityAutoRegistration { - err = cs.RegisterK8sCluster(f) - if err != nil { - log.Errorf("couldn't register to arrays in controller service: %s", err.Error()) - } - } - controllerService = cs + arrayLocker = &controllerService.Locker + controllerService.IsCSMDREnabled = isCSMDREnabled } else if strings.EqualFold(mode, "node") { - ns := &node.Service{ - Fs: f, - } - - err := ns.UpdateArrays(configPath, f) + var err error + nodeService, err = initNodeService(f, configPath) if err != nil { - log.Fatalf("couldn't initialize arrays in node service: %s", err.Error()) + log.Fatalf("couldn't initialize node service: %s", err.Error()) } - err = ns.Init() + nodeName = os.Getenv(identifiers.EnvKubeNodeName) + arrayLocker = &nodeService.Locker + } + + if isCSMDREnabled { + // Initialize CSM DR volume journal reconciler. + log.Infof("Initializing CSM-DR controller ") + _, err := drController.Initialize(nodeService, controllerService, arrayLocker, mode, nodeName, ":8080", false, ":8081") if err != nil { - log.Fatalf("couldn't create node service: %s", err.Error()) + log.Errorf("[METRO] Unable to initialize volume journal reconciler: %s", err.Error()) } - nodeService = ns } viper.SetConfigFile(configPath) viper.SetConfigType("yaml") viper.WatchConfig() viper.OnConfigChange(func(e fsnotify.Event) { - log.Println("Config file changed:", e.Name) + log.Infof("Config file changed: %s", e.Name) if strings.EqualFold(mode, "controller") { err := controllerService.UpdateArrays(configPath, f) if err != nil { log.Fatalf("couldn't initialize arrays in controller service: %s", err.Error()) } - if controllerService.K8sVisibilityAutoRegistration { - err = controllerService.RegisterK8sCluster(f) - if err != nil { - log.Errorf("couldn't register to arrays in controller service: %s", err.Error()) - } - } } else if strings.EqualFold(mode, "node") { err := nodeService.UpdateArrays(configPath, f) if err != nil { @@ -169,12 +185,12 @@ func main() { } }) - interList := []grpc.UnaryServerInterceptor{ + InterceptorsList := []grpc.UnaryServerInterceptor{ interceptors.NewCustomSerialLock(mode), interceptors.NewRewriteRequestIDInterceptor(), } - if enableTracing, ok := csictx.LookupEnv(context.Background(), common.EnvDebugEnableTracing); ok && enableTracing != "" { + if enableTracing, ok := csictx.LookupEnv(context.Background(), identifiers.EnvDebugEnableTracing); ok && enableTracing != "" { log.Infof("Detected debug flag. Enabling Interceptors..") t, closer, err := tracer.NewTracer(&config.Configuration{}) @@ -183,26 +199,33 @@ func main() { } defer closer.Close() // #nosec G307 opentracing.SetGlobalTracer(t) - interList = append(interList, grpc_opentracing.UnaryServerInterceptor(grpc_opentracing.WithTracer(t))) + InterceptorsList = append(InterceptorsList, grpc_opentracing.UnaryServerInterceptor(grpc_opentracing.WithTracer(t))) } - gocsi.Run(context.Background(), common.Name, + storageProvider := &gocsi.StoragePlugin{ + Controller: controllerService, + Identity: identityService, + Node: nodeService, + Interceptors: InterceptorsList, + RegisterAdditionalServers: controllerService.RegisterAdditionalServers, + + EnvVars: []string{ + // Enable request validation. + gocsi.EnvVarSpecReqValidation + "=true", + // Enable serial volume access. + gocsi.EnvVarSerialVolAccess + "=true", + }, + } + + runCSIPlugin(storageProvider) +} + +var runCSIPlugin = func(storageProvider *gocsi.StoragePlugin) { + gocsi.Run(context.Background(), identifiers.Name, "A PowerStore Container Storage Interface (CSI) Driver", usage, - &gocsi.StoragePlugin{ - Controller: controllerService, - Identity: identityService, - Node: nodeService, - Interceptors: interList, - RegisterAdditionalServers: controllerService.RegisterAdditionalServers, - - EnvVars: []string{ - // Enable request validation. - gocsi.EnvVarSpecReqValidation + "=true", - // Enable serial volume access. - gocsi.EnvVarSerialVolAccess + "=true", - }, - }) + storageProvider, + ) } func updateDriverConfigParams(v *viper.Viper) { @@ -212,15 +235,13 @@ func updateDriverConfigParams(v *viper.Viper) { fmt.Printf("Read CSI_LOG_FORMAT from log configuration file, format: %s\n", logFormat) // Use JSON logger as default - if !strings.EqualFold(logFormat, "text") { - log.SetFormatter(&log.JSONFormatter{ - TimestampFormat: time.RFC3339Nano, + if strings.EqualFold(logFormat, "JSON") { + log.SetFormatter(&logrus.JSONFormatter{ + TimestampFormat: time.RFC3339, }) - } else { - log.SetFormatter(&log.TextFormatter{}) } - level := log.DebugLevel + level := csmlog.DebugLevel if v.IsSet(logLevelParam) { logLevel := v.GetString(logLevelParam) if logLevel != "" { @@ -228,67 +249,113 @@ func updateDriverConfigParams(v *viper.Viper) { fmt.Printf("Read CSI_LOG_LEVEL from log configuration file, level: %s\n", logLevel) var err error - l, err := log.ParseLevel(logLevel) + l, err := csmlog.ParseLevel(logLevel) if err != nil { - log.WithError(err).Errorf("LOG_LEVEL %s value not recognized, setting to default error: %s ", logLevel, err.Error()) + log.Errorf("LOG_LEVEL %s value not recognized, setting to default error: %s ", logLevel, err.Error()) } else { level = l } } } - log.SetLevel(level) + csmlog.SetLevel(level) +} + +func initControllerService(f fs.Interface, configPath string) (*controller.Service, error) { + cs := &controller.Service{ + Fs: f, + } + + err := cs.UpdateArrays(configPath, f) + if err != nil { + return nil, fmt.Errorf("couldn't initialize arrays in controller service: %v", err) + } + + err = cs.Init() + if err != nil { + return nil, fmt.Errorf("couldn't create controller service: %v", err) + } + + ms, err := monitor.NewMonitorService(context.Background()) + if err != nil { + return nil, fmt.Errorf("could not start monitor service: %v", err) + } + err = ms.UpdateArrays(configPath, f) + if err != nil { + return nil, fmt.Errorf("failed to initialize arrays in the monitor service: %v", err) + } + + go ms.Start(context.Background(), 1*time.Minute) + + return cs, nil +} + +func initNodeService(f fs.Interface, configPath string) (*node.Service, error) { + ns := &node.Service{ + Fs: f, + } + + err := ns.UpdateArrays(configPath, f) + if err != nil { + return nil, fmt.Errorf("couldn't initialize arrays in node service: %v", err) + } + + err = ns.Init() + if err != nil { + return nil, fmt.Errorf("couldn't create node service: %v", err) + } + return ns, nil } const usage = ` - X_CSI_POWERSTORE_INSECURE - Specifies that the PowerStore's hostname and certificate chain - should not be verified. - - The default value is false. - - X_CSI_POWERSTORE_NODE_ID_PATH - Specifies the name of the text file contents of which will - be appended to the node ID - - X_CSI_POWERSTORE_KUBE_NODE_NAME - Specifies the name of the kubernetes node - - X_CSI_POWERSTORE_NODE_NAME_PREFIX - Specifies prefix which will be used when registering node - on PowerStore array - - X_CSI_POWERSTORE_NODE_CHROOT_PATH - Specifies path to chroot where to execute iSCSI commands - - X_CSI_POWERSTORE_TMP_DIR - Specifies path to the folder which will be used for csi-powerstore temporary files - - X_CSI_FC_PORTS_FILTER_FILE_PATH - Specifies path to the file which provide list of WWPN which - should be used by the driver for FC connection on this node - example content of the file: - 21:00:00:29:ff:48:9f:6e,21:00:00:29:ff:48:9f:6e - If file does not exist, empty or in invalid format, - then the driver will use all available FC ports - - X_CSI_POWERSTORE_THROTTLING_RATE_LIMIT - Specifies a number of concurrent requests to one storage API - - X_CSI_POWERSTORE_ENABLE_CHAP - Specifies whether driver should set CHAP credentials in the ISCSI - node database at the time of node plugin boot - - X_CSI_POWERSTORE_EXTERNAL_ACCESS - Specifies an IP of the additional router you wish to add for nfs export - Used to provide NFS volumes behind NAT - - X_CSI_POWERSTORE_CONFIG_PATH - Specifies the filepath to PowerStore arrays config file which will be used - for connection to PowerStore arrays - - X_CSI_REPLICATION_CONTEXT_PREFIX - Enables sidecars to read required information from volume context - - X_CSI_REPLICATION_PREFIX - Used as a prefix to find out if replication is enabled -` + X_CSI_POWERSTORE_INSECURE + Specifies that the PowerStore's hostname and certificate chain + should not be verified. + + The default value is false. + + X_CSI_POWERSTORE_NODE_ID_PATH + Specifies the name of the text file contents of which will + be appended to the node ID + + X_CSI_POWERSTORE_KUBE_NODE_NAME + Specifies the name of the kubernetes node + + X_CSI_POWERSTORE_NODE_NAME_PREFIX + Specifies prefix which will be used when registering node + on PowerStore array + + X_CSI_POWERSTORE_NODE_CHROOT_PATH + Specifies path to chroot where to execute iSCSI commands + + X_CSI_POWERSTORE_TMP_DIR + Specifies path to the folder which will be used for csi-powerstore temporary files + + X_CSI_FC_PORTS_FILTER_FILE_PATH + Specifies path to the file which provide list of WWPN which + should be used by the driver for FC connection on this node + example content of the file: + 21:00:00:29:ff:48:9f:6e,21:00:00:29:ff:48:9f:6e + If file does not exist, empty or in invalid format, + then the driver will use all available FC ports + + X_CSI_POWERSTORE_THROTTLING_RATE_LIMIT + Specifies a number of concurrent requests to one storage API + + X_CSI_POWERSTORE_ENABLE_CHAP + Specifies whether driver should set CHAP credentials in the ISCSI + node database at the time of node plugin boot + + X_CSI_POWERSTORE_EXTERNAL_ACCESS + Specifies an IP of the additional router you wish to add for nfs export + Used to provide NFS volumes behind NAT + + X_CSI_POWERSTORE_CONFIG_PATH + Specifies the filepath to PowerStore arrays config file which will be used + for connection to PowerStore arrays + + X_CSI_REPLICATION_CONTEXT_PREFIX + Enables sidecars to read required information from volume context + + X_CSI_REPLICATION_PREFIX + Used as a prefix to find out if replication is enabled + ` diff --git a/cmd/csi-powerstore/main_test.go b/cmd/csi-powerstore/main_test.go new file mode 100644 index 00000000..f158618a --- /dev/null +++ b/cmd/csi-powerstore/main_test.go @@ -0,0 +1,395 @@ +/* + * + * Copyright © 2021-2026 Dell Inc. or its subsidiaries. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package main + +import ( + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/dell/csi-powerstore/v2/mocks" + "github.com/dell/csi-powerstore/v2/pkg/controller" + "github.com/dell/csi-powerstore/v2/pkg/identifiers" + "github.com/dell/csi-powerstore/v2/pkg/identifiers/fs" + "github.com/dell/csi-powerstore/v2/pkg/identifiers/k8sutils" + "github.com/dell/csi-powerstore/v2/pkg/node" + "github.com/dell/csmlog" + "github.com/dell/gocsi" + "github.com/fsnotify/fsnotify" + "github.com/sirupsen/logrus" + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/rest" +) + +func TestUpdateDriverName(t *testing.T) { + tests := []struct { + name string + envVar string + expected string + }{ + { + name: "Environment variable is present", + envVar: "test-driver", + expected: "test-driver", + }, + { + name: "Environment variable is not present", + envVar: "", + expected: "", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Setenv(identifiers.EnvDriverName, tc.envVar) + + updateDriverName() + + assert.Equal(t, tc.expected, identifiers.Name) + }) + } +} + +func TestInitilizeDriverConfigParams(t *testing.T) { + tmpDir := t.TempDir() + content := `CSI_LOG_FORMAT: "JSON"` + driverConfigParams := filepath.Join(tmpDir, "driver-config-params.yaml") + writeToFile(t, driverConfigParams, content) + t.Setenv(identifiers.EnvConfigParamsFilePath, driverConfigParams) + initilizeDriverConfigParams() + assert.Equal(t, csmlog.DebugLevel, csmlog.GetLevel()) + writeToFile(t, driverConfigParams, "CSI_LOG_LEVEL: \"info\"") + time.Sleep(time.Second) + assert.Equal(t, csmlog.InfoLevel, csmlog.GetLevel()) +} + +func TestMainControllerMode(t *testing.T) { + tmpDir := t.TempDir() + config := copyConfigFileToTmpDir(t, "../../pkg/array/testdata/one-arr.yaml", tmpDir) + + defaultK8sConfigFunc := k8sutils.InClusterConfigFunc + defaultK8sClientsetFunc := k8sutils.NewForConfigFunc + + k8sutils.InClusterConfigFunc = func() (*rest.Config, error) { + return &rest.Config{}, nil + } + k8sutils.NewForConfigFunc = func(_ *rest.Config) (kubernetes.Interface, error) { + return fake.NewClientset(), nil + } + + defer func() { + k8sutils.InClusterConfigFunc = defaultK8sConfigFunc + k8sutils.NewForConfigFunc = defaultK8sClientsetFunc + }() + + // Set Manifest version similar to how the image would be built. + ManifestSemver = "1.0.0" + + // Set required environment variables + t.Setenv(identifiers.EnvArrayConfigFilePath, config) + t.Setenv("CSI_ENDPOINT", "mock_endpoint") + t.Setenv(identifiers.EnvDriverName, "test") + t.Setenv(identifiers.EnvDebugEnableTracing, "true") + t.Setenv("JAEGER_SERVICE_NAME", "controller-test") + t.Setenv(string(gocsi.EnvVarMode), "controller") + t.Setenv(identifiers.EnvCSMDREnabled, "true") + + array2 := ` - endpoint: "https://127.0.0.2/api/rest" + username: "admin" + globalID: "gid2" + password: "password" + skipCertificateValidation: true + blockProtocol: "auto" + isDefault: false` + + runCSIPlugin = func(test *gocsi.StoragePlugin) { + // Assertions + require.NotNil(t, test.Controller) + require.NotNil(t, test.Identity) + require.Nil(t, test.Node) + require.EqualValues(t, 1, len(test.Controller.(*controller.Service).Arrays())) + + // Update the config file + writeToFile(t, config, array2) + time.Sleep(time.Second) + + // Assertions + require.EqualValues(t, 2, len(test.Controller.(*controller.Service).Arrays())) + } + + defer func() { + if r := recover(); r != nil { + t.Fatalf("the code panicked with error: %v", r) + } + }() + + main() +} + +func TestMainNodeMode(t *testing.T) { + tmpDir := t.TempDir() + config := copyConfigFileToTmpDir(t, "../../pkg/array/testdata/one-arr.yaml", tmpDir) + + defaultK8sConfigFunc := k8sutils.InClusterConfigFunc + defaultK8sClientsetFunc := k8sutils.NewForConfigFunc + + k8sutils.InClusterConfigFunc = func() (*rest.Config, error) { + return &rest.Config{}, nil + } + k8sutils.NewForConfigFunc = func(_ *rest.Config) (kubernetes.Interface, error) { + return fake.NewClientset(), nil + } + + defer func() { + k8sutils.InClusterConfigFunc = defaultK8sConfigFunc + k8sutils.NewForConfigFunc = defaultK8sClientsetFunc + }() + + // Set required environment variables + t.Setenv(identifiers.EnvArrayConfigFilePath, config) + t.Setenv(gocsi.EnvVarMode, "node") + t.Setenv(identifiers.EnvDebugEnableTracing, "") + tempNodeIDFile, err := os.CreateTemp(tmpDir, "node-id") + require.NoError(t, err) + t.Setenv("X_CSI_POWERSTORE_NODE_ID_PATH", tempNodeIDFile.Name()) + + array2 := ` - endpoint: "https://127.0.0.2/api/rest" + username: "admin" + globalID: "gid2" + password: "password" + skipCertificateValidation: true + blockProtocol: "auto" + isDefault: false` + + runCSIPlugin = func(test *gocsi.StoragePlugin) { + // Assertions + require.Nil(t, test.Controller) + require.NotNil(t, test.Identity) + require.NotNil(t, test.Node) + require.EqualValues(t, 1, len(test.Node.(*node.Service).Arrays())) + + // Update the config file + writeToFile(t, config, array2) + time.Sleep(time.Second) + + // Assertions + require.EqualValues(t, 2, len(test.Node.(*node.Service).Arrays())) + } + + defer func() { + if r := recover(); r != nil { + t.Fatalf("the code panicked with error: %v", r) + } + }() + + main() +} + +func copyConfigFileToTmpDir(t *testing.T, src string, tmpDir string) string { + t.Helper() + + srcF, err := os.Open(src) + require.NoError(t, err) + defer srcF.Close() + + dstF, err := os.CreateTemp(tmpDir, "config_*.yaml") + require.NoError(t, err) + defer dstF.Close() + + _, err = io.Copy(dstF, srcF) + require.NoError(t, err) + + return dstF.Name() +} + +func writeToFile(t *testing.T, controllerConfigFile string, array2 string) { + f, err := os.OpenFile(controllerConfigFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + t.Errorf("failed to open confg file %s, err %v", controllerConfigFile, err) + } else { + defer f.Close() + _, err = f.WriteString(array2 + "\n") + if err != nil { + t.Errorf("failed to update confg file %s, err %v", controllerConfigFile, err) + } + } +} + +func TestUpdateDriverConfigParams(t *testing.T) { + v := viper.New() + v.SetConfigType("yaml") + v.SetDefault("CSI_LOG_FORMAT", "text") + v.SetDefault("CSI_LOG_LEVEL", "debug") + + viperChan := make(chan bool) + v.WatchConfig() + v.OnConfigChange(func(_ fsnotify.Event) { + updateDriverConfigParams(v) + viperChan <- true + }) + + logFormat := strings.ToLower(v.GetString("CSI_LOG_FORMAT")) + assert.Equal(t, "text", logFormat) + + updateDriverConfigParams(v) + level := csmlog.GetLevel() + + assert.Equal(t, csmlog.DebugLevel, level) + + v.Set("CSI_LOG_FORMAT", "json") + v.Set("CSI_LOG_LEVEL", "info") + updateDriverConfigParams(v) + level = csmlog.GetLevel() + + assert.Equal(t, csmlog.InfoLevel, level) + logFormatter := &csmlog.MyTextFormatter{ + Base: &logrus.TextFormatter{ + TimestampFormat: time.RFC3339, + }, + } + assert.Equal(t, time.RFC3339, logFormatter.Base.TimestampFormat) + + v.Set("CSI_LOG_LEVEL", "notalevel") + updateDriverConfigParams(v) + level = csmlog.GetLevel() + assert.Equal(t, csmlog.DebugLevel, level) +} + +func Test_initControllerService(t *testing.T) { + tests := []struct { + name string // description of this test case + // Named input parameters for target function. + init func() + f func() fs.Interface + configPath string + want *controller.Service + wantErr bool + }{ + { + name: "fail to update arrays", + init: func() {}, + f: func() fs.Interface { + fs := &mocks.FsInterface{} + fs.On("ReadFile", ".").Return([]byte{}, errors.New("read error")) + return fs + }, + configPath: "", + want: nil, + wantErr: true, + }, + { + name: "fail to initialize the controller service", + init: func() { + tempNewForConfigFunc := k8sutils.NewForConfigFunc + k8sutils.NewForConfigFunc = func(_ *rest.Config) (kubernetes.Interface, error) { + return nil, errors.New("new for config error") + } + t.Cleanup(func() { + k8sutils.NewForConfigFunc = tempNewForConfigFunc + }) + }, + f: func() fs.Interface { + fs := &mocks.FsInterface{} + fs.On("ReadFile", "/some/config.yaml").Return([]byte{}, nil) + return fs + }, + configPath: "/some/config.yaml", + want: nil, + wantErr: true, + }, + { + name: "fail to initialize the monitor service arrays", + init: func() { + tempNewForConfigFunc := k8sutils.NewForConfigFunc + k8sutils.NewForConfigFunc = func(_ *rest.Config) (kubernetes.Interface, error) { + return fake.NewClientset(), nil + } + tempInClusterConfigFunc := k8sutils.InClusterConfigFunc + k8sutils.InClusterConfigFunc = func() (*rest.Config, error) { + return nil, nil + } + t.Cleanup(func() { + k8sutils.NewForConfigFunc = tempNewForConfigFunc + k8sutils.InClusterConfigFunc = tempInClusterConfigFunc + }) + }, + f: func() fs.Interface { + fs := &mocks.FsInterface{} + fs.On("ReadFile", ".").Once().Return([]byte{}, nil) + fs.On("ReadFile", ".").Once().Return([]byte{}, errors.New("monitor read error")) + return fs + }, + configPath: "", + want: nil, + wantErr: true, + }, + { + name: "success", + init: func() { + tempNewForConfigFunc := k8sutils.NewForConfigFunc + k8sutils.NewForConfigFunc = func(_ *rest.Config) (kubernetes.Interface, error) { + return fake.NewClientset(), nil + } + tempInClusterConfigFunc := k8sutils.InClusterConfigFunc + k8sutils.InClusterConfigFunc = func() (*rest.Config, error) { + return nil, nil + } + t.Cleanup(func() { + k8sutils.NewForConfigFunc = tempNewForConfigFunc + k8sutils.InClusterConfigFunc = tempInClusterConfigFunc + }) + }, + f: func() fs.Interface { + fs := &mocks.FsInterface{} + fs.On("ReadFile", ".").Return([]byte{}, nil) + return fs + }, + configPath: "", + want: &controller.Service{ + Fs: &mocks.FsInterface{}, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.init() + got, gotErr := initControllerService(tt.f(), tt.configPath) + if gotErr != nil { + if !tt.wantErr { + t.Errorf("initControllerService() failed: %v", gotErr) + } + return + } + if tt.wantErr { + t.Fatal("initControllerService() succeeded unexpectedly") + } + + if got == nil { + t.Error("initControllerService() expected a service struct but got nil") + } + }) + } +} diff --git a/core/core_test.go b/core/core_test.go new file mode 100644 index 00000000..98c32579 --- /dev/null +++ b/core/core_test.go @@ -0,0 +1,67 @@ +/* + * + * Copyright © 2021-2025 Dell Inc. or its subsidiaries. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package core + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestCoreVariables(t *testing.T) { + // Check if SemVer is set to a non-default value + if SemVer != "unknown" { + assert.NotEqual(t, "unknown", SemVer, "SemVer should not be 'unknown' if set during build") + } else { + assert.Equal(t, "unknown", SemVer, "SemVer should be 'unknown' by default") + } + + // Check if CommitSha7 is set to a non-default value + if CommitSha7 != "" { + assert.NotEmpty(t, CommitSha7, "CommitSha7 should not be empty if set during build") + } else { + assert.Empty(t, CommitSha7, "CommitSha7 should be empty by default") + } + + // Check if CommitSha32 is set to a non-default value + if CommitSha32 != "" { + assert.NotEmpty(t, CommitSha32, "CommitSha32 should not be empty if set during build") + } else { + assert.Empty(t, CommitSha32, "CommitSha32 should be empty by default") + } + + // Check if CommitTime is set to a non-default value + if !CommitTime.IsZero() { + assert.False(t, CommitTime.IsZero(), "CommitTime should not be zero if set during build") + } else { + assert.True(t, CommitTime.IsZero(), "CommitTime should be zero by default") + } + + // Test setting values + SemVer = "1.0.0" + CommitSha7 = "abcdefg" + CommitSha32 = "abcdefg1234567890abcdefg1234567890" + CommitTime = time.Now() + + assert.Equal(t, "1.0.0", SemVer, "SemVer should be '1.0.0'") + assert.Equal(t, "abcdefg", CommitSha7, "CommitSha7 should be 'abcdefg'") + assert.Equal(t, "abcdefg1234567890abcdefg1234567890", CommitSha32, "CommitSha32 should be 'abcdefg1234567890abcdefg1234567890'") + assert.False(t, CommitTime.IsZero(), "CommitTime should not be zero") +} diff --git a/core/semver/semver.go b/core/semver/semver.go index 472f7c3a..af196c09 100644 --- a/core/semver/semver.go +++ b/core/semver/semver.go @@ -1,20 +1,16 @@ /* - * - * Copyright © 2020-2023 Dell Inc. or its subsidiaries. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ + Copyright © 2021 Dell Inc. or its subsidiaries. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ package main @@ -36,30 +32,46 @@ import ( "time" ) +var ( + format string + output string + export bool + tpl *template.Template +) + +func init() { + if flag.Lookup("f") == nil { + flag.StringVar( + &format, + "f", + "ver", + "The output format: env, go, json, mk, rpm, ver") + } + if flag.Lookup("o") == nil { + flag.StringVar( + &output, + "o", + "", + "The output file") + } + if flag.Lookup("x") == nil { + flag.BoolVar( + &export, + "x", + false, + "Export env vars. Used with -f env") + } +} + +func initFlags() { + format = flag.Lookup("f").Value.(flag.Getter).Get().(string) + output = flag.Lookup("o").Value.(flag.Getter).Get().(string) + export = flag.Lookup("x").Value.(flag.Getter).Get().(bool) +} + func main() { - var ( - tpl *template.Template - format string - output string - export bool - ) - - flag.StringVar( - &format, - "f", - "ver", - "The output format: env, go, json, mk, rpm, ver") - flag.StringVar( - &output, - "o", - "", - "The output file") - flag.BoolVar( - &export, - "x", - false, - "Export env vars. Used with -f env") flag.Parse() + initFlags() if strings.EqualFold("env", format) { format = "env" @@ -75,10 +87,9 @@ func main() { format = "ver" } else { if fileExists(format) { - buf, err := os.ReadFile(format) // #nosec G304 + buf, err := ReadFile(format) // #nosec G304 if err != nil { - fmt.Fprintf(os.Stderr, "error: read tpl failed: %v\n", err) - os.Exit(1) + errorExit(fmt.Sprintf("error: read tpl failed: %v\n", err)) } format = string(buf) } @@ -90,15 +101,14 @@ func main() { if len(output) > 0 { fout, err := os.Create(filepath.Clean(output)) if err != nil { - fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) + errorExit(fmt.Sprintf("error: %v\n", err)) } w = fout defer func() { if err := fout.Close(); err != nil { panic(err) } - }() + }() // #nosec G20 } gitdesc := chkErr(doExec("git", "describe", "--long", "--dirty")) @@ -106,8 +116,7 @@ func main() { `^[^\d]*(\d+)\.(\d+)\.(\d+)(?:-([a-zA-Z].+?))?(?:-(\d+)-g(.+?)(?:-(dirty))?)?\s*$`) m := rx.FindStringSubmatch(gitdesc) if len(m) == 0 { - fmt.Fprintf(os.Stderr, "error: match git describe failed: %s\n", gitdesc) - os.Exit(1) + errorExit(fmt.Sprintf("error: match git describe failed: %s\n", gitdesc)) } goos := os.Getenv("XGOOS") @@ -118,7 +127,16 @@ func main() { if goarch == "" { goarch = runtime.GOARCH } - + // get the build number. Jenkins exposes this as an + // env variable called BUILD_NUMBER + buildNumber := os.Getenv("BUILD_NUMBER") + if buildNumber == "" { + buildNumber = m[5] + } + buildType := os.Getenv("BUILD_TYPE") + if buildType == "" { + buildType = "X" + } ver := &semver{ GOOS: goos, GOARCH: goarch, @@ -128,6 +146,8 @@ func main() { Minor: toInt(m[2]), Patch: toInt(m[3]), Notes: m[4], + Type: buildType, + Build: toInt(buildNumber), Sha7: m[6], Sha32: chkErr(doExec("git", "log", "-n1", `--format=%H`)), Dirty: m[7] != "", @@ -137,6 +157,7 @@ func main() { ver.SemVerRPM = ver.RPM() ver.BuildDate = ver.Timestamp().Format("Mon, 02 Jan 2006 15:04:05 MST") ver.ReleaseDate = ver.Timestamp().Format("06-01-02") + switch format { case "env": for _, v := range ver.EnvVars() { @@ -150,8 +171,7 @@ func main() { enc := json.NewEncoder(w) enc.SetIndent("", " ") if err := enc.Encode(ver); err != nil { - fmt.Fprintf(os.Stderr, "error: encode to json failed: %v\n", err) - os.Exit(1) + errorExit(fmt.Sprintf("error: encode to json failed: %v\n", err)) } case "mk": for _, v := range ver.EnvVars() { @@ -160,50 +180,54 @@ func main() { fmt.Fprintf(w, "%s ?=", key) if len(p) == 1 { fmt.Fprintln(w) - continue - } - val := p[1] - if strings.HasPrefix(val, `"`) && - strings.HasSuffix(val, `"`) { - val = val[1 : len(val)-1] + } else { + val := p[1] + if strings.HasPrefix(val, `"`) && + strings.HasSuffix(val, `"`) { + val = val[1 : len(val)-1] + } + val = strings.Replace(val, "$", "$$", -1) + fmt.Fprintf(w, " %s\n", val) } - val = strings.Replace(val, "$", "$$", -1) - fmt.Fprintf(w, " %s\n", val) } case "rpm": fmt.Fprintln(w, ver.RPM()) case "tpl": if err := tpl.Execute(w, ver); err != nil { - fmt.Fprintf(os.Stderr, "error: template failed: %v\n", err) - os.Exit(1) + errorExit(fmt.Sprintf("error: template failed: %v\n", err)) } case "ver": fmt.Fprintln(w, ver.String()) } } -func doExec(cmd string, args ...string) ([]byte, error) { +var doExec = func(cmd string, args ...string) ([]byte, error) { c := exec.Command(cmd, args...) // #nosec G204 c.Stderr = os.Stderr return c.Output() } +func errorExit(message string) { + fmt.Fprintf(os.Stderr, "%s", message) + OSExit(1) +} + func chkErr(out []byte, err error) string { if err == nil { return strings.TrimSpace(string(out)) } - e, ok := err.(*exec.ExitError) + e, ok := GetExitError(err) if !ok { - os.Exit(1) + OSExit(1) } - st, ok := e.Sys().(syscall.WaitStatus) + status, ok := GetStatusError(e) if !ok { - os.Exit(1) + OSExit(1) } - os.Exit(st.ExitStatus()) + OSExit(status) return "" } @@ -215,7 +239,9 @@ type semver struct { Major int `json:"major"` Minor int `json:"minor"` Patch int `json:"patch"` + Build int `json:"build"` Notes string `json:"notes"` + Type string `json:"type"` Dirty bool `json:"dirty"` Sha7 string `json:"sha7"` Sha32 string `json:"sha32"` @@ -230,7 +256,10 @@ func (v *semver) String() string { buf := &bytes.Buffer{} fmt.Fprintf(buf, "%d.%d.%d", v.Major, v.Minor, v.Patch) if len(v.Notes) > 0 { - fmt.Fprintf(buf, "%s", v.Notes) + fmt.Fprintf(buf, "-%s", v.Notes) + } + if v.Build > 0 { + fmt.Fprintf(buf, "+%d", v.Build) } if v.Dirty { fmt.Fprint(buf, "+dirty") @@ -251,7 +280,9 @@ func (v *semver) EnvVars() []string { fmt.Sprintf("MAJOR=%d", v.Major), fmt.Sprintf("MINOR=%d", v.Minor), fmt.Sprintf("PATCH=%d", v.Patch), + fmt.Sprintf("BUILD=%3.3d", v.Build), fmt.Sprintf("NOTES=\"%s\"", v.Notes), + fmt.Sprintf("TYPE=%s", v.Type), fmt.Sprintf("DIRTY=%v", v.Dirty), fmt.Sprintf("SHA7=%s", v.Sha7), fmt.Sprintf("SHA32=%s", v.Sha32), @@ -307,14 +338,32 @@ var goarchToUname = map[string]string{ } func fileExists(filePath string) bool { - _, err := os.Stat(filePath) - if err == nil { + if _, err := os.Stat(filePath); !os.IsNotExist(err) { return true } - if os.IsNotExist(err) { - fmt.Printf("File %s doesn't exist", filePath) - } else { - fmt.Printf("Found error %v while checking stat of file %s ", err, filePath) - } return false } + +// ReadFile is a wrapper around os.ReadFile +var ReadFile = func(file string) ([]byte, error) { + return os.ReadFile(file) // #nosec G304 +} + +// OSExit is a wrapper around os.Exit +var OSExit = func(code int) { + os.Exit(code) +} + +// GetExitError is a wrapper around exec.ExitError +var GetExitError = func(err error) (e *exec.ExitError, ok bool) { + e, ok = err.(*exec.ExitError) + return e, ok +} + +// GetStatusError is a wrapper around syscall.WaitStatus +var GetStatusError = func(exitError *exec.ExitError) (status int, ok bool) { + if e, ok := exitError.Sys().(syscall.WaitStatus); ok { + return e.ExitStatus(), true + } + return 1, false +} diff --git a/core/semver/semver_test.go b/core/semver/semver_test.go new file mode 100644 index 00000000..2a2f93b3 --- /dev/null +++ b/core/semver/semver_test.go @@ -0,0 +1,300 @@ +/* + Copyright © 2025 Dell Inc. or its subsidiaries. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "errors" + "fmt" + "io" + "os" + "os/exec" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetStatusError(_ *testing.T) { + exitError := &exec.ExitError{ + ProcessState: &os.ProcessState{}, + } + _, _ = GetStatusError(exitError) +} + +func TestString(t *testing.T) { + s := semver{"", "", "", "", 1, 2, 3, 4, "", "", true, "", "", 64, "", "", "", ""} + assert.NotNil(t, s.String()) + + s = semver{"", "", "", "", 1, 2, 3, 4, "abc", "", true, "", "", 64, "", "", "", ""} + assert.NotNil(t, s.String()) +} + +func TestGetExitError(t *testing.T) { + err := errors.New("error") + _, ok := GetExitError(err) + assert.False(t, ok) +} + +func TestMainFunction(t *testing.T) { + tests := []struct { + name string + format string + outputFile string + expectEmptyFile bool + readFileFunc func(file string) ([]byte, error) + }{ + { + name: "Write mk format to file", + format: "mk", + outputFile: "test_output.mk", + }, + { + name: "Write env format to file", + format: "env", + outputFile: "test_output.env", + }, + { + name: "Write json format to file", + format: "json", + outputFile: "test_output.json", + }, + { + name: "Write ver format to file", + format: "ver", + outputFile: "test_output.ver", + }, + { + name: "Write rpm format to file", + format: "rpm", + outputFile: "test_output.rpm", + }, + { + name: "Write tpl format to file", + format: "../semver.tpl", + outputFile: "test_output.rpm", + }, + { + name: "Write tpl format to file but error reading source file", + format: "../semver.tpl", + outputFile: "test_output.rpm", + readFileFunc: func(_ string) ([]byte, error) { + return nil, errors.New("error reading source file") + }, + expectEmptyFile: true, + }, + { + // go format currently does not print any output, expect an empty file + name: "Write go format to file", + format: "go", + outputFile: "test_output.go", + expectEmptyFile: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + osArgs := os.Args + os.Args = append(os.Args, "-f", tt.format) + os.Args = append(os.Args, "-o", tt.outputFile) + os.Args = append(os.Args, "-x", "true") + + oldReadFile := ReadFile + if tt.readFileFunc != nil { + ReadFile = tt.readFileFunc + } + oldOSExit := OSExit + OSExit = func(_ int) {} + + oldDoExec := doExec + doExec = func(_ string, _ ...string) ([]byte, error) { + return []byte("v2.15.0-77-g38b3a19-dirty"), nil + } + + main() + + // Open the file + file, err := os.Open(tt.outputFile) + if err != nil { + t.Error(err) + } + defer file.Close() + + // Read the file contents + contents, err := io.ReadAll(file) + if err != nil { + t.Error(err) + } + + defer os.Remove(tt.outputFile) + + // make sure file is not empty + if tt.expectEmptyFile { + assert.Equal(t, 0, len(contents)) + } else { + assert.NotEqual(t, 0, len(contents)) + } + os.Args = osArgs + ReadFile = oldReadFile + OSExit = oldOSExit + doExec = oldDoExec + }) + } +} + +func TestChkErr(t *testing.T) { + tests := []struct { + name string + out []byte + err error + wantOut string + wantErr bool + getExitError func(err error) (*exec.ExitError, bool) + getStatusError func(exitError *exec.ExitError) (int, bool) + }{ + { + name: "No error", + out: []byte("output"), + err: nil, + wantOut: "output", + wantErr: false, + getExitError: func(_ error) (*exec.ExitError, bool) { + return nil, true + }, + getStatusError: func(_ *exec.ExitError) (int, bool) { + return 0, true + }, + }, + { + name: "Error with command", + out: []byte("output"), + err: errors.New("error"), + wantOut: "", + wantErr: true, + getExitError: func(_ error) (*exec.ExitError, bool) { + return nil, false + }, + getStatusError: func(_ *exec.ExitError) (int, bool) { + return 1, false + }, + }, + { + name: "Error casting to ExitError", + out: []byte("output"), + err: errors.New("error"), + wantOut: "", + wantErr: true, + getExitError: func(_ error) (*exec.ExitError, bool) { + return nil, true + }, + getStatusError: func(_ *exec.ExitError) (int, bool) { + return 1, false + }, + }, + { + name: "Error getting status from ExitError", + out: []byte("output"), + err: errors.New("error"), + wantOut: "", + wantErr: true, + getExitError: func(_ error) (*exec.ExitError, bool) { + return nil, false + }, + getStatusError: func(_ *exec.ExitError) (int, bool) { + return 0, true + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + GetExitError = tt.getExitError + GetStatusError = tt.getStatusError + OSExit = func(_ int) {} + + gotOut := chkErr(tt.out, tt.err) + if gotOut != tt.wantOut { + t.Errorf("chkErr() gotOut = %v, want %v", gotOut, tt.wantOut) + } + }) + } +} + +func TestFileExists(t *testing.T) { + tests := []struct { + name string + filePath string + want bool + }{ + { + name: "File exists", + filePath: "semver.go", + want: true, + }, + { + name: "File does not exist", + filePath: "non-existent.txt", + want: false, + }, + { + name: "File path is empty", + filePath: "", + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := fileExists(tt.filePath) + if got != tt.want { + t.Errorf("fileExists(%s) = %v, want %v", tt.filePath, got, tt.want) + } + }) + } +} + +func TestErrorExit(t *testing.T) { + message := "error message" + + if os.Getenv("INVOKE_ERROR_EXIT") == "1" { + errorExit(message) + return + } + // call the test again with INVOKE_ERROR_EXIT=1 so the errorExit function is invoked and we can check the return code + cmd := exec.Command(os.Args[0], "-test.run=TestErrorExit") // #nosec G204 + cmd.Env = append(os.Environ(), "INVOKE_ERROR_EXIT=1") + + stderr, err := cmd.StderrPipe() + if err != nil { + fmt.Println("Error creating stderr pipe:", err) + return + } + + if err := cmd.Start(); err != nil { + t.Error(err) + } + + buf := make([]byte, 1024) + n, err := stderr.Read(buf) + if err != nil { + t.Error(err) + } + + err = cmd.Wait() + if e, ok := err.(*exec.ExitError); ok && e.Success() { + t.Error(err) + } + + // check the output is the message we logged in errorExit + assert.Equal(t, message, string(buf[:n])) +} diff --git a/dell-csi-helm-installer/README.md b/dell-csi-helm-installer/README.md index 50a3135c..f3fbdce7 100644 --- a/dell-csi-helm-installer/README.md +++ b/dell-csi-helm-installer/README.md @@ -19,7 +19,7 @@ Installing any of the Dell EMC CSI Drivers requires a few utilities to be instal | Dependency | Usage | | ------------- | ----- | | `kubectl` | Kubectl is used to validate that the Kubernetes system meets the requirements of the driver. | -| `helm` | Helm v3 is used as the deployment tool for Charts. See, [Install HELM 3](https://helm.sh/docs/intro/install/) for instructions to install HELM 3. | +| `helm` | Helm v3 is used as the deployment tool for Charts. See, [Install Helm 3](https://helm.sh/docs/intro/install/) for instructions to install Helm 3. | | `sshpass` | sshpass is used to check certain pre-requisities in worker nodes (in chosen drivers). | @@ -36,7 +36,7 @@ This project provides the following capabilitites, each one is discussed in deta Most of these usages require the creation/specification of a values file. These files specify configuration settings that are passed into the driver and configure it for use. To create one of these files, the following steps should be followed: -1. Download a template file for the driver to a new location, naming this new file is at the users discretion. The template files are always found at `https://github.com/dell/helm-charts/raw/csi-powerstore-2.8.0/charts/csi-powerstore/values.yaml` +1. Download a template file for the driver to a new location, naming this new file is at the users discretion. The template files are always found at `https://github.com/dell/helm-charts/raw/csi-powerstore-2.15.0/charts/csi-powerstore/values.yaml` 2. Edit the file such that it contains the proper configuration settings for the specific environment. These files are yaml formatted so maintaining the file structure is important. For example, to create a values file for the PowerStore driver the following steps can be executed @@ -45,7 +45,7 @@ For example, to create a values file for the PowerStore driver the following ste cd dell-csi-helm-installer # download the template file -wget -O my-powerstore-settings.yaml https://github.com/dell/helm-charts/raw/csi-powerstore-2.8.0/charts/csi-powerstore/values.yaml +wget -O my-powerstore-settings.yaml https://github.com/dell/helm-charts/raw/csi-powerstore-2.15.0/charts/csi-powerstore/values.yaml # edit the newly created values file vi my-powerstore-settings.yaml diff --git a/dell-csi-helm-installer/csi-install.sh b/dell-csi-helm-installer/csi-install.sh index a982488b..14c8756a 100755 --- a/dell-csi-helm-installer/csi-install.sh +++ b/dell-csi-helm-installer/csi-install.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright © 2020-2023 Dell Inc., or its subsidiaries. All Rights Reserved. +# Copyright © 2020-2025 Dell Inc., or its subsidiaries. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -10,14 +10,14 @@ SCRIPTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" DRIVERDIR="${SCRIPTDIR}/../" -HELMCHARTVERSION="csi-powerstore-2.8.0" +HELMCHARTVERSION="csi-powerstore-2.15.0" DRIVER="csi-powerstore" VERIFYSCRIPT="${SCRIPTDIR}/verify.sh" PROG="${0}" NODE_VERIFY=1 VERIFY=1 MODE="install" -DEFAULT_DRIVER_VERSION="v2.8.0" +DEFAULT_DRIVER_VERSION="v2.15.0" WATCHLIST="" # export the name of the debug log, so child processes will see it @@ -141,11 +141,21 @@ function install_driver() { fi HELMOUTPUT="/tmp/csi-install.$$.out" + + # install all of the helm dependencies + log step "Installing helm dependencies" + run_command helm dependency build "${DRIVERDIR}/${DRIVER}" >>"${HELMOUTPUT}" 2>&1 + if [ $? -ne 0 ]; then + cat "${HELMOUTPUT}" + log error "Failed to install ${DRIVER} helm dependencies" + fi + + # install the driver run_command helm ${1} \ --set openshift=${OPENSHIFT} \ --values "${VALUES}" \ --namespace ${NS} "${RELEASE}" \ - "${DRIVERDIR}/${DRIVER}" >"${HELMOUTPUT}" 2>&1 + "${DRIVERDIR}/${DRIVER}" >>"${HELMOUTPUT}" 2>&1 if [ $? -ne 0 ]; then cat "${HELMOUTPUT}" @@ -404,6 +414,7 @@ OPENSHIFT=$(isOpenShift) # Get the kubernetes major and minor version numbers. kMajorVersion=$(run_command kubectl version | grep 'Server Version' | sed -E 's/.*v([0-9]+)\.[0-9]+\.[0-9]+.*/\1/') kMinorVersion=$(run_command kubectl version | grep 'Server Version' | sed -E 's/.*v[0-9]+\.([0-9]+)\.[0-9]+.*/\1/') +kNonGAVersion=$(run_command kubectl version | grep 'Server Version' | sed -n 's/.*\(-[alpha|beta][^ ]*\).*/\1/p') # validate the parameters passed in diff --git a/dell-csi-helm-installer/csi-offline-bundle.md b/dell-csi-helm-installer/csi-offline-bundle.md index bce4692c..5db223da 100644 --- a/dell-csi-helm-installer/csi-offline-bundle.md +++ b/dell-csi-helm-installer/csi-offline-bundle.md @@ -2,8 +2,7 @@ ## Description -The `csi-offline-bundle.sh` script can be used to create a package usable for offline installation of the Dell EMC CSI Storage Providers, via either Helm -or the Dell CSI Operator. +The `csi-offline-bundle.sh` script can be used to create a package for the offline installation of Dell CSI storage providers for deployment via Helm. This includes the following drivers: * [PowerFlex](https://github.com/dell/csi-vxflexos) @@ -12,8 +11,8 @@ This includes the following drivers: * [PowerStore](https://github.com/dell/csi-powerstore) * [Unity](https://github.com/dell/csi-unity) -As well as the Dell CSI Operator -* [Dell CSI Operator](https://github.com/dell/dell-csi-operator) +The `csm-offline-bundle.sh` script can be used to create a package for the offline installation of Dell CSI storage providers for deployment via the CSM Operator. +* [Dell CSM Operator](https://github.com/dell/csm-operator) ## Dependencies @@ -46,84 +45,93 @@ To perform an offline installation of a driver or the Operator, the following st This needs to be performed on a linux system with access to the internet as a git repo will need to be cloned, and container images pulled from public registries. The build an offline bundle, the following steps are needed: -1. Perform a `git clone` of the desired repository. For a helm based install, the specific driver repo should be cloned. For an Operator based deployment, the Dell CSI Operator repo should be cloned -2. Run the `csi-offline-bundle.sh` script with an argument of `-c` in order to create an offline bundle +1. Perform a `git clone` of the desired repository. For a Helm based install, the specific driver repo should be cloned. For an Operator based deployment, the Dell CSM Operator repo should be cloned +2. Run the offline bundle script with an argument of `-c` in order to create an offline bundle - For Helm installs, the `csi-offline-bundle.sh` script will be found in the `dell-csi-helm-installer` directory - - For Operator installs, the `csi-offline-bundle.sh` script will be found in the `scripts` directory + - For Operator installs, the `csm-offline-bundle.sh` script will be found in the `scripts` directory The script will perform the following steps: - - Determine required images by parsing either the driver Helm charts (if run from a cloned CSI Driver git repository) or the Dell CSI Operator configuration files (if run from a clone of the Dell CSI Operator repository) + - Determine required images by parsing either the driver Helm charts (if run from a cloned CSI Driver git repository) or the Dell CSM Operator configuration files (if run from a clone of the Dell CSM Operator repository) - Perform an image `pull` of each image required - Save all required images to a file by running `docker save` or `podman save` - Build a `tar.gz` file containing the images as well as files required to installer the driver and/or Operator The resulting offline bundle file can be copied to another machine, if necessary, to gain access to the desired image registry. -For example, here is the output of a request to build an offline bundle for the Dell CSI Operator: +For example, here is the output of a request to build an offline bundle for the Dell CSM Operator: ``` -[user@anothersystem /home/user]# git clone https://github.com/dell/dell-csi-operator.git +[user@anothersystem /home/user]# git clone https://github.com/dell/csm-operator.git ``` ``` -[user@anothersystem /home/user]# cd dell-csi-operator +[user@anothersystem /home/user]# cd csm-operator ``` ``` -[user@system /home/user/dell-csi-operator]# scripts/csi-offline-bundle.sh -c +[user@system /home/user/csm-operator]# bash scripts/csm-offline-bundle.sh -c * * Building image manifest file + Processing file /root/csm-operator/operatorconfig/driverconfig/common/default.yaml + Processing file /root/csm-operator/bundle/manifests/dell-csm-operator.clusterserviceversion.yaml * -* Pulling container images - - dellemc/csi-isilon:v1.2.0 - dellemc/csi-isilon:v1.3.0.000R - dellemc/csipowermax-reverseproxy:v1.0.0.000R - dellemc/csi-powermax:v1.2.0.000R - dellemc/csi-powermax:v1.4.0.000R - dellemc/csi-powerstore:v1.1.0.000R - dellemc/csi-unity:v1.3.0.000R - dellemc/csi-vxflexos:v1.1.5.000R - dellemc/csi-vxflexos:v1.2.0.000R - dellemc/dell-csi-operator:v1.1.0.000R - quay.io/k8scsi/csi-attacher:v2.0.0 - quay.io/k8scsi/csi-attacher:v2.2.0 - quay.io/k8scsi/csi-node-driver-registrar:v1.2.0 - quay.io/k8scsi/csi-provisioner:v1.4.0 - quay.io/k8scsi/csi-provisioner:v1.6.0 - quay.io/k8scsi/csi-resizer:v0.5.0 - quay.io/k8scsi/csi-snapshotter:v2.1.1 - -* -* Saving images - +* Pulling and saving container images + + quay.io/dell/container-storage-module/csi-isilon:v2.15.0 + quay.io/dell/container-storage-module/csi-metadata-retriever:v1.10.0 + quay.io/dell/container-storage-module/csipowermax-reverseproxy:v2.13.0 + quay.io/dell/container-storage-module/csi-powermax:v2.15.0 + quay.io/dell/container-storage-module/csi-powerstore:v2.15.0 + quay.io/dell/container-storage-module/csi-unity:v2.15.0 + quay.io/dell/container-storage-module/csi-vxflexos:v2.15.0 + quay.io/dell/container-storage-module/csm-authorization-sidecar:v1.14.0 + quay.io/dell/container-storage-module/csm-metrics-powerflex:v1.12.0 + quay.io/dell/container-storage-module/csm-metrics-powerscale:v1.9.0 + quay.io/dell/container-storage-module/csm-topology:v1.12.0 + quay.io/dell/container-storage-module/dell-csi-replicator:v1.12.0 + quay.io/dell/container-storage-module/dell-replication-controller:v1.12.0 + quay.io/dell/container-storage-modules/sdc:4.5.2.1 + quay.io/dell/container-storage-modules/dell-csm-operator:v1.9.0 + registry.redhat.io/openshift4/ose-kube-rbac-proxy-rhel9:v4.16.0-202409051837.p0.g8ea2c99.assembly.stream.el9 + nginxinc/nginx-unprivileged:1.27 + otel/opentelemetry-collector:0.42.0 + registry.k8s.io/sig-storage/csi-attacher:v4.8.0 + registry.k8s.io/sig-storage/csi-external-health-monitor-controller:v0.14.0 + registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.13.0 + registry.k8s.io/sig-storage/csi-provisioner:v5.1.0 + registry.k8s.io/sig-storage/csi-resizer:v1.13.1 + registry.k8s.io/sig-storage/csi-snapshotter:v8.2.0 * * Copying necessary files - /dell/git/dell-csi-operator/config - /dell/git/dell-csi-operator/deploy - /dell/git/dell-csi-operator/samples - /dell/git/dell-csi-operator/scripts - /dell/git/dell-csi-operator/README.md - /dell/git/dell-csi-operator/LICENSE + /root/csm-operator/deploy + /root/csm-operator/operatorconfig + /root/csm-operator/samples + /root/csm-operator/scripts + /root/csm-operator/README.md + /root/csm-operator/LICENSE * * Compressing release -dell-csi-operator-bundle/ -dell-csi-operator-bundle/samples/ +dell-csm-operator-bundle/ +dell-csm-operator-bundle/deploy/ +dell-csm-operator-bundle/deploy/operator.yaml +dell-csm-operator-bundle/deploy/crds/ +dell-csm-operator-bundle/deploy/crds/storage.dell.com_containerstoragemodules.yaml +dell-csm-operator-bundle/deploy/olm/ +dell-csm-operator-bundle/deploy/olm/operator_community.yaml ... - ... -dell-csi-operator-bundle/LICENSE -dell-csi-operator-bundle/README.md +dell-csm-operator-bundle/README.md +dell-csm-operator-bundle/LICENSE * * Complete -Offline bundle file is: /dell/git/dell-csi-operator/dell-csi-operator-bundle.tar.gz +Offline bundle file is: /root/csm-operator/dell-csm-operator-bundle.tar.gz ``` @@ -134,7 +142,7 @@ This needs to be performed on a linux system with access to an image registry th To prepare for driver or Operator installation, the following steps need to be performed: 1. Copy the offline bundle file to a system with access to an image registry available to your Kubernetes/OpenShift cluster 2. Expand the bundle file by running `tar xvfz ` -3. Run the `csi-offline-bundle.sh` script and supply the `-p` option as well as the path to the internal registry with the `-r` option +3. Run the `csm-offline-bundle.sh` script and supply the `-p` option as well as the path to the internal registry with the `-r` option The script will then perform the following steps: - Load the required container images into the local system @@ -143,69 +151,57 @@ The script will then perform the following steps: - Modify the Helm charts or Operator configuration to refer to the newly tagged/pushed images -An example of preparing the bundle for installation (192.168.75.40:5000 refers to a image registry accessible to Kubernetes/OpenShift): +An example of preparing the bundle for installation: ``` -[user@anothersystem /tmp]# tar xvfz dell-csi-operator-bundle.tar.gz -dell-csi-operator-bundle/ -dell-csi-operator-bundle/samples/ +[user@anothersystem /tmp]# tar xvfz dell-csm-operator-bundle.tar.gz +dell-csm-operator-bundle/ +dell-csm-operator-bundle/deploy/ +dell-csm-operator-bundle/deploy/operator.yaml +dell-csm-operator-bundle/deploy/crds/ +dell-csm-operator-bundle/deploy/crds/storage.dell.com_containerstoragemodules.yaml +dell-csm-operator-bundle/deploy/olm/ +dell-csm-operator-bundle/deploy/olm/operator_community.yaml ... - ... -dell-csi-operator-bundle/LICENSE -dell-csi-operator-bundle/README.md +dell-csm-operator-bundle/README.md +dell-csm-operator-bundle/LICENSE ``` ``` -[user@anothersystem /tmp]# cd dell-csi-operator-bundle +[user@anothersystem /tmp]# cd dell-csm-operator-bundle ``` ``` -[user@anothersystem /tmp/dell-csi-operator-bundle]# scripts/csi-offline-bundle.sh -p -r 192.168.75.40:5000/operator +[user@anothersystem /tmp/dell-csm-operator-bundle]# bash scripts/csm-offline-bundle.sh -p -r localregistry:5000/dell-csm-operator/ Preparing a offline bundle for installation * * Loading docker images +Loaded image: quay.io/dell/container-storage-modules/csi-powerstore:v2.12.0 +Loaded image: quay.io/dell/container-storage-modules/csi-isilon:v2.12.0 +... +... +Loaded image: registry.k8s.io/sig-storage/csi-resizer:v1.12.0 +Loaded image: registry.k8s.io/sig-storage/csi-snapshotter:v8.1.0 * * Tagging and pushing images - dellemc/csi-isilon:v1.2.0 -> 192.168.75.40:5000/operator/csi-isilon:v1.2.0 - dellemc/csi-isilon:v1.3.0.000R -> 192.168.75.40:5000/operator/csi-isilon:v1.3.0.000R - dellemc/csipowermax-reverseproxy:v1.0.0.000R -> 192.168.75.40:5000/operator/csipowermax-reverseproxy:v1.0.0.000R - dellemc/csi-powermax:v1.2.0.000R -> 192.168.75.40:5000/operator/csi-powermax:v1.2.0.000R - dellemc/csi-powermax:v1.4.0.000R -> 192.168.75.40:5000/operator/csi-powermax:v1.4.0.000R - dellemc/csi-powerstore:v1.1.0.000R -> 192.168.75.40:5000/operator/csi-powerstore:v1.1.0.000R - dellemc/csi-unity:v1.3.0.000R -> 192.168.75.40:5000/operator/csi-unity:v1.3.0.000R - dellemc/csi-vxflexos:v1.1.5.000R -> 192.168.75.40:5000/operator/csi-vxflexos:v1.1.5.000R - dellemc/csi-vxflexos:v1.2.0.000R -> 192.168.75.40:5000/operator/csi-vxflexos:v1.2.0.000R - dellemc/dell-csi-operator:v1.1.0.000R -> 192.168.75.40:5000/operator/dell-csi-operator:v1.1.0.000R - quay.io/k8scsi/csi-attacher:v2.0.0 -> 192.168.75.40:5000/operator/csi-attacher:v2.0.0 - quay.io/k8scsi/csi-attacher:v2.2.0 -> 192.168.75.40:5000/operator/csi-attacher:v2.2.0 - quay.io/k8scsi/csi-node-driver-registrar:v1.2.0 -> 192.168.75.40:5000/operator/csi-node-driver-registrar:v1.2.0 - quay.io/k8scsi/csi-provisioner:v1.4.0 -> 192.168.75.40:5000/operator/csi-provisioner:v1.4.0 - quay.io/k8scsi/csi-provisioner:v1.6.0 -> 192.168.75.40:5000/operator/csi-provisioner:v1.6.0 - quay.io/k8scsi/csi-resizer:v0.5.0 -> 192.168.75.40:5000/operator/csi-resizer:v0.5.0 - quay.io/k8scsi/csi-snapshotter:v2.1.1 -> 192.168.75.40:5000/operator/csi-snapshotter:v2.1.1 + quay.io/dell/container-storage-modules/csi-isilon:v2.12.0 -> localregistry:5000/dell-csm-operator/csi-isilon:v2.12.0 + quay.io/dell/container-storage-modules/csi-metadata-retriever:v1.9.0 -> localregistry:5000/dell-csm-operator/csi-metadata-retriever:v1.9.0 + ... + ... + registry.k8s.io/sig-storage/csi-resizer:v1.12.0 -> localregistry:5000/dell-csm-operator/csi-resizer:v1.12.0 + registry.k8s.io/sig-storage/csi-snapshotter:v8.1.0 -> localregistry:5000/dell-csm-operator/csi-snapshotter:v8.1.0 * -* Preparing operator files within /tmp/dell-csi-operator-bundle - - changing: dellemc/csi-isilon:v1.2.0 -> 192.168.75.40:5000/operator/csi-isilon:v1.2.0 - changing: dellemc/csi-isilon:v1.3.0.000R -> 192.168.75.40:5000/operator/csi-isilon:v1.3.0.000R - changing: dellemc/csipowermax-reverseproxy:v1.0.0.000R -> 192.168.75.40:5000/operator/csipowermax-reverseproxy:v1.0.0.000R - changing: dellemc/csi-powermax:v1.2.0.000R -> 192.168.75.40:5000/operator/csi-powermax:v1.2.0.000R - changing: dellemc/csi-powermax:v1.4.0.000R -> 192.168.75.40:5000/operator/csi-powermax:v1.4.0.000R - changing: dellemc/csi-powerstore:v1.1.0.000R -> 192.168.75.40:5000/operator/csi-powerstore:v1.1.0.000R - changing: dellemc/csi-unity:v1.3.0.000R -> 192.168.75.40:5000/operator/csi-unity:v1.3.0.000R - changing: dellemc/csi-vxflexos:v1.1.5.000R -> 192.168.75.40:5000/operator/csi-vxflexos:v1.1.5.000R - changing: dellemc/csi-vxflexos:v1.2.0.000R -> 192.168.75.40:5000/operator/csi-vxflexos:v1.2.0.000R - changing: dellemc/dell-csi-operator:v1.1.0.000R -> 192.168.75.40:5000/operator/dell-csi-operator:v1.1.0.000R - changing: quay.io/k8scsi/csi-attacher:v2.0.0 -> 192.168.75.40:5000/operator/csi-attacher:v2.0.0 - changing: quay.io/k8scsi/csi-attacher:v2.2.0 -> 192.168.75.40:5000/operator/csi-attacher:v2.2.0 - changing: quay.io/k8scsi/csi-node-driver-registrar:v1.2.0 -> 192.168.75.40:5000/operator/csi-node-driver-registrar:v1.2.0 - changing: quay.io/k8scsi/csi-provisioner:v1.4.0 -> 192.168.75.40:5000/operator/csi-provisioner:v1.4.0 - changing: quay.io/k8scsi/csi-provisioner:v1.6.0 -> 192.168.75.40:5000/operator/csi-provisioner:v1.6.0 - changing: quay.io/k8scsi/csi-resizer:v0.5.0 -> 192.168.75.40:5000/operator/csi-resizer:v0.5.0 - changing: quay.io/k8scsi/csi-snapshotter:v2.1.1 -> 192.168.75.40:5000/operator/csi-snapshotter:v2.1.1 +* Preparing files within /root/dell-csm-operator-bundle + + changing: quay.io/dell/container-storage-modules/csi-isilon:v2.12.0 -> localregistry:5000/dell-csm-operator/csi-isilon:v2.12.0 + changing: quay.io/dell/container-storage-modules/csi-metadata-retriever:v1.9.0 -> localregistry:5000/dell-csm-operator/csi-metadata-retriever:v1.9.0 + ... + ... + changing: registry.k8s.io/sig-storage/csi-resizer:v1.12.0 -> localregistry:5000/dell-csm-operator/csi-resizer:v1.12.0 + changing: registry.k8s.io/sig-storage/csi-snapshotter:v8.1.0 -> localregistry:5000/dell-csm-operator/csi-snapshotter:v8.1.0 * * Complete @@ -215,5 +211,3 @@ Preparing a offline bundle for installation ### Perform either a Helm installation or Operator installation Now that the required images have been made available and the Helm Charts/Operator configuration updated, installation can proceed by following the instructions that are documented within the driver or Operator repo. - - diff --git a/dell-csi-helm-installer/csi-offline-bundle.sh b/dell-csi-helm-installer/csi-offline-bundle.sh index 22316d23..2fd49a58 100755 --- a/dell-csi-helm-installer/csi-offline-bundle.sh +++ b/dell-csi-helm-installer/csi-offline-bundle.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright © 2020-2022 Dell Inc., or its subsidiaries. All Rights Reserved. +# Copyright © 2020-2025 Dell Inc., or its subsidiaries. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -24,7 +24,8 @@ usage() { echo " Supply the registry name/path which will hold the images" echo " For example: my.registry.com:5000/dell/csi" echo "-h Displays this information" - echo "-v Pass the helm chart version " + echo "-v Pass the helm chart version" + echo "-n Use the nightly tag for all CSI images on quay.io/dell" echo echo "Exactly one of '-c' or '-p' needs to be specified" echo @@ -36,7 +37,7 @@ status() { echo echo "*" echo "* $@" - echo + echo } # run_command @@ -63,6 +64,7 @@ run_command() { # build_image_manifest # builds a manifest of all the images referred to by the helm chart build_image_manifest() { + local REGEX_COMMENTS="(#.*)" local REGEX="([-_./:A-Za-z0-9]{3,}):([-_.A-Za-z0-9]{1,})" status "Building image manifest file" @@ -82,12 +84,12 @@ build_image_manifest() { # - search all files in a diectory looking for strings that make $REGEX # - exclude anything with double '//'' as that is a URL and not an image name # - make sure at least one '/' is found - find "${D}" -type f -exec egrep -oh "${REGEX}" {} \; | egrep -v '//' | egrep '/' >> "${IMAGEMANIFEST}.tmp" + find "${D}" -type f -exec egrep -v "${REGEX_COMMENTS}" {} \; | egrep -oh "${REGEX}"| egrep -v '//' | egrep '/' >> "${IMAGEMANIFEST}.tmp" fi done # Forming this only for drivers supporting standalone helm charts - if [ ! -z ${DRIVERREPO} ]; then + if [ ! -z ${DRIVERREPO} ]; then echo "${DRIVERREPO}/${DRIVERNAME}\:${DRIVERVERSIONVALUESYAML}" echo "${DRIVERREPO}/${DRIVERNAME}:${DRIVERVERSIONVALUESYAML}" >> "${IMAGEMANIFEST}.tmp" fi @@ -108,15 +110,20 @@ archive_images() { # the images, pull first in case some are not local while read line; do echo " $line" - run_command "${DOCKER}" pull "${line}" + if [[ "$NIGHTLY" = "true" ]] && [[ "$line" =~ quay.io/dell/container-storage-modules ]]; then + dockerImage=$(echo $line | sed 's/:[^:]*$/:nightly/') + run_command "${DOCKER}" pull "${dockerImage}" && run_command "${DOCKER}" tag "${dockerImage}" "${line}" + else + run_command "${DOCKER}" pull "${line}" + fi + IMAGEFILE=$(echo "${line}" | sed 's|[/:]|-|g') # if we already have the image exported, skip it if [ ! -f "${IMAGEFILEDIR}/${IMAGEFILE}.tar" ]; then run_command "${DOCKER}" save -o "${IMAGEFILEDIR}/${IMAGEFILE}.tar" "${line}" fi done < "${IMAGEMANIFEST}" - -} +} # restore_images # load the images from an archive into the local registry @@ -145,7 +152,7 @@ copy_files() { else cp -R "${f}" "${DISTDIR}" fi - + if [ $? -ne 0 ]; then echo "Unable to copy ${f} to the distribution directory" exit 1 @@ -225,12 +232,13 @@ set_mode() { CREATE="false" PREPARE="false" REGISTRY="" +NIGHTLY="false" DRIVER="csi-powerstore" -HELMCHARTVERSION="csi-powerstore-2.8.0" +HELMCHARTVERSION="csi-powerstore-2.15.0" -while getopts "cprv:h" opt; do +while getopts "cprnv:h" opt; do case $opt in - + c) CREATE="true" ;; @@ -248,6 +256,9 @@ while getopts "cprv:h" opt; do usage exit 0 ;; + n) + NIGHTLY="true" + ;; \?) echo "Invalid option: -$OPTARG" >&2 exit 1 @@ -264,12 +275,12 @@ done SCRIPTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" REPODIR="$( dirname "${SCRIPTDIR}" )" if [ ! -d "$REPODIR/helm-charts" ]; then - + if [ ! -d "$SCRIPTDIR/helm-charts" ]; then git clone --quiet -c advice.detachedHead=false -b $HELMCHARTVERSION https://github.com/dell/helm-charts fi mv helm-charts $REPODIR -else +else if [ -d "$SCRIPTDIR/helm-charts" ]; then rm -rf $SCRIPTDIR/helm-charts fi @@ -312,7 +323,7 @@ if [ "${MODE}" == "helm" ]; then "${REPODIR}/LICENSE" ) else - DRIVERNAME="dell-csi-operator" + DRIVERNAME="dell-csm-operator" DISTBASE="${REPODIR}" DRIVERDIR="${DRIVERNAME}-bundle" DISTDIR="${DISTBASE}/${DRIVERDIR}" @@ -358,7 +369,7 @@ if [ "${REGISTRY: -1}" != "/" ]; then fi # figure out if we should use docker or podman, preferring docker -DOCKER=$(which docker 2>/dev/null || which podman 2>/dev/null) +DOCKER=$(which docker 2>/dev/null || which podman 2>/dev/null) if [ "${DOCKER}" == "" ]; then echo "Unable to find either docker or podman in $PATH" exit 1 diff --git a/dell-csi-helm-installer/csi-uninstall.sh b/dell-csi-helm-installer/csi-uninstall.sh index 08dd5857..b696c694 100755 --- a/dell-csi-helm-installer/csi-uninstall.sh +++ b/dell-csi-helm-installer/csi-uninstall.sh @@ -70,6 +70,18 @@ function check_for_driver() { fi } +# Since helm does not automatically clean up crds on an uninstall. +# This step must be done manually. +# check here for more information https://helm.sh/docs/chart_best_practices/custom_resource_definitions +function remove_crd_after_uninstall() { + # Remove the volume journals crd + decho "Removal of the CSI PowerStore Custom Resource Definitions..." + run_command kubectl delete crd volumejournals.dr.storage.dell.com + if [ $? -ne 0 ]; then + decho "Removal of the CSI PowerStore Custom Resource Definition (volumejournals.dr.storage.dell.com) failed" + fi +} + # get the list of valid CSI Drivers, this will be the list of directories in drivers/ that contain helm charts DRIVERDIR="${SCRIPTDIR}/../helm-charts/charts" @@ -125,6 +137,8 @@ if [ $? -ne 0 ]; then exit 1 fi +remove_crd_after_uninstall + decho "Removal of the CSI Driver is in progress." decho "It may take a few minutes for all pods to terminate." diff --git a/dell-csi-helm-installer/verify-csi-powerstore.sh b/dell-csi-helm-installer/verify-csi-powerstore.sh index 73503111..65959729 100755 --- a/dell-csi-helm-installer/verify-csi-powerstore.sh +++ b/dell-csi-helm-installer/verify-csi-powerstore.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright © 2021-2023 Dell Inc., or its subsidiaries. All Rights Reserved. +# Copyright © 2021-2025 Dell Inc., or its subsidiaries. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -10,8 +10,8 @@ # verify-csi-powerstore method function verify-csi-powerstore() { - verify_k8s_versions "1.24" "1.28" - verify_openshift_versions "4.12" "4.13" + verify_k8s_versions "1.31" "1.33" + verify_openshift_versions "4.18" "4.19" verify_namespace "${NS}" verify_required_secrets "${RELEASE}-config" verify_alpha_snap_resources @@ -20,6 +20,7 @@ function verify-csi-powerstore() { verify_nvmetcp_installation verify_nvmefc_installation verify_helm_3 + verify_authorization_proxy_server } function verify_optional_replication_requirements() { diff --git a/dell-csi-helm-installer/verify.sh b/dell-csi-helm-installer/verify.sh index 2d7f1f8b..b18a8824 100755 --- a/dell-csi-helm-installer/verify.sh +++ b/dell-csi-helm-installer/verify.sh @@ -244,13 +244,26 @@ function verify_k8s_versions() { local MIN=${1} local MAX=${2} local V="${kMajorVersion}.${kMinorVersion}" + + # check non supported version (k8s alpha/beta) + if [ -n "${kNonGAVersion}" ]; then + echo "Installing on an unreleased version of Kubernetes : "${kNonGAVersion}". Acknowledge and proceed with installation? (y/n)" + read -n 1 -p "Press 'y' to continue or any other key to exit: " CONT + decho + if [ "${CONT}" != "Y" -a "${CONT}" != "y" ]; then + decho "quitting at user request" + exit 2 + fi + fi + # check minimum log arrow log smart_step "Verifying minimum Kubernetes version" "small" error=0 if [[ ${V} < ${MIN} ]]; then error=1 - found_error "Kubernetes version ${V} is too old. Minimum required version is: ${MIN}" + found_warning "Kubernetes version ${V} is too old. Minimum required version is: ${MIN}" + found_warning "To ensure the driver is fully supported run cert-csi and make sure all tests pass. More details: https://dell.github.io/csm-docs/docs/support/cert-csi/" fi check_error error @@ -261,6 +274,7 @@ function verify_k8s_versions() { if [[ ${V} > ${MAX} ]]; then error=1 found_warning "Kubernetes version ${V} is newer than the version that has been tested. Latest tested version is: ${MAX}" + found_warning "To ensure the driver is fully supported run cert-csi and make sure all tests pass. More details: https://dell.github.io/csm-docs/docs/support/cert-csi/" fi check_error error @@ -283,7 +297,8 @@ function verify_openshift_versions() { error=0 if (( ${V%%.*} < ${MIN%%.*} || ( ${V%%.*} == ${MIN%%.*} && ${V##*.} < ${MIN##*.} ) )) ; then error=1 - found_error "OpenShift version ${V} is too old. Minimum required version is: ${MIN}" + found_warning "OpenShift version ${V} is too old. Minimum required version is: ${MIN}" + found_warning "To ensure the driver is fully supported run cert-csi and make sure all tests pass. More details: https://dell.github.io/csm-docs/docs/support/cert-csi/" fi check_error error @@ -294,6 +309,7 @@ function verify_openshift_versions() { if (( ${V%%.*} > ${MAX%%.*} || ( ${V%%.*} == ${MAX%%.*} && ${V##*.} > ${MAX##*.} ) )) ; then error=1 found_warning "OpenShift version ${V} is newer than the version that has been tested. Latest tested version is: ${MAX}" + found_warning "To ensure the driver is fully supported run cert-csi and make sure all tests pass. More details: https://dell.github.io/csm-docs/docs/support/cert-csi/" fi check_error error } @@ -339,6 +355,27 @@ function verify_alpha_snap_resources() { check_error error } +# verify that the requirements for disaster recover support exist +function verify_dr_requirements() { + log step "Verifying disaster recover support" + decho + log arrow + log smart_step "Verifying that disaster recover CRDs are available" "small" + + error=0 + # check for the CRDs. These are required for installation + CRDS=("VolumeJournal") + for C in "${CRDS[@]}"; do + # Verify if DR related CRDs are there on the system. + run_command kubectl explain ${C} 2>&1 >/dev/null + if [ $? -ne 0 ]; then + error=1 + found_error "The CRD for ${C} is not installed. These need to be installed by the Kubernetes administrator" + fi + done + check_error error +} + # verify that the requirements for snapshot support exist function verify_snap_requirements() { log step "Verifying snapshot support" @@ -619,6 +656,7 @@ MASTER_NODES=$(run_command kubectl get nodes -o wide | awk ' /master/{ print $6; # Get the kubernetes major and minor version numbers. kMajorVersion=$(run_command kubectl version | grep 'Server Version' | sed -E 's/.*v([0-9]+)\.[0-9]+\.[0-9]+.*/\1/') kMinorVersion=$(run_command kubectl version | grep 'Server Version' | sed -E 's/.*v[0-9]+\.([0-9]+)\.[0-9]+.*/\1/') +kNonGAVersion=$(run_command kubectl version | grep 'Server Version' | sed -n 's/.*\(-[alpha|beta][^ ]*\).*/\1/p') while getopts ":h-:" optchar; do case "${optchar}" in diff --git a/docker-files/Dockerfile.centos b/docker-files/Dockerfile.centos deleted file mode 100644 index b1fbb241..00000000 --- a/docker-files/Dockerfile.centos +++ /dev/null @@ -1,45 +0,0 @@ -# -# -# Copyright © 2020-2023 Dell Inc. or its subsidiaries. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# - -# Dockerfile to build PowerStore CSI Driver -# based on CentOS -ARG BASEIMAGE - -FROM $BASEIMAGE AS driver - -LABEL vendor="Dell Inc." \ - name="csi-powerstore" \ - summary="CSI Driver for Dell EMC PowerStore" \ - description="CSI Driver for provisioning persistent storage from Dell EMC PowerStore" \ - version="2.8.0" \ - license="Apache-2.0" - -COPY licenses /licenses - -# dependencies, following by cleaning the cache -RUN echo "%_netsharedpath /sys:/proc" >> /etc/rpm/macros.dist && yum update -y && yum install -y e2fsprogs xfsprogs nfs-utils nfs4-acl-tools acl which device-mapper-multipath \ - && \ - yum clean all \ - && \ - rm -rf /var/cache/run - -# validate some cli utilities are found -RUN which mkfs.ext4 -RUN which mkfs.xfs -RUN echo "export PATH=$PATH:/sbin:/bin" > /etc/profile.d/ubuntu_path.sh - -COPY "csi-powerstore" . -ENTRYPOINT ["/csi-powerstore"] diff --git a/docker-files/Dockerfile.ubi b/docker-files/Dockerfile.ubi deleted file mode 100644 index 5697340e..00000000 --- a/docker-files/Dockerfile.ubi +++ /dev/null @@ -1,46 +0,0 @@ -# -# -# Copyright © 2020-2023 Dell Inc. or its subsidiaries. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# - -# Dockerfile to build PowerStore CSI Driver -# based on standard UBI image -# Requires: RHEL host with subscription -FROM registry.access.redhat.com/ubi8/ubi:latest - -LABEL vendor="Dell Inc." \ - name="csi-powerstore" \ - summary="CSI Driver for Dell EMC PowerStore" \ - description="CSI Driver for provisioning persistent storage from Dell EMC PowerStore" \ - version="2.8.0" \ - license="Apache-2.0" - -COPY licenses /licenses - -# dependencies, following by cleaning the cache -RUN yum update -y \ - && \ - yum install -y e2fsprogs xfsprogs nfs-utils nfs4-acl-tools acl which device-mapper-multipath \ - && \ - yum clean all \ - && \ - rm -rf /var/cache/run - -# validate some cli utilities are found -RUN which mkfs.ext4 -RUN which mkfs.xfs -RUN echo "export PATH=$PATH:/sbin:/bin" > /etc/profile.d/ubuntu_path.sh - -COPY "csi-powerstore" . -ENTRYPOINT ["/csi-powerstore"] diff --git a/docker-files/Dockerfile.ubi.alt b/docker-files/Dockerfile.ubi.alt deleted file mode 100644 index 09bb5c72..00000000 --- a/docker-files/Dockerfile.ubi.alt +++ /dev/null @@ -1,59 +0,0 @@ -# -# -# Copyright © 2020-2023 Dell Inc. or its subsidiaries. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# - -# Dockerfile to build PowerStore CSI Driver -# based on UBI standard image. -# Alternative way without RHEL host/subscription, using centos.repo -FROM registry.access.redhat.com/ubi8/ubi-minimal:8.7-1085 - -LABEL vendor="Dell Inc." \ - name="csi-powerstore" \ - summary="CSI Driver for Dell EMC PowerStore" \ - description="CSI Driver for provisioning persistent storage from Dell EMC PowerStore" \ - version="2.8.0" \ - license="Apache-2.0" - -COPY licenses /licenses - -# Adding Centos.repo file -RUN echo $'[centos] \n\ -name=centos \n\ -baseurl=http://mirror.centos.org/centos-8/8/BaseOS/x86_64/os \n\ -enabled=1 \n\ -gpgcheck=0' > /etc/yum.repos.d/centos.repo - -# dependencies, following by cleaning the cache -RUN yum -y update && \ - yum -y install \ - e2fsprogs.x86_64 \ - xfsprogs.x86_64 \ - nfs-utils.x86_64 \ - nfs4-acl-tools \ - acl \ - which \ - device-mapper-multipath \ - && \ - yum clean all \ - && \ - rm -rf /var/cache/run - -# validate some cli utilities are found -RUN which mke2fs -RUN which mkfs.xfs -RUN echo "export PATH=$PATH:/sbin:/bin" > /etc/profile.d/ubuntu_path.sh - -COPY "csi-powerstore" . -ENTRYPOINT ["/csi-powerstore"] diff --git a/docker-files/Dockerfile.ubi.min b/docker-files/Dockerfile.ubi.min deleted file mode 100644 index 23609df6..00000000 --- a/docker-files/Dockerfile.ubi.min +++ /dev/null @@ -1,52 +0,0 @@ -# -# -# Copyright © 2020-2023 Dell Inc. or its subsidiaries. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# - -# Dockerfile to build PowerStore CSI Driver -# based on UBI-minimal image -# Requires: RHEL host with subscription -# UBI Image: ubi8/ubi-minimal:8.7-1085 -FROM registry.access.redhat.com/ubi8/ubi-minimal@sha256:ab03679e683010d485ef0399e056b09a38d7843ba4a36ee7dec337dd0037f7a7 - -LABEL vendor="Dell Inc." \ - name="csi-powerstore" \ - summary="CSI Driver for Dell EMC PowerStore" \ - description="CSI Driver for provisioning persistent storage from Dell EMC PowerStore" \ - version="2.8.0" \ - license="Apache-2.0" - -COPY licenses /licenses - -# dependencies, following by cleaning the cache -RUN microdnf update -y \ - && \ - microdnf install -y \ - e2fsprogs \ - xfsprogs \ - nfs-utils \ - nfs4-acl-tools \ - acl \ - which \ - device-mapper-multipath \ - && \ - microdnf clean all - -# validate some cli utilities are found -RUN which mkfs.ext4 -RUN which mkfs.xfs -RUN echo "export PATH=$PATH:/sbin:/bin" > /etc/profile.d/ubuntu_path.sh - -COPY "csi-powerstore" . -ENTRYPOINT ["/csi-powerstore"] diff --git a/docker.mk b/docker.mk deleted file mode 100644 index af2557f1..00000000 --- a/docker.mk +++ /dev/null @@ -1,72 +0,0 @@ -# -# -# Copyright © 2020-2023 Dell Inc. or its subsidiaries. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# - -# for variables override --include vars.mk - -# Includes the following generated file to get semantic version information -include semver.mk -ifdef NOTES - RELNOTE="$(NOTES)" -else - RELNOTE= -endif - -ifndef DOCKER_REGISTRY - DOCKER_REGISTRY=dellemc -endif - -ifndef DOCKER_IMAGE_NAME - DOCKER_IMAGE_NAME=csi-powerstore -endif - -ifndef BASEIMAGE - BASEIMAGE=registry.access.redhat.com/ubi9/ubi-micro@sha256:630cf7bdef807f048cadfe7180d6c27eb3aaa99323ffc3628811da230ed3322a -endif - -# Add 'build-base-image' as a dependency if UBI Micro is used as the base image. -# This is required to load all the depedent packages into UBI Miro image. -ifeq ($(DOCKER_FILE), docker-files/Dockerfile.ubi.micro) - DEPENDENCIES=build-base-image -endif - -# figure out if podman or docker should be used (use podman if found) -ifneq (, $(shell which podman 2>/dev/null)) - BUILDER=podman -else - BUILDER=docker -endif - -docker: $(DEPENDENCIES) - @echo "MAJOR $(MAJOR) MINOR $(MINOR) PATCH $(PATCH) RELNOTE $(RELNOTE) SEMVER $(SEMVER)" - @echo "$(DOCKER_FILE)" - $(BUILDER) build -f $(DOCKER_FILE) -t "$(DOCKER_REGISTRY)/$(DOCKER_IMAGE_NAME):v$(MAJOR).$(MINOR).$(PATCH)$(RELNOTE)" --build-arg BASEIMAGE=$(BASEIMAGE) . - - -docker-no-cache: $(DEPENDENCIES) - @echo "MAJOR $(MAJOR) MINOR $(MINOR) PATCH $(PATCH) RELNOTE $(RELNOTE) SEMVER $(SEMVER)" - @echo "$(DOCKER_FILE) --no-cache" - $(BUILDER) build --no-cache -f $(DOCKER_FILE) -t "$(DOCKER_REGISTRY)/$(DOCKER_IMAGE_NAME):v$(MAJOR).$(MINOR).$(PATCH)$(RELNOTE)" --build-arg BASEIMAGE=$(BASEIMAGE) . - -push: - echo "MAJOR $(MAJOR) MINOR $(MINOR) PATCH $(PATCH) RELNOTE $(RELNOTE) SEMVER $(SEMVER)" - $(BUILDER) push "$(DOCKER_REGISTRY)/$(DOCKER_IMAGE_NAME):v$(MAJOR).$(MINOR).$(PATCH)$(RELNOTE)" - -build-base-image: - @echo "Building base image from $(BASEIMAGE) and loading dependencies..." - ./buildubimicro.sh $(BASEIMAGE) - @echo "Base image build: SUCCESS" - $(eval BASEIMAGE=localhost/csipowerstore-ubimicro:latest) diff --git a/go.mod b/go.mod index d510fb09..3945eaf3 100644 --- a/go.mod +++ b/go.mod @@ -1,118 +1,137 @@ module github.com/dell/csi-powerstore/v2 -go 1.21 +go 1.25.0 require ( + github.com/dell/csi-metadata-retriever v1.13.0 + github.com/dell/csm-dr v1.0.0 + github.com/dell/csmlog v1.0.0 + github.com/dell/dell-csi-extensions/common v1.10.0 + github.com/dell/dell-csi-extensions/podmon v1.10.0 + github.com/dell/dell-csi-extensions/replication v1.13.0 + github.com/dell/dell-csi-extensions/volumeGroupSnapshot v1.8.1 + github.com/dell/gobrick v1.16.0 + github.com/dell/gocsi v1.16.0 + github.com/dell/gofsutil v1.21.0 + github.com/dell/goiscsi v1.14.0 + github.com/dell/gonvme v1.13.0 + github.com/dell/gopowerstore v1.21.0 github.com/akutz/gosync v0.1.0 github.com/apparentlymart/go-cidr v1.1.0 - github.com/container-storage-interface/spec v1.6.0 - github.com/dell/csi-metadata-retriever v1.5.0 - github.com/dell/dell-csi-extensions/common v1.2.0 - github.com/dell/dell-csi-extensions/podmon v1.2.0 - github.com/dell/dell-csi-extensions/replication v1.5.0 - github.com/dell/dell-csi-extensions/volumeGroupSnapshot v1.3.0 - github.com/dell/gobrick v1.9.0 - github.com/dell/gocsi v1.8.1-0.20230915044639-4bab90258ed0 - github.com/dell/gofsutil v1.13.1 - github.com/dell/goiscsi v1.8.0 - github.com/dell/gonvme v1.5.0 - github.com/dell/gopowerstore v1.13.1-0.20231012074319-4009fe74aa49 - github.com/fsnotify/fsnotify v1.5.4 - github.com/go-openapi/strfmt v0.21.3 + github.com/container-storage-interface/spec v1.7.0 + github.com/fsnotify/fsnotify v1.9.0 + github.com/go-openapi/strfmt v0.25.0 + github.com/gogo/protobuf v1.3.2 github.com/golang/mock v1.6.0 - github.com/golang/protobuf v1.5.3 - github.com/gorilla/mux v1.8.0 - github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 + github.com/golang/protobuf v1.5.4 + github.com/google/uuid v1.6.0 + github.com/gorilla/mux v1.8.1 + github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 github.com/kubernetes-csi/csi-lib-utils v0.11.0 github.com/onsi/ginkgo v1.16.5 - github.com/onsi/gomega v1.23.0 + github.com/onsi/gomega v1.39.0 github.com/opentracing/opentracing-go v1.2.0 github.com/sirupsen/logrus v1.9.3 - github.com/spf13/viper v1.12.0 - github.com/stretchr/testify v1.8.0 + github.com/spf13/viper v1.21.0 + github.com/stretchr/testify v1.11.1 github.com/uber/jaeger-client-go v2.30.0+incompatible github.com/uber/jaeger-lib v2.4.1+incompatible - golang.org/x/net v0.14.0 - google.golang.org/grpc v1.57.0 - google.golang.org/protobuf v1.31.0 - gopkg.in/yaml.v3 v3.0.1 - k8s.io/apimachinery v0.26.1 - k8s.io/client-go v0.26.1 + go.uber.org/mock v0.6.0 + golang.org/x/net v0.49.0 + google.golang.org/grpc v1.78.0 + google.golang.org/protobuf v1.36.11 + k8s.io/api v0.35.0 + k8s.io/apimachinery v0.35.0 + k8s.io/client-go v0.35.0 + k8s.io/component-helpers v0.35.0 + sigs.k8s.io/controller-runtime v0.22.4 + sigs.k8s.io/yaml v1.6.0 ) require ( github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect - github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/coreos/go-semver v0.3.0 // indirect - github.com/coreos/go-systemd/v22 v22.3.2 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/emicklei/go-restful/v3 v3.10.0 // indirect - github.com/go-logr/logr v1.2.3 // indirect - github.com/go-openapi/errors v0.20.2 // indirect - github.com/go-openapi/jsonpointer v0.19.5 // indirect - github.com/go-openapi/jsonreference v0.20.0 // indirect - github.com/go-openapi/swag v0.19.14 // indirect - github.com/gogo/protobuf v1.3.2 // indirect - github.com/google/gnostic v0.5.7-v3refs // indirect - github.com/google/go-cmp v0.5.9 // indirect - github.com/google/gofuzz v1.2.0 // indirect - github.com/hashicorp/hcl v1.0.0 // indirect - github.com/imdario/mergo v0.3.6 // indirect - github.com/josharian/intern v1.0.0 // indirect + github.com/bombsimon/logrusr/v4 v4.1.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/coreos/go-semver v0.3.1 // indirect + github.com/coreos/go-systemd/v22 v22.6.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/emicklei/go-restful/v3 v3.13.0 // indirect + github.com/evanphx/json-patch/v5 v5.9.11 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-openapi/errors v0.22.4 // indirect + github.com/go-openapi/jsonpointer v0.22.3 // indirect + github.com/go-openapi/jsonreference v0.21.3 // indirect + github.com/go-openapi/swag v0.25.4 // indirect + github.com/go-openapi/swag/cmdutils v0.25.4 // indirect + github.com/go-openapi/swag/conv v0.25.4 // indirect + github.com/go-openapi/swag/fileutils v0.25.4 // indirect + github.com/go-openapi/swag/jsonname v0.25.4 // indirect + github.com/go-openapi/swag/jsonutils v0.25.4 // indirect + github.com/go-openapi/swag/loading v0.25.4 // indirect + github.com/go-openapi/swag/mangling v0.25.4 // indirect + github.com/go-openapi/swag/netutils v0.25.4 // indirect + github.com/go-openapi/swag/stringutils v0.25.4 // indirect + github.com/go-openapi/swag/typeutils v0.25.4 // indirect + github.com/go-openapi/swag/yamlutils v0.25.4 // indirect + github.com/go-viper/mapstructure/v2 v2.4.0 // indirect + github.com/google/btree v1.1.3 // indirect + github.com/google/gnostic-models v0.7.1 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/magiconair/properties v1.8.6 // indirect - github.com/mailru/easyjson v0.7.6 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/nxadm/tail v1.4.8 // indirect + github.com/nxadm/tail v1.4.11 // indirect github.com/oklog/ulid v1.3.1 // indirect - github.com/pelletier/go-toml v1.9.5 // indirect - github.com/pelletier/go-toml/v2 v2.0.1 // indirect + github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_golang v1.12.2 // indirect - github.com/prometheus/client_model v0.2.0 // indirect - github.com/prometheus/common v0.34.0 // indirect - github.com/prometheus/procfs v0.7.3 // indirect - github.com/spf13/afero v1.8.2 // indirect - github.com/spf13/cast v1.5.0 // indirect - github.com/spf13/jwalterweatherman v1.1.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect - github.com/stretchr/objx v0.4.0 // indirect - github.com/subosito/gotenv v1.3.0 // indirect - go.etcd.io/etcd/api/v3 v3.5.4 // indirect - go.etcd.io/etcd/client/pkg/v3 v3.5.4 // indirect - go.etcd.io/etcd/client/v3 v3.5.4 // indirect - go.mongodb.org/mongo-driver v1.11.2 // indirect - go.uber.org/atomic v1.9.0 // indirect - go.uber.org/multierr v1.8.0 // indirect - go.uber.org/zap v1.21.0 // indirect - golang.org/x/oauth2 v0.7.0 // indirect - golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f // indirect - golang.org/x/sys v0.11.0 // indirect - golang.org/x/term v0.11.0 // indirect - golang.org/x/text v0.12.0 // indirect - golang.org/x/time v0.0.0-20220411224347-583f2d630306 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230815205213-6bfd019c3878 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/prometheus/client_golang v1.23.2 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.67.4 // indirect + github.com/prometheus/procfs v0.19.2 // indirect + github.com/sagikazarmark/locafero v0.12.0 // indirect + github.com/spf13/afero v1.15.0 // indirect + github.com/spf13/cast v1.10.0 // indirect + github.com/spf13/pflag v1.0.10 // indirect + github.com/stretchr/objx v0.5.3 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + github.com/x448/float16 v0.8.4 // indirect + go.etcd.io/etcd/api/v3 v3.6.6 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.6.6 // indirect + go.etcd.io/etcd/client/v3 v3.6.6 // indirect + go.mongodb.org/mongo-driver v1.17.6 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect + go.opentelemetry.io/otel v1.38.0 // indirect + go.opentelemetry.io/otel/trace v1.38.0 // indirect + go.uber.org/atomic v1.11.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.1 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/oauth2 v0.33.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/sys v0.40.0 // indirect + golang.org/x/term v0.39.0 // indirect + golang.org/x/text v0.33.0 // indirect + golang.org/x/time v0.14.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b // indirect + gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/ini.v1 v1.66.6 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect - k8s.io/api v0.26.1 // indirect - k8s.io/component-base v0.24.1 // indirect - k8s.io/klog/v2 v2.80.1 // indirect - k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect - k8s.io/utils v0.0.0-20230202215443-34013725500c // indirect - sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect - sigs.k8s.io/yaml v1.3.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apiextensions-apiserver v0.34.2 // indirect + k8s.io/component-base v0.34.2 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20251125145642-4e65d59e963e // indirect + k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.1 // indirect ) diff --git a/go.sum b/go.sum index 6718d751..774e9111 100644 --- a/go.sum +++ b/go.sum @@ -3,43 +3,51 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= -cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= -cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= -cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/dell/csi-metadata-retriever v1.13.0 h1:6eb/VaVoY3NVGXUppMJEDZirsH1y8Ki/SHkJCkmI+L8= +github.com/dell/csi-metadata-retriever v1.13.0/go.mod h1:ZiOG9J1MjTUslncrbfgZ2v8sxdOB6ZdRtVjRCAQ0tmc= +github.com/dell/csm-dr v1.0.0 h1:LW6MHU/YYiaxI3yUV+riEIWGbPV7UW0OgD4AWCct/HM= +github.com/dell/csm-dr v1.0.0/go.mod h1:/yKWAo804JSVNzUYLPc43v7dE0E0eH53P02Kil7HMRw= +github.com/dell/csmlog v1.0.0 h1:EzW+nMJBD0QTNP88OoaAJUOMVilS9cWkO248BTcJt/4= +github.com/dell/csmlog v1.0.0/go.mod h1:7rBzSv9xF5t233+J+9vkStjFsmyYO3L/B9tDTy3+9ZU= +github.com/dell/dell-csi-extensions/common v1.10.0 h1:WIFPWVEBUyzOTCOPAlcgQsiRRGAufyKJYbIATbNZXIY= +github.com/dell/dell-csi-extensions/common v1.10.0/go.mod h1:zRHzmPX5SQQnqQ1LEIxG4hYqLBeQOSiD8TkEhU0eWTY= +github.com/dell/dell-csi-extensions/podmon v1.10.0 h1:YeM9OmgJHE+n6aNaeEC96EuVev5x3pddggcM7Ws7pkk= +github.com/dell/dell-csi-extensions/podmon v1.10.0/go.mod h1:+g7fdyw1Zx74NBJQgi1BCtsywqk37MJd9JN86IPJJu0= +github.com/dell/dell-csi-extensions/replication v1.13.0 h1:DSpoZ3vX65a3KDxUv0OinLkY2qUAQtRX3E1c1e3fnvA= +github.com/dell/dell-csi-extensions/replication v1.13.0/go.mod h1:aJBwd55amqbY3kk8SG7NjwH7nxBscceDwc1rKesUG1g= +github.com/dell/dell-csi-extensions/volumeGroupSnapshot v1.8.1 h1:W0UcLCZ8qyJ+NBRDfrZefN+fMs2i73ydkIsq6RjP7bM= +github.com/dell/dell-csi-extensions/volumeGroupSnapshot v1.8.1/go.mod h1:C4Ji1GCEayZ733hcHkvM6JDcboWJjk43H7xp30a/trc= +github.com/dell/gobrick v1.16.0 h1:z/a9qXnT3hx3D4I+SJUMnIgJtcCx0j3gzmPPDUWtoYs= +github.com/dell/gobrick v1.16.0/go.mod h1:9uoH8EsNi9yAsUZj2gZFgB5kqdlyvArqx0tYC7Qg9IM= +github.com/dell/gocsi v1.16.0 h1:avhQPD11rYzT6/dPxpZfFsJV+T/T0x1GJqqbco45W8c= +github.com/dell/gocsi v1.16.0/go.mod h1:Fz5dQv/kWf5Y1EXZEzxLBQSsnW2HE/WY95R0WCDQLO4= +github.com/dell/gofsutil v1.21.0 h1:SeusAYjiO/1ogvg/TapvCyHcrM9z+OvdaMU5i9Ijn3M= +github.com/dell/gofsutil v1.21.0/go.mod h1:qBGEz1wMOtqTODuJfiBZhUHT0JjexBblu2oa+sEclNs= +github.com/dell/goiscsi v1.14.0 h1:kNDqOlpJ3cLSJh7Hfyn/Kz/FMCKHzV0s/xx4EqnelFw= +github.com/dell/goiscsi v1.14.0/go.mod h1:SCSC8dJCqTosU7SspaoLv6ICTKNEz08rt/I8nZ3+ptc= +github.com/dell/gonvme v1.13.0 h1:j8A1BzYA48gelih3xWd/J6LQ71CbC8Lbdyv0jG8uUNU= +github.com/dell/gonvme v1.13.0/go.mod h1:L5K7V4JZTf12m3k2wdwKwP+/eA6pr8DvlCsJU1QTGOQ= +github.com/dell/gopowerstore v1.21.0 h1:CjuBg6OLHzNh0lsWIh7mVDKqFvZIf65c5AT+BYP9L3s= +github.com/dell/gopowerstore v1.21.0/go.mod h1:MxuIIOHhcaIgCjzXR4DT5NSRe+XIr/5xWM3dH47fmUk= github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= @@ -53,6 +61,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM= github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= @@ -71,13 +81,8 @@ github.com/apparentlymart/go-cidr v1.1.0/go.mod h1:EBcsNrHc3zQeuaeCeCtQruQm+n9/Y github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= -github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= -github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= -github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -88,91 +93,73 @@ github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJm github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/bombsimon/logrusr/v4 v4.1.0 h1:uZNPbwusB0eUXlO8hIUwStE6Lr5bLN6IgYgG+75kuh4= +github.com/bombsimon/logrusr/v4 v4.1.0/go.mod h1:pjfHC5e59CvjTBIU3V3sGhFWFAnsnhOR03TRc6im0l8= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/container-storage-interface/spec v1.5.0/go.mod h1:8K96oQNkJ7pFcC2R9Z1ynGGBB1I93kcS6PGg3SsOk8s= -github.com/container-storage-interface/spec v1.6.0 h1:vwN9uCciKygX/a0toYryoYD5+qI9ZFeAMuhEEKO+JBA= -github.com/container-storage-interface/spec v1.6.0/go.mod h1:8K96oQNkJ7pFcC2R9Z1ynGGBB1I93kcS6PGg3SsOk8s= +github.com/container-storage-interface/spec v1.7.0 h1:gW8eyFQUZWWrMWa8p1seJ28gwDoN5CVJ4uAbQ+Hdycw= +github.com/container-storage-interface/spec v1.7.0/go.mod h1:JYuzLqr9VVNoDJl44xp/8fmCOvWPDKzuGTwCoklhuqk= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= +github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/go-systemd/v22 v22.6.0 h1:aGVa/v8B7hpb0TKl0MWoAavPDmHvobFe5R5zn0bCJWo= +github.com/coreos/go-systemd/v22 v22.6.0/go.mod h1:iG+pp635Fo7ZmV/j14KUcmEyWF+0X7Lua8rrTWzYgWU= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dell/csi-metadata-retriever v1.5.0 h1:01VA0gopvSd2VtCfr3rR3qik3ArwxFz8tz8MbxloU1I= -github.com/dell/csi-metadata-retriever v1.5.0/go.mod h1:5V/D6/YfnmHQgAXvuuDCFe/EyhnG0KPgr0Sm+b5pN3g= -github.com/dell/dell-csi-extensions/common v1.2.0 h1:vh6f0qNfHSIGZnT5fZLFOt6ABoQAQDIk64z7f+B3xX0= -github.com/dell/dell-csi-extensions/common v1.2.0/go.mod h1:RLDVq6tz2yVzsX804Daopj/JBB147uqH2NKx6O3G0vA= -github.com/dell/dell-csi-extensions/podmon v1.2.0 h1:bYoTPMz/ILl8+cjK+zkwqX5FfiOB8H4Qa2C17sKzsV4= -github.com/dell/dell-csi-extensions/podmon v1.2.0/go.mod h1:MXkd5u3Vt876LEqDeqywlK0Re/IA0FZlHKFWvvdhyzI= -github.com/dell/dell-csi-extensions/replication v1.5.0 h1:xbnWCmNy/nhh6zdBHyM/UqCh/oBfLFwijdBC7UCNANg= -github.com/dell/dell-csi-extensions/replication v1.5.0/go.mod h1:puNHmHJWoWeMNj5NXER7oXZtxVnkWZjFNP8mwK0ev18= -github.com/dell/dell-csi-extensions/volumeGroupSnapshot v1.3.0 h1:yXocN5AwXzrJXGvx0a3rcD22mQJ7zFzWfpp+IfNs4nY= -github.com/dell/dell-csi-extensions/volumeGroupSnapshot v1.3.0/go.mod h1:EWT6KIoauXYlcGiss60KwlnTwxFI6KCt3hklW0HZIOc= -github.com/dell/gobrick v1.9.0 h1:kx69ygz1QV/uCAyIx9pX9gqiwDK7I4WOv5ZUs2zcfPg= -github.com/dell/gobrick v1.9.0/go.mod h1:NK9V+t6LYMWAgHaT4hJiv8FYQdsWzZDz78hir6GAiTI= -github.com/dell/gocsi v1.8.1-0.20230915044639-4bab90258ed0 h1:EL+IakztajSHXZXIv2RvXcLP4E5FGqrZsAuoq3ZpMpA= -github.com/dell/gocsi v1.8.1-0.20230915044639-4bab90258ed0/go.mod h1:dclFEZScxwDjv/jdZ74CH5Pr2rYTDRllmX9zlU0Lzq8= -github.com/dell/gofsutil v1.13.1 h1:hu26rfykH0gvpSxPe5lTBVCHZA3m896/iO+2Ekz0U7A= -github.com/dell/gofsutil v1.13.1/go.mod h1:UPRuS1blrPnfT2K3nWRrLHIosZsBznDglovA6DRMmUI= -github.com/dell/goiscsi v1.8.0 h1:kocGVOdgnufc6eGpfmwP66hyhY7OVgIafaS/+uM6ogU= -github.com/dell/goiscsi v1.8.0/go.mod h1:PTlQGJaGKYgia95mGwwHSBgvfOr3BfLIjGNh1HT6p+s= -github.com/dell/gonvme v1.5.0 h1:n73WeQSFaVOlAqjhtk5T3pbu7eZgMTLpPo8/8JymOJ8= -github.com/dell/gonvme v1.5.0/go.mod h1:7MFbd7lWSaQwR5pf9ZnVZqhkAKkveSwQEO67jDBZuX0= -github.com/dell/gopowerstore v1.13.1-0.20231012074319-4009fe74aa49 h1:4omEzu7pEzUZPJpYql5sTy4qIw/+bt5GFsaVPu8jtyo= -github.com/dell/gopowerstore v1.13.1-0.20231012074319-4009fe74aa49/go.mod h1:MJLVe9FLxhb/dXv8RuKtHwqGAAmRuAIMFqNOTK3ZMz0= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful/v3 v3.10.0 h1:X4gma4HM7hFm6WMeAsTfqA0GOfdNoCzBIkHGoRLGXuM= -github.com/emicklei/go-restful/v3 v3.10.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= +github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.11.0+incompatible h1:glyUF9yIYtMHzn8xaKw5rMhdWcwsYV8dZHIq5567/xs= github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= +github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= -github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= -github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= -github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -180,39 +167,71 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/zapr v1.2.0/go.mod h1:Qa4Bsj2Vb+FAVeAKsLD8RLQ+YRJB8YDmOAKxaBQf7Ro= -github.com/go-openapi/errors v0.20.2 h1:dxy7PGTqEh94zj2E3h1cUmQQWiM1+aeCROfAr02EmK8= -github.com/go-openapi/errors v0.20.2/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-openapi/errors v0.22.4 h1:oi2K9mHTOb5DPW2Zjdzs/NIvwi2N3fARKaTJLdNabaM= +github.com/go-openapi/errors v0.22.4/go.mod h1:z9S8ASTUqx7+CP1Q8dD8ewGH/1JWFFLX/2PmAYNQLgk= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= -github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.22.3 h1:dKMwfV4fmt6Ah90zloTbUKWMD+0he+12XYAsPotrkn8= +github.com/go-openapi/jsonpointer v0.22.3/go.mod h1:0lBbqeRsQ5lIanv3LHZBrmRGHLHcQoOXQnf88fHlGWo= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= -github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA= -github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= -github.com/go-openapi/strfmt v0.21.3 h1:xwhj5X6CjXEZZHMWy1zKJxvW9AfHC9pkyUjLvHtKG7o= -github.com/go-openapi/strfmt v0.21.3/go.mod h1:k+RzNO0Da+k3FrrynSNN8F7n/peCmQQqbbXjtDfvmGg= +github.com/go-openapi/jsonreference v0.21.3 h1:96Dn+MRPa0nYAR8DR1E03SblB5FJvh7W6krPI0Z7qMc= +github.com/go-openapi/jsonreference v0.21.3/go.mod h1:RqkUP0MrLf37HqxZxrIAtTWW4ZJIK1VzduhXYBEeGc4= +github.com/go-openapi/strfmt v0.25.0 h1:7R0RX7mbKLa9EYCTHRcCuIPcaqlyQiWNPTXwClK0saQ= +github.com/go-openapi/strfmt v0.25.0/go.mod h1:nNXct7OzbwrMY9+5tLX4I21pzcmE6ccMGXl3jFdPfn8= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= -github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/swag v0.25.4 h1:OyUPUFYDPDBMkqyxOTkqDYFnrhuhi9NR6QVUvIochMU= +github.com/go-openapi/swag v0.25.4/go.mod h1:zNfJ9WZABGHCFg2RnY0S4IOkAcVTzJ6z2Bi+Q4i6qFQ= +github.com/go-openapi/swag/cmdutils v0.25.4 h1:8rYhB5n6WawR192/BfUu2iVlxqVR9aRgGJP6WaBoW+4= +github.com/go-openapi/swag/cmdutils v0.25.4/go.mod h1:pdae/AFo6WxLl5L0rq87eRzVPm/XRHM3MoYgRMvG4A0= +github.com/go-openapi/swag/conv v0.25.4 h1:/Dd7p0LZXczgUcC/Ikm1+YqVzkEeCc9LnOWjfkpkfe4= +github.com/go-openapi/swag/conv v0.25.4/go.mod h1:3LXfie/lwoAv0NHoEuY1hjoFAYkvlqI/Bn5EQDD3PPU= +github.com/go-openapi/swag/fileutils v0.25.4 h1:2oI0XNW5y6UWZTC7vAxC8hmsK/tOkWXHJQH4lKjqw+Y= +github.com/go-openapi/swag/fileutils v0.25.4/go.mod h1:cdOT/PKbwcysVQ9Tpr0q20lQKH7MGhOEb6EwmHOirUk= +github.com/go-openapi/swag/jsonname v0.25.4 h1:bZH0+MsS03MbnwBXYhuTttMOqk+5KcQ9869Vye1bNHI= +github.com/go-openapi/swag/jsonname v0.25.4/go.mod h1:GPVEk9CWVhNvWhZgrnvRA6utbAltopbKwDu8mXNUMag= +github.com/go-openapi/swag/jsonutils v0.25.4 h1:VSchfbGhD4UTf4vCdR2F4TLBdLwHyUDTd1/q4i+jGZA= +github.com/go-openapi/swag/jsonutils v0.25.4/go.mod h1:7OYGXpvVFPn4PpaSdPHJBtF0iGnbEaTk8AvBkoWnaAY= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.4 h1:IACsSvBhiNJwlDix7wq39SS2Fh7lUOCJRmx/4SN4sVo= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.4/go.mod h1:Mt0Ost9l3cUzVv4OEZG+WSeoHwjWLnarzMePNDAOBiM= +github.com/go-openapi/swag/loading v0.25.4 h1:jN4MvLj0X6yhCDduRsxDDw1aHe+ZWoLjW+9ZQWIKn2s= +github.com/go-openapi/swag/loading v0.25.4/go.mod h1:rpUM1ZiyEP9+mNLIQUdMiD7dCETXvkkC30z53i+ftTE= +github.com/go-openapi/swag/mangling v0.25.4 h1:2b9kBJk9JvPgxr36V23FxJLdwBrpijI26Bx5JH4Hp48= +github.com/go-openapi/swag/mangling v0.25.4/go.mod h1:6dxwu6QyORHpIIApsdZgb6wBk/DPU15MdyYj/ikn0Hg= +github.com/go-openapi/swag/netutils v0.25.4 h1:Gqe6K71bGRb3ZQLusdI8p/y1KLgV4M/k+/HzVSqT8H0= +github.com/go-openapi/swag/netutils v0.25.4/go.mod h1:m2W8dtdaoX7oj9rEttLyTeEFFEBvnAx9qHd5nJEBzYg= +github.com/go-openapi/swag/stringutils v0.25.4 h1:O6dU1Rd8bej4HPA3/CLPciNBBDwZj9HiEpdVsb8B5A8= +github.com/go-openapi/swag/stringutils v0.25.4/go.mod h1:GTsRvhJW5xM5gkgiFe0fV3PUlFm0dr8vki6/VSRaZK0= +github.com/go-openapi/swag/typeutils v0.25.4 h1:1/fbZOUN472NTc39zpa+YGHn3jzHWhv42wAJSN91wRw= +github.com/go-openapi/swag/typeutils v0.25.4/go.mod h1:Ou7g//Wx8tTLS9vG0UmzfCsjZjKhpjxayRKTHXf2pTE= +github.com/go-openapi/swag/yamlutils v0.25.4 h1:6jdaeSItEUb7ioS9lFoCZ65Cne1/RZtPBZ9A56h92Sw= +github.com/go-openapi/swag/yamlutils v0.25.4/go.mod h1:MNzq1ulQu+yd8Kl7wPOut/YHAAU/H6hL91fF+E2RFwc= +github.com/go-openapi/testify/enable/yaml/v2 v2.0.2 h1:0+Y41Pz1NkbTHz8NngxTuAXxEodtNSI1WG1c/m5Akw4= +github.com/go-openapi/testify/enable/yaml/v2 v2.0.2/go.mod h1:kme83333GCtJQHXQ8UKX3IBZu6z8T5Dvy5+CW3NLUUg= +github.com/go-openapi/testify/v2 v2.0.2 h1:X999g3jeLcoY8qctY/c/Z8iBHTbwLz7R2WXd6Ub6wls= +github.com/go-openapi/testify/v2 v2.0.2/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= +github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= +github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -225,9 +244,6 @@ github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -235,7 +251,6 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -245,69 +260,65 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= -github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= -github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= +github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= +github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/gnostic-models v0.7.1 h1:SisTfuFKJSKM5CPZkffwi6coztzzeYUhc3v4yxLWH8c= +github.com/google/gnostic-models v0.7.1/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= -github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= -github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= -github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= +github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= +github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 h1:qnpSQwGEnkcRpTqNOIR6bJbR0gAorgP9CSALpRcKoAA= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1/go.mod h1:lXGCsh6c22WGtjr+qGHj1otzZpV/1kwTMAqkwZsnWRU= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.0 h1:FbSCl+KggFl+Ocym490i/EyXF4lPgLoUtcSWquBM0Rs= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.0/go.mod h1:qOchhhIlmRcqk/O9uCo/puJlyo07YINaIqdZfZG3Jkc= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 h1:NmZ1PKzSTQbuGHw9DGPFomqkkLWMC+vZCkfs+FHv1Vg= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3/go.mod h1:zQrxl1YP88HQlA6i9c63DSVPFklWpGX4OWAc9bFuaH4= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -323,7 +334,6 @@ github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= @@ -331,16 +341,13 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= -github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jarcoal/httpmock v1.2.0 h1:gSvTxxFR/MEMfsGrvRbdfpRUMBStovlSRLw0Ep1bwwc= -github.com/jarcoal/httpmock v1.2.0/go.mod h1:oCoTsnAz4+UoOUIf5lJOWV2QQIW5UoeUI6aM2YnWAZk= +github.com/jarcoal/httpmock v1.4.0 h1:BvhqnH0JAYbNudL2GMJKgOHe2CtKlzJ/5rWKyp+hc2k= +github.com/jarcoal/httpmock v1.4.0/go.mod h1:ftW1xULwo+j0R0JJkJIIi7UKigZUXCLLanykgjwBXL0= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I= +github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -356,32 +363,29 @@ github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kubernetes-csi/csi-lib-utils v0.11.0 h1:FHWOBtAZBA/hVk7v/qaXgG9Sxv0/n06DebPFuDwumqg= github.com/kubernetes-csi/csi-lib-utils v0.11.0/go.mod h1:BmGZZB16L18+9+Lgg9YWwBKfNEHIDdgGfAyuW6p2NV0= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= -github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= -github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= @@ -392,31 +396,27 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A= -github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= +github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -425,69 +425,66 @@ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108 github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.4.0 h1:+Ig9nvqgS5OBSACXNk15PLdp0U9XPYROt9CFzVdFGIs= -github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= +github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns= +github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.23.0 h1:/oxKu9c2HVap+F3PfKort2Hw5DEU+HGlW8n+tguWsys= -github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= +github.com/onsi/gomega v1.39.0 h1:y2ROC3hKFmQZJNFeGAMeHZKkjBL65mIZcvrLQBF9k6Q= +github.com/onsi/gomega v1.39.0/go.mod h1:ZCU1pkQcXDO5Sl9/VVEGlDyp+zm0m1cmeG5TOzLgdh4= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= -github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml/v2 v2.0.1 h1:8e3L2cCQzLFi2CR4g7vGFuFxX7Jl1kKX8gW+iV0GUKU= -github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34= -github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.34.0 h1:RBmGO9d/FVjqHT0yUGQwBJhkwKV+wPCn7KGpvfab0uE= -github.com/prometheus/common v0.34.0/go.mod h1:gB3sOl7P0TvJabZpLY5uQMpUqRCPPCyRLCZYc7JZTNE= +github.com/prometheus/common v0.67.4 h1:yR3NqWO1/UyO1w2PhUvXlGQs/PtFmoveVO0KZ4+Lvsc= +github.com/prometheus/common v0.67.4/go.mod h1:gP0fq6YjjNCLssJCQp0yk4M8W6ikLURwkdd/YKtTbyI= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= -github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws= +github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sagikazarmark/locafero v0.12.0 h1:/NQhBAkUb4+fH1jivKHWusDYFjMOOKU88eegjfxfHb4= +github.com/sagikazarmark/locafero v0.12.0/go.mod h1:sZh36u/YSZ918v0Io+U9ogLYQJ9tLLBmM4eneO6WwsI= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -498,107 +495,133 @@ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVs github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= +github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/afero v1.8.2 h1:xehSyVa0YnHWsJ49JFljMpg1HX19V6NDZ1fkm1Xznbo= -github.com/spf13/afero v1.8.2/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo= +github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= +github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= -github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= +github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= +github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= -github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= -github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= -github.com/spf13/viper v1.12.0 h1:CZ7eSOd3kZoaYDLbXnmzgQI5RlciuXBMA+18HwHRfZQ= -github.com/spf13/viper v1.12.0/go.mod h1:b6COn30jlNxbm/V2IqWiNWkJ+vZNiMNksliPCiuKtSI= +github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU= +github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/subosito/gotenv v1.3.0 h1:mjC+YW8QpAdXibNi+vNWgzmgBH4+5l5dCXv8cNysBLI= -github.com/subosito/gotenv v1.3.0/go.mod h1:YzJjq/33h7nrwdY+iHMhEOEEbW0ovIz0tB6t6PwAXzs= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/thecodeteam/gosync v0.1.0 h1:RcD9owCaiK0Jg1rIDPgirdcLCL1jCD6XlDVSg0MfHmE= github.com/thecodeteam/gosync v0.1.0/go.mod h1:43QHsngcnWc8GE1aCmi7PEypslflHjCzXFleuWKEb00= -github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= -github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= +github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o= github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg= github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= -github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= -github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= -github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 h1:S2dVYn90KE98chqDkyE9Z4N61UnQd+KOfgp5Iu53llk= +github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/etcd/api/v3 v3.5.4 h1:OHVyt3TopwtUQ2GKdd5wu3PmmipR4FTwCqoEjSyRdIc= -go.etcd.io/etcd/api/v3 v3.5.4/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A= -go.etcd.io/etcd/client/pkg/v3 v3.5.4 h1:lrneYvz923dvC14R54XcA7FXoZ3mlGZAgmwhfm7HqOg= -go.etcd.io/etcd/client/pkg/v3 v3.5.4/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/v3 v3.5.4 h1:p83BUL3tAYS0OT/r0qglgc3M1JjhM0diV8DSWAhVXv4= -go.etcd.io/etcd/client/v3 v3.5.4/go.mod h1:ZaRkVgBZC+L+dLCjTcF1hRXpgZXQPOvnA/Ak/gq3kiY= -go.mongodb.org/mongo-driver v1.10.0/go.mod h1:wsihk0Kdgv8Kqu1Anit4sfK+22vSFbUrAVEYRhCXrA8= -go.mongodb.org/mongo-driver v1.11.2 h1:+1v2rDQUWNcGW7/7E0Jvdz51V38XXxJfhzbV17aNHCw= -go.mongodb.org/mongo-driver v1.11.2/go.mod h1:s7p5vEtfbeR1gYi6pnj3c3/urpbLv2T5Sfd6Rp2HBB8= +go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo= +go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E= +go.etcd.io/etcd/api/v3 v3.6.6 h1:mcaMp3+7JawWv69p6QShYWS8cIWUOl32bFLb6qf8pOQ= +go.etcd.io/etcd/api/v3 v3.6.6/go.mod h1:f/om26iXl2wSkcTA1zGQv8reJRSLVdoEBsi4JdfMrx4= +go.etcd.io/etcd/client/pkg/v3 v3.6.6 h1:uoqgzSOv2H9KlIF5O1Lsd8sW+eMLuV6wzE3q5GJGQNs= +go.etcd.io/etcd/client/pkg/v3 v3.6.6/go.mod h1:YngfUVmvsvOJ2rRgStIyHsKtOt9SZI2aBJrZiWJhCbI= +go.etcd.io/etcd/client/v3 v3.6.6 h1:G5z1wMf5B9SNexoxOHUGBaULurOZPIgGPsW6CN492ec= +go.etcd.io/etcd/client/v3 v3.6.6/go.mod h1:36Qv6baQ07znPR3+n7t+Rk5VHEzVYPvFfGmfF4wBHV8= +go.etcd.io/etcd/pkg/v3 v3.6.6 h1:wylOivS/UxXTZ0Le5fOdxCjatW5ql9dcWEggQQHSorw= +go.etcd.io/etcd/pkg/v3 v3.6.6/go.mod h1:9TKZL7WUEVHXYM3srP3ESZfIms34s1G72eNtWA9YKg4= +go.etcd.io/etcd/server/v3 v3.6.6 h1:YSRWGJPzU+lIREwUQI4MfyLZrkUyzjJOVpMxJvZePaY= +go.etcd.io/etcd/server/v3 v3.6.6/go.mod h1:A1OQ1x3PaiENDLywMjCiMwV1pwJSpb0h9Z5ORP2dv6I= +go.etcd.io/raft/v3 v3.6.0 h1:5NtvbDVYpnfZWcIHgGRk9DyzkBIXOi8j+DDp1IcnUWQ= +go.etcd.io/raft/v3 v3.6.0/go.mod h1:nLvLevg6+xrVtHUmVaTcTz603gQPHfh7kUAwV6YpfGo= +go.mongodb.org/mongo-driver v1.17.6 h1:87JUG1wZfWsr6rIz3ZmpH90rL5tea7O3IHuSwHUpsss= +go.mongodb.org/mongo-driver v1.17.6/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 h1:YH4g8lQroajqUwWbq/tr2QX1JFmEXaDLgG+ew9bLMWo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0/go.mod h1:fvPi2qXDqFs8M4B4fmJhE92TyQs9Ydjlg3RvfUp+NbQ= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel/exporters/otlp v0.20.0 h1:PTNgq9MRmQqqJY0REVbZFvwkYOA85vbdQU/nVfxDyqg= go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 h1:OeNbIYk/2C15ckl7glBlOBp5+WlYsOElzTNmiPW/x60= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0/go.mod h1:7Bept48yIeqxP2OZ9/AqIpYS94h2or0aB4FypJTc8ZM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 h1:tgJ0uaNS4c98WRNUEx5U3aDlrDOI5Rs+1Vifcw4DJ8U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0/go.mod h1:U7HYyW0zt/a9x5J1Kjs+r1f/d4ZHnYFclhYY2+YbeoE= go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= +go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= +go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= +go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= -go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= -go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= +go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= -go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= -go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8= -go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= +go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= +go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -608,11 +631,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8= +golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -639,7 +659,6 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -649,10 +668,9 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI= +golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -669,7 +687,6 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -679,60 +696,32 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= -golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= +golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= -golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= +golang.org/x/oauth2 v0.33.0 h1:4Q+qn+E5z8gPRJfmRy7C2gGG3T4jIprK6aSYgTXGRpo= +golang.org/x/oauth2 v0.33.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f h1:Ax0t5p6N38Ga0dThY21weqDEyz2oklo4IvDkpigvkD8= -golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -764,69 +753,44 @@ golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= +golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= -golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= +golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY= +golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= -golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE= +golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220411224347-583f2d630306 h1:+gHMid33q6pen7kv9xvT+JRinntgeXO2AeZVd0AWD3w= -golang.org/x/time v0.0.0-20220411224347-583f2d630306/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -863,38 +827,24 @@ golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= +golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA= +golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.5.0 h1:JELs8RLM12qJGXU4u/TO3V25KW8GreMKl9pdkk14RM0= +gomodules.xyz/jsonpatch/v2 v2.5.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= @@ -906,26 +856,12 @@ google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= -google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -943,41 +879,16 @@ google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 h1:L6iMMGrtzgHsWofoFcihmDEMYeDR9KN/ThbPWGrh++g= -google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5/go.mod h1:oH/ZOT02u4kWEp7oYBGYFFkCdKS/uYR9Z7+0/xuuFp8= -google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e h1:z3vDksarJxsAKM5dmEGv0GHwE2hKJ096wZra71Vs4sw= -google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230815205213-6bfd019c3878 h1:lv6/DhyiFFGsmzxbsUUTOkN29II+zeWHxvT8Lpdxsv0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230815205213-6bfd019c3878/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= +google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 h1:fCvbg86sFXwdrl5LgVcTEvNC+2txB5mgROGmRL5mrls= +google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:+rXWjjaukWZun3mLfjmVnQi18E1AsFbDN9QdJ5YXLto= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b h1:Mv8VFug0MP9e5vUxfBcE3vUkV6CImK3cMNMIDFjmzxU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -986,22 +897,13 @@ google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQ google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= +google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1014,22 +916,24 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= +gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.66.6 h1:LATuAqN/shcYAOkv3wl2L4rkaKqkcgTBQjOyYDvcPKI= -gopkg.in/ini.v1 v1.66.6/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= @@ -1041,10 +945,8 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= @@ -1057,51 +959,47 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.22.0/go.mod h1:0AoXXqst47OI/L0oGKq9DG61dvGRPXs7X4/B7KyjBCU= -k8s.io/api v0.24.1/go.mod h1:JhoOvNiLXKTPQ60zh2g0ewpA+bnEYf5q44Flhquh4vQ= -k8s.io/api v0.26.1 h1:f+SWYiPd/GsiWwVRz+NbFyCgvv75Pk9NK6dlkZgpCRQ= -k8s.io/api v0.26.1/go.mod h1:xd/GBNgR0f707+ATNyPmQ1oyKSgndzXij81FzWGsejg= +k8s.io/api v0.35.0 h1:iBAU5LTyBI9vw3L5glmat1njFK34srdLmktWwLTprlY= +k8s.io/api v0.35.0/go.mod h1:AQ0SNTzm4ZAczM03QH42c7l3bih1TbAXYo0DkF8ktnA= +k8s.io/apiextensions-apiserver v0.34.2 h1:WStKftnGeoKP4AZRz/BaAAEJvYp4mlZGN0UCv+uvsqo= +k8s.io/apiextensions-apiserver v0.34.2/go.mod h1:398CJrsgXF1wytdaanynDpJ67zG4Xq7yj91GrmYN2SE= k8s.io/apimachinery v0.22.0/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= -k8s.io/apimachinery v0.24.1/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= -k8s.io/apimachinery v0.26.1 h1:8EZ/eGJL+hY/MYCNwhmDzVqq2lPl3N3Bo8rvweJwXUQ= -k8s.io/apimachinery v0.26.1/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74= +k8s.io/apimachinery v0.35.0 h1:Z2L3IHvPVv/MJ7xRxHEtk6GoJElaAqDCCU0S6ncYok8= +k8s.io/apimachinery v0.35.0/go.mod h1:jQCgFZFR1F4Ik7hvr2g84RTJSZegBc8yHgFWKn//hns= k8s.io/client-go v0.22.0/go.mod h1:GUjIuXR5PiEv/RVK5OODUsm6eZk7wtSWZSaSJbpFdGg= -k8s.io/client-go v0.24.1/go.mod h1:f1kIDqcEYmwXS/vTbbhopMUbhKp2JhOeVTfxgaCIlF8= -k8s.io/client-go v0.26.1 h1:87CXzYJnAMGaa/IDDfRdhTzxk/wzGZ+/HUQpqgVSZXU= -k8s.io/client-go v0.26.1/go.mod h1:IWNSglg+rQ3OcvDkhY6+QLeasV4OYHDjdqeWkDQZwGE= +k8s.io/client-go v0.35.0 h1:IAW0ifFbfQQwQmga0UdoH0yvdqrbwMdq9vIFEhRpxBE= +k8s.io/client-go v0.35.0/go.mod h1:q2E5AAyqcbeLGPdoRB+Nxe3KYTfPce1Dnu1myQdqz9o= k8s.io/component-base v0.22.0/go.mod h1:SXj6Z+V6P6GsBhHZVbWCw9hFjUdUYnJerlhhPnYCBCg= -k8s.io/component-base v0.24.1 h1:APv6W/YmfOWZfo+XJ1mZwep/f7g7Tpwvdbo9CQLDuts= -k8s.io/component-base v0.24.1/go.mod h1:DW5vQGYVCog8WYpNob3PMmmsY8A3L9QZNg4j/dV3s38= +k8s.io/component-base v0.34.2 h1:HQRqK9x2sSAsd8+R4xxRirlTjowsg6fWCPwWYeSvogQ= +k8s.io/component-base v0.34.2/go.mod h1:9xw2FHJavUHBFpiGkZoKuYZ5pdtLKe97DEByaA+hHbM= +k8s.io/component-helpers v0.35.0 h1:wcXv7HJRksgVjM4VlXJ1CNFBpyDHruRI99RrBtrJceA= +k8s.io/component-helpers v0.35.0/go.mod h1:ahX0m/LTYmu7fL3W8zYiIwnQ/5gT28Ex4o2pymF63Co= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= -k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= -k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= -k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= -k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42/go.mod h1:Z/45zLw8lUo4wdiUkI+v/ImEGAvu3WatcZl3lPMR4Rk= -k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E= -k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= +k8s.io/kube-openapi v0.0.0-20251125145642-4e65d59e963e h1:iW9ChlU0cU16w8MpVYjXk12dqQ4BPFBEgif+ap7/hqQ= +k8s.io/kube-openapi v0.0.0-20251125145642-4e65d59e963e/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ= k8s.io/utils v0.0.0-20210707171843-4b05e18ac7d9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20230202215443-34013725500c h1:YVqDar2X7YiQa/DVAXFMDIfGF8uGrHQemlrwRU5NlVI= -k8s.io/utils v0.0.0-20230202215443-34013725500c/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 h1:SjGebBtkBqHFOli+05xYbK8YF1Dzkbzn+gDM4X9T4Ck= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/controller-runtime v0.22.4 h1:GEjV7KV3TY8e+tJ2LCTxUTanW4z/FmNB7l327UfMq9A= +sigs.k8s.io/controller-runtime v0.22.4/go.mod h1:+QX1XUpTXN4mLoblf4tqr5CQcyHPAki2HLXqQMY6vh8= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= -sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= +sigs.k8s.io/structured-merge-diff/v6 v6.3.1 h1:JrhdFMqOd/+3ByqlP2I45kTOZmTRLBUm5pvRjeheg7E= +sigs.k8s.io/structured-merge-diff/v6 v6.3.1/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= -sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= -sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/helper.mk b/helper.mk new file mode 100644 index 00000000..01633ce3 --- /dev/null +++ b/helper.mk @@ -0,0 +1,17 @@ +# Copyright © 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Dell Technologies, Dell and other trademarks are trademarks of Dell Inc. +# or its subsidiaries. Other trademarks may be trademarks of their respective +# owners. + +generate: + go generate ./cmd/csi-powerstore + go run core/semver/semver.go -f mk > semver.mk + +download-csm-common: + git clone --depth 1 git@github.com:CSM/csm.git temp-repo + cp temp-repo/config/csm-common.mk . + rm -rf temp-repo + +vendor: + GOPRIVATE=github.com go mod vendor diff --git a/images.mk b/images.mk new file mode 100644 index 00000000..5121dc17 --- /dev/null +++ b/images.mk @@ -0,0 +1,22 @@ +# Copyright © 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Dell Technologies, Dell and other trademarks are trademarks of Dell Inc. +# or its subsidiaries. Other trademarks may be trademarks of their respective +# owners. + +include overrides.mk +include helper.mk + +images: download-csm-common generate vendor + $(eval include csm-common.mk) + @echo "Base Images is set to: $(BASEIMAGE)" + @echo "Building: $(IMAGE_REGISTRY)/$(IMAGE_NAME):$(IMAGE_TAG)" + $(BUILDER) build --pull $(NOCACHE) -t "$(IMAGE_REGISTRY)/$(IMAGE_NAME):$(IMAGE_TAG)" --build-arg GOIMAGE=$(DEFAULT_GOIMAGE) --build-arg BASEIMAGE=$(CSM_BASEIMAGE) . + +images-no-cache: + @echo "Building with --no-cache ..." + @make images NOCACHE=--no-cache + +push: + @echo "Pushing: $(IMAGE_REGISTRY)/$(IMAGE_NAME):$(IMAGE_TAG)" + $(BUILDER) push "$(IMAGE_REGISTRY)/$(IMAGE_NAME):$(IMAGE_TAG)" diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 00000000..c36cc9ce --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,7 @@ +site_name: 'csi-powerstore' +site_description: 'csi-powerstore Documentation.' +docs_dir: docs +plugins: + - techdocs-core +theme: + name: material diff --git a/mocks/Configurator.go b/mocks/Configurator.go new file mode 100644 index 00000000..4ab39680 --- /dev/null +++ b/mocks/Configurator.go @@ -0,0 +1,57 @@ +// Code generated by mockery. DO NOT EDIT. + +package mocks + +import ( + mock "github.com/stretchr/testify/mock" + config "github.com/uber/jaeger-client-go/config" +) + +// Configurator is an autogenerated mock type for the Configurator type +type Configurator struct { + mock.Mock +} + +// FromEnv provides a mock function with no fields +func (_m *Configurator) FromEnv() (*config.Configuration, error) { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for FromEnv") + } + + var r0 *config.Configuration + var r1 error + if rf, ok := ret.Get(0).(func() (*config.Configuration, error)); ok { + return rf() + } + if rf, ok := ret.Get(0).(func() *config.Configuration); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*config.Configuration) + } + } + + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// NewConfigurator creates a new instance of Configurator. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewConfigurator(t interface { + mock.TestingT + Cleanup(func()) +}) *Configurator { + mock := &Configurator{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/mocks/Consumer.go b/mocks/Consumer.go index 22ce9467..f5ddabbd 100644 --- a/mocks/Consumer.go +++ b/mocks/Consumer.go @@ -1,36 +1,27 @@ -/* - * - * Copyright © 2021-2023 Dell Inc. or its subsidiaries. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -// Code generated by mockery v1.0.0. DO NOT EDIT. +// Code generated by mockery. DO NOT EDIT. package mocks -import array "github.com/dell/csi-powerstore/v2/pkg/array" -import fs "github.com/dell/csi-powerstore/v2/pkg/common/fs" -import mock "github.com/stretchr/testify/mock" +import ( + array "github.com/dell/csi-powerstore/v2/pkg/array" + fs "github.com/dell/csi-powerstore/v2/pkg/identifiers/fs" + + mock "github.com/stretchr/testify/mock" +) // Consumer is an autogenerated mock type for the Consumer type type Consumer struct { mock.Mock } -// Arrays provides a mock function with given fields: +// Arrays provides a mock function with no fields func (_m *Consumer) Arrays() map[string]*array.PowerStoreArray { ret := _m.Called() + if len(ret) == 0 { + panic("no return value specified for Arrays") + } + var r0 map[string]*array.PowerStoreArray if rf, ok := ret.Get(0).(func() map[string]*array.PowerStoreArray); ok { r0 = rf() @@ -43,10 +34,14 @@ func (_m *Consumer) Arrays() map[string]*array.PowerStoreArray { return r0 } -// DefaultArray provides a mock function with given fields: +// DefaultArray provides a mock function with no fields func (_m *Consumer) DefaultArray() *array.PowerStoreArray { ret := _m.Called() + if len(ret) == 0 { + panic("no return value specified for DefaultArray") + } + var r0 *array.PowerStoreArray if rf, ok := ret.Get(0).(func() *array.PowerStoreArray); ok { r0 = rf() @@ -59,20 +54,6 @@ func (_m *Consumer) DefaultArray() *array.PowerStoreArray { return r0 } -// RegisterK8sCluster provides a mock function with given fields: _a0 -func (_m *Consumer) RegisterK8sCluster(_a0 fs.Interface) error { - ret := _m.Called(_a0) - - var r0 error - if rf, ok := ret.Get(0).(func(fs.Interface) error); ok { - r0 = rf(_a0) - } else { - r0 = ret.Error(0) - } - - return r0 -} - // SetArrays provides a mock function with given fields: _a0 func (_m *Consumer) SetArrays(_a0 map[string]*array.PowerStoreArray) { _m.Called(_a0) @@ -87,6 +68,10 @@ func (_m *Consumer) SetDefaultArray(_a0 *array.PowerStoreArray) { func (_m *Consumer) UpdateArrays(_a0 string, _a1 fs.Interface) error { ret := _m.Called(_a0, _a1) + if len(ret) == 0 { + panic("no return value specified for UpdateArrays") + } + var r0 error if rf, ok := ret.Get(0).(func(string, fs.Interface) error); ok { r0 = rf(_a0, _a1) @@ -96,3 +81,17 @@ func (_m *Consumer) UpdateArrays(_a0 string, _a1 fs.Interface) error { return r0 } + +// NewConsumer creates a new instance of Consumer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewConsumer(t interface { + mock.TestingT + Cleanup(func()) +}) *Consumer { + mock := &Consumer{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/mocks/ControllerInterface.go b/mocks/ControllerInterface.go index a7411f5f..3d82e72d 100644 --- a/mocks/ControllerInterface.go +++ b/mocks/ControllerInterface.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2021-2023 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2021-2025 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,13 +19,16 @@ package mocks import ( - context "context" - array "github.com/dell/csi-powerstore/v2/pkg/array" + common "github.com/dell/dell-csi-extensions/common" + + context "context" csi "github.com/container-storage-interface/spec/lib/go/csi" - fs "github.com/dell/csi-powerstore/v2/pkg/common/fs" + fs "github.com/dell/csi-powerstore/v2/pkg/identifiers/fs" + + grpc "google.golang.org/grpc" mock "github.com/stretchr/testify/mock" ) @@ -35,10 +38,14 @@ type ControllerInterface struct { mock.Mock } -// Arrays provides a mock function with given fields: +// Arrays provides a mock function with no fields func (_m *ControllerInterface) Arrays() map[string]*array.PowerStoreArray { ret := _m.Called() + if len(ret) == 0 { + panic("no return value specified for Arrays") + } + var r0 map[string]*array.PowerStoreArray if rf, ok := ret.Get(0).(func() map[string]*array.PowerStoreArray); ok { r0 = rf() @@ -51,22 +58,119 @@ func (_m *ControllerInterface) Arrays() map[string]*array.PowerStoreArray { return r0 } -// ControllerPublishVolume provides a mock function with given fields: ctx, req -func (_m *ControllerInterface) ControllerPublishVolume(ctx context.Context, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) { - ret := _m.Called(ctx, req) +// ControllerExpandVolume provides a mock function with given fields: _a0, _a1 +func (_m *ControllerInterface) ControllerExpandVolume(_a0 context.Context, _a1 *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error) { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for ControllerExpandVolume") + } + + var r0 *csi.ControllerExpandVolumeResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(context.Context, *csi.ControllerExpandVolumeRequest) *csi.ControllerExpandVolumeResponse); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*csi.ControllerExpandVolumeResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *csi.ControllerExpandVolumeRequest) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ControllerGetCapabilities provides a mock function with given fields: _a0, _a1 +func (_m *ControllerInterface) ControllerGetCapabilities(_a0 context.Context, _a1 *csi.ControllerGetCapabilitiesRequest) (*csi.ControllerGetCapabilitiesResponse, error) { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for ControllerGetCapabilities") + } + + var r0 *csi.ControllerGetCapabilitiesResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *csi.ControllerGetCapabilitiesRequest) (*csi.ControllerGetCapabilitiesResponse, error)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(context.Context, *csi.ControllerGetCapabilitiesRequest) *csi.ControllerGetCapabilitiesResponse); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*csi.ControllerGetCapabilitiesResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *csi.ControllerGetCapabilitiesRequest) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ControllerGetVolume provides a mock function with given fields: _a0, _a1 +func (_m *ControllerInterface) ControllerGetVolume(_a0 context.Context, _a1 *csi.ControllerGetVolumeRequest) (*csi.ControllerGetVolumeResponse, error) { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for ControllerGetVolume") + } + + var r0 *csi.ControllerGetVolumeResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *csi.ControllerGetVolumeRequest) (*csi.ControllerGetVolumeResponse, error)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(context.Context, *csi.ControllerGetVolumeRequest) *csi.ControllerGetVolumeResponse); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*csi.ControllerGetVolumeResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *csi.ControllerGetVolumeRequest) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ControllerPublishVolume provides a mock function with given fields: _a0, _a1 +func (_m *ControllerInterface) ControllerPublishVolume(_a0 context.Context, _a1 *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for ControllerPublishVolume") + } var r0 *csi.ControllerPublishVolumeResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error)); ok { + return rf(_a0, _a1) + } if rf, ok := ret.Get(0).(func(context.Context, *csi.ControllerPublishVolumeRequest) *csi.ControllerPublishVolumeResponse); ok { - r0 = rf(ctx, req) + r0 = rf(_a0, _a1) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*csi.ControllerPublishVolumeResponse) } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, *csi.ControllerPublishVolumeRequest) error); ok { - r1 = rf(ctx, req) + r1 = rf(_a0, _a1) } else { r1 = ret.Error(1) } @@ -74,22 +178,59 @@ func (_m *ControllerInterface) ControllerPublishVolume(ctx context.Context, req return r0, r1 } -// ControllerUnpublishVolume provides a mock function with given fields: ctx, req -func (_m *ControllerInterface) ControllerUnpublishVolume(ctx context.Context, req *csi.ControllerUnpublishVolumeRequest) (*csi.ControllerUnpublishVolumeResponse, error) { - ret := _m.Called(ctx, req) +// ControllerUnpublishVolume provides a mock function with given fields: _a0, _a1 +func (_m *ControllerInterface) ControllerUnpublishVolume(_a0 context.Context, _a1 *csi.ControllerUnpublishVolumeRequest) (*csi.ControllerUnpublishVolumeResponse, error) { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for ControllerUnpublishVolume") + } var r0 *csi.ControllerUnpublishVolumeResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *csi.ControllerUnpublishVolumeRequest) (*csi.ControllerUnpublishVolumeResponse, error)); ok { + return rf(_a0, _a1) + } if rf, ok := ret.Get(0).(func(context.Context, *csi.ControllerUnpublishVolumeRequest) *csi.ControllerUnpublishVolumeResponse); ok { - r0 = rf(ctx, req) + r0 = rf(_a0, _a1) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*csi.ControllerUnpublishVolumeResponse) } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, *csi.ControllerUnpublishVolumeRequest) error); ok { - r1 = rf(ctx, req) + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// CreateSnapshot provides a mock function with given fields: _a0, _a1 +func (_m *ControllerInterface) CreateSnapshot(_a0 context.Context, _a1 *csi.CreateSnapshotRequest) (*csi.CreateSnapshotResponse, error) { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for CreateSnapshot") + } + + var r0 *csi.CreateSnapshotResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *csi.CreateSnapshotRequest) (*csi.CreateSnapshotResponse, error)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(context.Context, *csi.CreateSnapshotRequest) *csi.CreateSnapshotResponse); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*csi.CreateSnapshotResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *csi.CreateSnapshotRequest) error); ok { + r1 = rf(_a0, _a1) } else { r1 = ret.Error(1) } @@ -97,22 +238,29 @@ func (_m *ControllerInterface) ControllerUnpublishVolume(ctx context.Context, re return r0, r1 } -// CreateVolume provides a mock function with given fields: ctx, req -func (_m *ControllerInterface) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { - ret := _m.Called(ctx, req) +// CreateVolume provides a mock function with given fields: _a0, _a1 +func (_m *ControllerInterface) CreateVolume(_a0 context.Context, _a1 *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for CreateVolume") + } var r0 *csi.CreateVolumeResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error)); ok { + return rf(_a0, _a1) + } if rf, ok := ret.Get(0).(func(context.Context, *csi.CreateVolumeRequest) *csi.CreateVolumeResponse); ok { - r0 = rf(ctx, req) + r0 = rf(_a0, _a1) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*csi.CreateVolumeResponse) } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, *csi.CreateVolumeRequest) error); ok { - r1 = rf(ctx, req) + r1 = rf(_a0, _a1) } else { r1 = ret.Error(1) } @@ -120,10 +268,14 @@ func (_m *ControllerInterface) CreateVolume(ctx context.Context, req *csi.Create return r0, r1 } -// DefaultArray provides a mock function with given fields: +// DefaultArray provides a mock function with no fields func (_m *ControllerInterface) DefaultArray() *array.PowerStoreArray { ret := _m.Called() + if len(ret) == 0 { + panic("no return value specified for DefaultArray") + } + var r0 *array.PowerStoreArray if rf, ok := ret.Get(0).(func() *array.PowerStoreArray); ok { r0 = rf() @@ -136,22 +288,59 @@ func (_m *ControllerInterface) DefaultArray() *array.PowerStoreArray { return r0 } -// DeleteVolume provides a mock function with given fields: ctx, req -func (_m *ControllerInterface) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) { - ret := _m.Called(ctx, req) +// DeleteSnapshot provides a mock function with given fields: _a0, _a1 +func (_m *ControllerInterface) DeleteSnapshot(_a0 context.Context, _a1 *csi.DeleteSnapshotRequest) (*csi.DeleteSnapshotResponse, error) { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for DeleteSnapshot") + } + + var r0 *csi.DeleteSnapshotResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *csi.DeleteSnapshotRequest) (*csi.DeleteSnapshotResponse, error)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(context.Context, *csi.DeleteSnapshotRequest) *csi.DeleteSnapshotResponse); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*csi.DeleteSnapshotResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *csi.DeleteSnapshotRequest) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// DeleteVolume provides a mock function with given fields: _a0, _a1 +func (_m *ControllerInterface) DeleteVolume(_a0 context.Context, _a1 *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for DeleteVolume") + } var r0 *csi.DeleteVolumeResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error)); ok { + return rf(_a0, _a1) + } if rf, ok := ret.Get(0).(func(context.Context, *csi.DeleteVolumeRequest) *csi.DeleteVolumeResponse); ok { - r0 = rf(ctx, req) + r0 = rf(_a0, _a1) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*csi.DeleteVolumeResponse) } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, *csi.DeleteVolumeRequest) error); ok { - r1 = rf(ctx, req) + r1 = rf(_a0, _a1) } else { r1 = ret.Error(1) } @@ -159,18 +348,129 @@ func (_m *ControllerInterface) DeleteVolume(ctx context.Context, req *csi.Delete return r0, r1 } -// RegisterK8sCluster provides a mock function with given fields: _a0 -func (_m *ControllerInterface) RegisterK8sCluster(_a0 fs.Interface) error { - ret := _m.Called(_a0) +// GetCapacity provides a mock function with given fields: _a0, _a1 +func (_m *ControllerInterface) GetCapacity(_a0 context.Context, _a1 *csi.GetCapacityRequest) (*csi.GetCapacityResponse, error) { + ret := _m.Called(_a0, _a1) - var r0 error - if rf, ok := ret.Get(0).(func(fs.Interface) error); ok { - r0 = rf(_a0) + if len(ret) == 0 { + panic("no return value specified for GetCapacity") + } + + var r0 *csi.GetCapacityResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *csi.GetCapacityRequest) (*csi.GetCapacityResponse, error)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(context.Context, *csi.GetCapacityRequest) *csi.GetCapacityResponse); ok { + r0 = rf(_a0, _a1) } else { - r0 = ret.Error(0) + if ret.Get(0) != nil { + r0 = ret.Get(0).(*csi.GetCapacityResponse) + } } - return r0 + if rf, ok := ret.Get(1).(func(context.Context, *csi.GetCapacityRequest) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ListSnapshots provides a mock function with given fields: _a0, _a1 +func (_m *ControllerInterface) ListSnapshots(_a0 context.Context, _a1 *csi.ListSnapshotsRequest) (*csi.ListSnapshotsResponse, error) { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for ListSnapshots") + } + + var r0 *csi.ListSnapshotsResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *csi.ListSnapshotsRequest) (*csi.ListSnapshotsResponse, error)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(context.Context, *csi.ListSnapshotsRequest) *csi.ListSnapshotsResponse); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*csi.ListSnapshotsResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *csi.ListSnapshotsRequest) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ListVolumes provides a mock function with given fields: _a0, _a1 +func (_m *ControllerInterface) ListVolumes(_a0 context.Context, _a1 *csi.ListVolumesRequest) (*csi.ListVolumesResponse, error) { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for ListVolumes") + } + + var r0 *csi.ListVolumesResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *csi.ListVolumesRequest) (*csi.ListVolumesResponse, error)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(context.Context, *csi.ListVolumesRequest) *csi.ListVolumesResponse); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*csi.ListVolumesResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *csi.ListVolumesRequest) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ProbeController provides a mock function with given fields: _a0, _a1 +func (_m *ControllerInterface) ProbeController(_a0 context.Context, _a1 *common.ProbeControllerRequest) (*common.ProbeControllerResponse, error) { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for ProbeController") + } + + var r0 *common.ProbeControllerResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *common.ProbeControllerRequest) (*common.ProbeControllerResponse, error)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(context.Context, *common.ProbeControllerRequest) *common.ProbeControllerResponse); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*common.ProbeControllerResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *common.ProbeControllerRequest) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// RegisterAdditionalServers provides a mock function with given fields: _a0 +func (_m *ControllerInterface) RegisterAdditionalServers(_a0 *grpc.Server) { + _m.Called(_a0) } // SetArrays provides a mock function with given fields: _a0 @@ -187,6 +487,10 @@ func (_m *ControllerInterface) SetDefaultArray(_a0 *array.PowerStoreArray) { func (_m *ControllerInterface) UpdateArrays(_a0 string, _a1 fs.Interface) error { ret := _m.Called(_a0, _a1) + if len(ret) == 0 { + panic("no return value specified for UpdateArrays") + } + var r0 error if rf, ok := ret.Get(0).(func(string, fs.Interface) error); ok { r0 = rf(_a0, _a1) @@ -196,3 +500,47 @@ func (_m *ControllerInterface) UpdateArrays(_a0 string, _a1 fs.Interface) error return r0 } + +// ValidateVolumeCapabilities provides a mock function with given fields: _a0, _a1 +func (_m *ControllerInterface) ValidateVolumeCapabilities(_a0 context.Context, _a1 *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for ValidateVolumeCapabilities") + } + + var r0 *csi.ValidateVolumeCapabilitiesResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(context.Context, *csi.ValidateVolumeCapabilitiesRequest) *csi.ValidateVolumeCapabilitiesResponse); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*csi.ValidateVolumeCapabilitiesResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *csi.ValidateVolumeCapabilitiesRequest) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// NewControllerInterface creates a new instance of ControllerInterface. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewControllerInterface(t interface { + mock.TestingT + Cleanup(func()) +}) *ControllerInterface { + mock := &ControllerInterface{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/mocks/FcConnector.go b/mocks/FcConnector.go index 3ebdb947..a2649926 100644 --- a/mocks/FcConnector.go +++ b/mocks/FcConnector.go @@ -1,19 +1,3 @@ -/* - * - * Copyright © 2021-2022 Dell Inc. or its subsidiaries. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - // Code generated by mockery. DO NOT EDIT. package mocks @@ -34,14 +18,21 @@ type FcConnector struct { func (_m *FcConnector) ConnectVolume(ctx context.Context, info gobrick.FCVolumeInfo) (gobrick.Device, error) { ret := _m.Called(ctx, info) + if len(ret) == 0 { + panic("no return value specified for ConnectVolume") + } + var r0 gobrick.Device + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, gobrick.FCVolumeInfo) (gobrick.Device, error)); ok { + return rf(ctx, info) + } if rf, ok := ret.Get(0).(func(context.Context, gobrick.FCVolumeInfo) gobrick.Device); ok { r0 = rf(ctx, info) } else { r0 = ret.Get(0).(gobrick.Device) } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, gobrick.FCVolumeInfo) error); ok { r1 = rf(ctx, info) } else { @@ -55,6 +46,10 @@ func (_m *FcConnector) ConnectVolume(ctx context.Context, info gobrick.FCVolumeI func (_m *FcConnector) DisconnectVolumeByDeviceName(ctx context.Context, name string) error { ret := _m.Called(ctx, name) + if len(ret) == 0 { + panic("no return value specified for DisconnectVolumeByDeviceName") + } + var r0 error if rf, ok := ret.Get(0).(func(context.Context, string) error); ok { r0 = rf(ctx, name) @@ -65,11 +60,37 @@ func (_m *FcConnector) DisconnectVolumeByDeviceName(ctx context.Context, name st return r0 } +// DisconnectVolumeByWWN provides a mock function with given fields: ctx, wwn +func (_m *FcConnector) DisconnectVolumeByWWN(ctx context.Context, wwn string) error { + ret := _m.Called(ctx, wwn) + + if len(ret) == 0 { + panic("no return value specified for DisconnectVolumeByWWN") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, string) error); ok { + r0 = rf(ctx, wwn) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // GetInitiatorPorts provides a mock function with given fields: ctx func (_m *FcConnector) GetInitiatorPorts(ctx context.Context) ([]string, error) { ret := _m.Called(ctx) + if len(ret) == 0 { + panic("no return value specified for GetInitiatorPorts") + } + var r0 []string + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) ([]string, error)); ok { + return rf(ctx) + } if rf, ok := ret.Get(0).(func(context.Context) []string); ok { r0 = rf(ctx) } else { @@ -78,7 +99,6 @@ func (_m *FcConnector) GetInitiatorPorts(ctx context.Context) ([]string, error) } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context) error); ok { r1 = rf(ctx) } else { @@ -87,3 +107,17 @@ func (_m *FcConnector) GetInitiatorPorts(ctx context.Context) ([]string, error) return r0, r1 } + +// NewFcConnector creates a new instance of FcConnector. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewFcConnector(t interface { + mock.TestingT + Cleanup(func()) +}) *FcConnector { + mock := &FcConnector{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/mocks/FileInfo.go b/mocks/FileInfo.go index 30d5ce05..e0dc3c7c 100644 --- a/mocks/FileInfo.go +++ b/mocks/FileInfo.go @@ -1,19 +1,3 @@ -/* - * - * Copyright © 2021-2022 Dell Inc. or its subsidiaries. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - // Code generated by mockery. DO NOT EDIT. package mocks @@ -30,10 +14,14 @@ type FileInfo struct { mock.Mock } -// IsDir provides a mock function with given fields: +// IsDir provides a mock function with no fields func (_m *FileInfo) IsDir() bool { ret := _m.Called() + if len(ret) == 0 { + panic("no return value specified for IsDir") + } + var r0 bool if rf, ok := ret.Get(0).(func() bool); ok { r0 = rf() @@ -44,10 +32,14 @@ func (_m *FileInfo) IsDir() bool { return r0 } -// ModTime provides a mock function with given fields: +// ModTime provides a mock function with no fields func (_m *FileInfo) ModTime() time.Time { ret := _m.Called() + if len(ret) == 0 { + panic("no return value specified for ModTime") + } + var r0 time.Time if rf, ok := ret.Get(0).(func() time.Time); ok { r0 = rf() @@ -58,10 +50,14 @@ func (_m *FileInfo) ModTime() time.Time { return r0 } -// Mode provides a mock function with given fields: +// Mode provides a mock function with no fields func (_m *FileInfo) Mode() iofs.FileMode { ret := _m.Called() + if len(ret) == 0 { + panic("no return value specified for Mode") + } + var r0 iofs.FileMode if rf, ok := ret.Get(0).(func() iofs.FileMode); ok { r0 = rf() @@ -72,10 +68,14 @@ func (_m *FileInfo) Mode() iofs.FileMode { return r0 } -// Name provides a mock function with given fields: +// Name provides a mock function with no fields func (_m *FileInfo) Name() string { ret := _m.Called() + if len(ret) == 0 { + panic("no return value specified for Name") + } + var r0 string if rf, ok := ret.Get(0).(func() string); ok { r0 = rf() @@ -86,10 +86,14 @@ func (_m *FileInfo) Name() string { return r0 } -// Size provides a mock function with given fields: +// Size provides a mock function with no fields func (_m *FileInfo) Size() int64 { ret := _m.Called() + if len(ret) == 0 { + panic("no return value specified for Size") + } + var r0 int64 if rf, ok := ret.Get(0).(func() int64); ok { r0 = rf() @@ -100,10 +104,14 @@ func (_m *FileInfo) Size() int64 { return r0 } -// Sys provides a mock function with given fields: +// Sys provides a mock function with no fields func (_m *FileInfo) Sys() interface{} { ret := _m.Called() + if len(ret) == 0 { + panic("no return value specified for Sys") + } + var r0 interface{} if rf, ok := ret.Get(0).(func() interface{}); ok { r0 = rf() @@ -115,3 +123,17 @@ func (_m *FileInfo) Sys() interface{} { return r0 } + +// NewFileInfo creates a new instance of FileInfo. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewFileInfo(t interface { + mock.TestingT + Cleanup(func()) +}) *FileInfo { + mock := &FileInfo{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/mocks/FsInterface.go b/mocks/FsInterface.go index 84c1b3eb..438a43d0 100644 --- a/mocks/FsInterface.go +++ b/mocks/FsInterface.go @@ -21,7 +21,7 @@ package mocks import ( context "context" - commonfs "github.com/dell/csi-powerstore/v2/pkg/common/fs" + commonfs "github.com/dell/csi-powerstore/v2/pkg/identifiers/fs" fs "io/fs" diff --git a/mocks/GeneralSnapshot.go b/mocks/GeneralSnapshot.go deleted file mode 100644 index 4acd4150..00000000 --- a/mocks/GeneralSnapshot.go +++ /dev/null @@ -1,85 +0,0 @@ -/* - * - * Copyright © 2021-2023 Dell Inc. or its subsidiaries. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -// Code generated by mockery. DO NOT EDIT. - -package mocks - -import ( - controller "github.com/dell/csi-powerstore/v2/pkg/controller" - mock "github.com/stretchr/testify/mock" -) - -// GeneralSnapshot is an autogenerated mock type for the GeneralSnapshot type -type GeneralSnapshot struct { - mock.Mock -} - -// GetID provides a mock function with given fields: -func (_m *GeneralSnapshot) GetID() string { - ret := _m.Called() - - var r0 string - if rf, ok := ret.Get(0).(func() string); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(string) - } - - return r0 -} - -// GetSize provides a mock function with given fields: -func (_m *GeneralSnapshot) GetSize() int64 { - ret := _m.Called() - - var r0 int64 - if rf, ok := ret.Get(0).(func() int64); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(int64) - } - - return r0 -} - -// GetSourceID provides a mock function with given fields: -func (_m *GeneralSnapshot) GetSourceID() string { - ret := _m.Called() - - var r0 string - if rf, ok := ret.Get(0).(func() string); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(string) - } - - return r0 -} - -// GetType provides a mock function with given fields: -func (_m *GeneralSnapshot) GetType() controller.SnapshotType { - ret := _m.Called() - - var r0 controller.SnapshotType - if rf, ok := ret.Get(0).(func() controller.SnapshotType); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(controller.SnapshotType) - } - - return r0 -} diff --git a/mocks/ISCSIConnector.go b/mocks/ISCSIConnector.go index a9942b1e..32e7d6c2 100644 --- a/mocks/ISCSIConnector.go +++ b/mocks/ISCSIConnector.go @@ -1,19 +1,3 @@ -/* - * - * Copyright © 2021-2022 Dell Inc. or its subsidiaries. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - // Code generated by mockery. DO NOT EDIT. package mocks @@ -34,14 +18,21 @@ type ISCSIConnector struct { func (_m *ISCSIConnector) ConnectVolume(ctx context.Context, info gobrick.ISCSIVolumeInfo) (gobrick.Device, error) { ret := _m.Called(ctx, info) + if len(ret) == 0 { + panic("no return value specified for ConnectVolume") + } + var r0 gobrick.Device + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, gobrick.ISCSIVolumeInfo) (gobrick.Device, error)); ok { + return rf(ctx, info) + } if rf, ok := ret.Get(0).(func(context.Context, gobrick.ISCSIVolumeInfo) gobrick.Device); ok { r0 = rf(ctx, info) } else { r0 = ret.Get(0).(gobrick.Device) } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, gobrick.ISCSIVolumeInfo) error); ok { r1 = rf(ctx, info) } else { @@ -55,6 +46,10 @@ func (_m *ISCSIConnector) ConnectVolume(ctx context.Context, info gobrick.ISCSIV func (_m *ISCSIConnector) DisconnectVolumeByDeviceName(ctx context.Context, name string) error { ret := _m.Called(ctx, name) + if len(ret) == 0 { + panic("no return value specified for DisconnectVolumeByDeviceName") + } + var r0 error if rf, ok := ret.Get(0).(func(context.Context, string) error); ok { r0 = rf(ctx, name) @@ -69,7 +64,15 @@ func (_m *ISCSIConnector) DisconnectVolumeByDeviceName(ctx context.Context, name func (_m *ISCSIConnector) GetInitiatorName(ctx context.Context) ([]string, error) { ret := _m.Called(ctx) + if len(ret) == 0 { + panic("no return value specified for GetInitiatorName") + } + var r0 []string + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) ([]string, error)); ok { + return rf(ctx) + } if rf, ok := ret.Get(0).(func(context.Context) []string); ok { r0 = rf(ctx) } else { @@ -78,7 +81,6 @@ func (_m *ISCSIConnector) GetInitiatorName(ctx context.Context) ([]string, error } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context) error); ok { r1 = rf(ctx) } else { @@ -87,3 +89,17 @@ func (_m *ISCSIConnector) GetInitiatorName(ctx context.Context) ([]string, error return r0, r1 } + +// NewISCSIConnector creates a new instance of ISCSIConnector. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewISCSIConnector(t interface { + mock.TestingT + Cleanup(func()) +}) *ISCSIConnector { + mock := &ISCSIConnector{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/mocks/Interface.go b/mocks/Interface.go new file mode 100644 index 00000000..08a87626 --- /dev/null +++ b/mocks/Interface.go @@ -0,0 +1,342 @@ +// Code generated by mockery. DO NOT EDIT. + +package mocks + +import ( + context "context" + + array "github.com/dell/csi-powerstore/v2/pkg/array" + + csi "github.com/container-storage-interface/spec/lib/go/csi" + + fs "github.com/dell/csi-powerstore/v2/pkg/identifiers/fs" + + mock "github.com/stretchr/testify/mock" +) + +// Interface is an autogenerated mock type for the Interface type +type Interface struct { + mock.Mock +} + +// Arrays provides a mock function with no fields +func (_m *Interface) Arrays() map[string]*array.PowerStoreArray { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for Arrays") + } + + var r0 map[string]*array.PowerStoreArray + if rf, ok := ret.Get(0).(func() map[string]*array.PowerStoreArray); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(map[string]*array.PowerStoreArray) + } + } + + return r0 +} + +// DefaultArray provides a mock function with no fields +func (_m *Interface) DefaultArray() *array.PowerStoreArray { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for DefaultArray") + } + + var r0 *array.PowerStoreArray + if rf, ok := ret.Get(0).(func() *array.PowerStoreArray); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*array.PowerStoreArray) + } + } + + return r0 +} + +// NodeExpandVolume provides a mock function with given fields: _a0, _a1 +func (_m *Interface) NodeExpandVolume(_a0 context.Context, _a1 *csi.NodeExpandVolumeRequest) (*csi.NodeExpandVolumeResponse, error) { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for NodeExpandVolume") + } + + var r0 *csi.NodeExpandVolumeResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *csi.NodeExpandVolumeRequest) (*csi.NodeExpandVolumeResponse, error)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(context.Context, *csi.NodeExpandVolumeRequest) *csi.NodeExpandVolumeResponse); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*csi.NodeExpandVolumeResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *csi.NodeExpandVolumeRequest) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// NodeGetCapabilities provides a mock function with given fields: _a0, _a1 +func (_m *Interface) NodeGetCapabilities(_a0 context.Context, _a1 *csi.NodeGetCapabilitiesRequest) (*csi.NodeGetCapabilitiesResponse, error) { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for NodeGetCapabilities") + } + + var r0 *csi.NodeGetCapabilitiesResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *csi.NodeGetCapabilitiesRequest) (*csi.NodeGetCapabilitiesResponse, error)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(context.Context, *csi.NodeGetCapabilitiesRequest) *csi.NodeGetCapabilitiesResponse); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*csi.NodeGetCapabilitiesResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *csi.NodeGetCapabilitiesRequest) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// NodeGetInfo provides a mock function with given fields: _a0, _a1 +func (_m *Interface) NodeGetInfo(_a0 context.Context, _a1 *csi.NodeGetInfoRequest) (*csi.NodeGetInfoResponse, error) { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for NodeGetInfo") + } + + var r0 *csi.NodeGetInfoResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *csi.NodeGetInfoRequest) (*csi.NodeGetInfoResponse, error)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(context.Context, *csi.NodeGetInfoRequest) *csi.NodeGetInfoResponse); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*csi.NodeGetInfoResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *csi.NodeGetInfoRequest) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// NodeGetVolumeStats provides a mock function with given fields: _a0, _a1 +func (_m *Interface) NodeGetVolumeStats(_a0 context.Context, _a1 *csi.NodeGetVolumeStatsRequest) (*csi.NodeGetVolumeStatsResponse, error) { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for NodeGetVolumeStats") + } + + var r0 *csi.NodeGetVolumeStatsResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *csi.NodeGetVolumeStatsRequest) (*csi.NodeGetVolumeStatsResponse, error)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(context.Context, *csi.NodeGetVolumeStatsRequest) *csi.NodeGetVolumeStatsResponse); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*csi.NodeGetVolumeStatsResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *csi.NodeGetVolumeStatsRequest) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// NodePublishVolume provides a mock function with given fields: _a0, _a1 +func (_m *Interface) NodePublishVolume(_a0 context.Context, _a1 *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for NodePublishVolume") + } + + var r0 *csi.NodePublishVolumeResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(context.Context, *csi.NodePublishVolumeRequest) *csi.NodePublishVolumeResponse); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*csi.NodePublishVolumeResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *csi.NodePublishVolumeRequest) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// NodeStageVolume provides a mock function with given fields: _a0, _a1 +func (_m *Interface) NodeStageVolume(_a0 context.Context, _a1 *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for NodeStageVolume") + } + + var r0 *csi.NodeStageVolumeResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(context.Context, *csi.NodeStageVolumeRequest) *csi.NodeStageVolumeResponse); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*csi.NodeStageVolumeResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *csi.NodeStageVolumeRequest) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// NodeUnpublishVolume provides a mock function with given fields: _a0, _a1 +func (_m *Interface) NodeUnpublishVolume(_a0 context.Context, _a1 *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for NodeUnpublishVolume") + } + + var r0 *csi.NodeUnpublishVolumeResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(context.Context, *csi.NodeUnpublishVolumeRequest) *csi.NodeUnpublishVolumeResponse); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*csi.NodeUnpublishVolumeResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *csi.NodeUnpublishVolumeRequest) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// NodeUnstageVolume provides a mock function with given fields: _a0, _a1 +func (_m *Interface) NodeUnstageVolume(_a0 context.Context, _a1 *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for NodeUnstageVolume") + } + + var r0 *csi.NodeUnstageVolumeResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(context.Context, *csi.NodeUnstageVolumeRequest) *csi.NodeUnstageVolumeResponse); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*csi.NodeUnstageVolumeResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *csi.NodeUnstageVolumeRequest) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// SetArrays provides a mock function with given fields: _a0 +func (_m *Interface) SetArrays(_a0 map[string]*array.PowerStoreArray) { + _m.Called(_a0) +} + +// SetDefaultArray provides a mock function with given fields: _a0 +func (_m *Interface) SetDefaultArray(_a0 *array.PowerStoreArray) { + _m.Called(_a0) +} + +// UpdateArrays provides a mock function with given fields: _a0, _a1 +func (_m *Interface) UpdateArrays(_a0 string, _a1 fs.Interface) error { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for UpdateArrays") + } + + var r0 error + if rf, ok := ret.Get(0).(func(string, fs.Interface) error); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// NewInterface creates a new instance of Interface. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewInterface(t interface { + mock.TestingT + Cleanup(func()) +}) *Interface { + mock := &Interface{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/mocks/NASCooldownTracker.go b/mocks/NASCooldownTracker.go new file mode 100644 index 00000000..cfb0fede --- /dev/null +++ b/mocks/NASCooldownTracker.go @@ -0,0 +1,70 @@ +// Code generated by mockery. DO NOT EDIT. + +package mocks + +import mock "github.com/stretchr/testify/mock" + +// NASCooldownTracker is an autogenerated mock type for the NASCooldownTracker type +type NASCooldownTracker struct { + mock.Mock +} + +// FallbackRetry provides a mock function with given fields: nasList +func (_m *NASCooldownTracker) FallbackRetry(nasList []string) string { + ret := _m.Called(nasList) + + if len(ret) == 0 { + panic("no return value specified for FallbackRetry") + } + + var r0 string + if rf, ok := ret.Get(0).(func([]string) string); ok { + r0 = rf(nasList) + } else { + r0 = ret.Get(0).(string) + } + + return r0 +} + +// IsInCooldown provides a mock function with given fields: nas +func (_m *NASCooldownTracker) IsInCooldown(nas string) bool { + ret := _m.Called(nas) + + if len(ret) == 0 { + panic("no return value specified for IsInCooldown") + } + + var r0 bool + if rf, ok := ret.Get(0).(func(string) bool); ok { + r0 = rf(nas) + } else { + r0 = ret.Get(0).(bool) + } + + return r0 +} + +// MarkFailure provides a mock function with given fields: nas +func (_m *NASCooldownTracker) MarkFailure(nas string) { + _m.Called(nas) +} + +// ResetFailure provides a mock function with given fields: nas +func (_m *NASCooldownTracker) ResetFailure(nas string) { + _m.Called(nas) +} + +// NewNASCooldownTracker creates a new instance of NASCooldownTracker. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewNASCooldownTracker(t interface { + mock.TestingT + Cleanup(func()) +}) *NASCooldownTracker { + mock := &NASCooldownTracker{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/mocks/NFSv4ACLsInterface.go b/mocks/NFSv4ACLsInterface.go index 90d0ea4e..ea9d5b11 100644 --- a/mocks/NFSv4ACLsInterface.go +++ b/mocks/NFSv4ACLsInterface.go @@ -1,20 +1,4 @@ -/* - * - * Copyright © 2022 Dell Inc. or its subsidiaries. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -// Code generated by mockery v1.0.0. DO NOT EDIT. +// Code generated by mockery. DO NOT EDIT. package mocks @@ -29,6 +13,10 @@ type NFSv4ACLsInterface struct { func (_m *NFSv4ACLsInterface) SetNfsv4Acls(acls string, dir string) error { ret := _m.Called(acls, dir) + if len(ret) == 0 { + panic("no return value specified for SetNfsv4Acls") + } + var r0 error if rf, ok := ret.Get(0).(func(string, string) error); ok { r0 = rf(acls, dir) @@ -38,3 +26,17 @@ func (_m *NFSv4ACLsInterface) SetNfsv4Acls(acls string, dir string) error { return r0 } + +// NewNFSv4ACLsInterface creates a new instance of NFSv4ACLsInterface. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewNFSv4ACLsInterface(t interface { + mock.TestingT + Cleanup(func()) +}) *NFSv4ACLsInterface { + mock := &NFSv4ACLsInterface{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/mocks/NVMEConnector.go b/mocks/NVMEConnector.go index 8ed00435..3fcfbf62 100644 --- a/mocks/NVMEConnector.go +++ b/mocks/NVMEConnector.go @@ -1,20 +1,4 @@ -/* - * - * Copyright © 2022 Dell Inc. or its subsidiaries. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -// Code generated by mockery v2.12.2. DO NOT EDIT. +// Code generated by mockery. DO NOT EDIT. package mocks @@ -23,8 +7,6 @@ import ( gobrick "github.com/dell/gobrick" mock "github.com/stretchr/testify/mock" - - testing "testing" ) // NVMEConnector is an autogenerated mock type for the NVMEConnector type @@ -36,14 +18,21 @@ type NVMEConnector struct { func (_m *NVMEConnector) ConnectVolume(ctx context.Context, info gobrick.NVMeVolumeInfo, useFC bool) (gobrick.Device, error) { ret := _m.Called(ctx, info, useFC) + if len(ret) == 0 { + panic("no return value specified for ConnectVolume") + } + var r0 gobrick.Device + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, gobrick.NVMeVolumeInfo, bool) (gobrick.Device, error)); ok { + return rf(ctx, info, useFC) + } if rf, ok := ret.Get(0).(func(context.Context, gobrick.NVMeVolumeInfo, bool) gobrick.Device); ok { r0 = rf(ctx, info, useFC) } else { r0 = ret.Get(0).(gobrick.Device) } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, gobrick.NVMeVolumeInfo, bool) error); ok { r1 = rf(ctx, info, useFC) } else { @@ -57,6 +46,10 @@ func (_m *NVMEConnector) ConnectVolume(ctx context.Context, info gobrick.NVMeVol func (_m *NVMEConnector) DisconnectVolumeByDeviceName(ctx context.Context, name string) error { ret := _m.Called(ctx, name) + if len(ret) == 0 { + panic("no return value specified for DisconnectVolumeByDeviceName") + } + var r0 error if rf, ok := ret.Get(0).(func(context.Context, string) error); ok { r0 = rf(ctx, name) @@ -71,7 +64,15 @@ func (_m *NVMEConnector) DisconnectVolumeByDeviceName(ctx context.Context, name func (_m *NVMEConnector) GetInitiatorName(ctx context.Context) ([]string, error) { ret := _m.Called(ctx) + if len(ret) == 0 { + panic("no return value specified for GetInitiatorName") + } + var r0 []string + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) ([]string, error)); ok { + return rf(ctx) + } if rf, ok := ret.Get(0).(func(context.Context) []string); ok { r0 = rf(ctx) } else { @@ -80,7 +81,6 @@ func (_m *NVMEConnector) GetInitiatorName(ctx context.Context) ([]string, error) } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context) error); ok { r1 = rf(ctx) } else { @@ -90,8 +90,12 @@ func (_m *NVMEConnector) GetInitiatorName(ctx context.Context) ([]string, error) return r0, r1 } -// NewNVMEConnector creates a new instance of NVMEConnector. It also registers the testing.TB interface on the mock and a cleanup function to assert the mocks expectations. -func NewNVMEConnector(t testing.TB) *NVMEConnector { +// NewNVMEConnector creates a new instance of NVMEConnector. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewNVMEConnector(t interface { + mock.TestingT + Cleanup(func()) +}) *NVMEConnector { mock := &NVMEConnector{} mock.Mock.Test(t) diff --git a/mocks/NodeInterface.go b/mocks/NodeInterface.go new file mode 100644 index 00000000..b6d7e6a3 --- /dev/null +++ b/mocks/NodeInterface.go @@ -0,0 +1,246 @@ +/* + * + * Copyright © 2025 Dell Inc. or its subsidiaries. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/dell/csi-powerstore/v2/pkg/node (interfaces: Interface) +// +// Generated by this command: +// +// mockgen -destination=../../mocks/NodeInterface.go -package=mocks github.com/dell/csi-powerstore/v2/pkg/node Interface +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + array "github.com/dell/csi-powerstore/v2/pkg/array" + fs "github.com/dell/csi-powerstore/v2/pkg/identifiers/fs" + csi "github.com/container-storage-interface/spec/lib/go/csi" + gomock "go.uber.org/mock/gomock" +) + +// MockInterface is a mock of Interface interface. +type MockInterface struct { + ctrl *gomock.Controller + recorder *MockInterfaceMockRecorder + isgomock struct{} +} + +// MockInterfaceMockRecorder is the mock recorder for MockInterface. +type MockInterfaceMockRecorder struct { + mock *MockInterface +} + +// NewMockInterface creates a new mock instance. +func NewMockInterface(ctrl *gomock.Controller) *MockInterface { + mock := &MockInterface{ctrl: ctrl} + mock.recorder = &MockInterfaceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockInterface) EXPECT() *MockInterfaceMockRecorder { + return m.recorder +} + +// Arrays mocks base method. +func (m *MockInterface) Arrays() map[string]*array.PowerStoreArray { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Arrays") + ret0, _ := ret[0].(map[string]*array.PowerStoreArray) + return ret0 +} + +// Arrays indicates an expected call of Arrays. +func (mr *MockInterfaceMockRecorder) Arrays() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Arrays", reflect.TypeOf((*MockInterface)(nil).Arrays)) +} + +// DefaultArray mocks base method. +func (m *MockInterface) DefaultArray() *array.PowerStoreArray { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DefaultArray") + ret0, _ := ret[0].(*array.PowerStoreArray) + return ret0 +} + +// DefaultArray indicates an expected call of DefaultArray. +func (mr *MockInterfaceMockRecorder) DefaultArray() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DefaultArray", reflect.TypeOf((*MockInterface)(nil).DefaultArray)) +} + +// NodeExpandVolume mocks base method. +func (m *MockInterface) NodeExpandVolume(arg0 context.Context, arg1 *csi.NodeExpandVolumeRequest) (*csi.NodeExpandVolumeResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NodeExpandVolume", arg0, arg1) + ret0, _ := ret[0].(*csi.NodeExpandVolumeResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// NodeExpandVolume indicates an expected call of NodeExpandVolume. +func (mr *MockInterfaceMockRecorder) NodeExpandVolume(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NodeExpandVolume", reflect.TypeOf((*MockInterface)(nil).NodeExpandVolume), arg0, arg1) +} + +// NodeGetCapabilities mocks base method. +func (m *MockInterface) NodeGetCapabilities(arg0 context.Context, arg1 *csi.NodeGetCapabilitiesRequest) (*csi.NodeGetCapabilitiesResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NodeGetCapabilities", arg0, arg1) + ret0, _ := ret[0].(*csi.NodeGetCapabilitiesResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// NodeGetCapabilities indicates an expected call of NodeGetCapabilities. +func (mr *MockInterfaceMockRecorder) NodeGetCapabilities(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NodeGetCapabilities", reflect.TypeOf((*MockInterface)(nil).NodeGetCapabilities), arg0, arg1) +} + +// NodeGetInfo mocks base method. +func (m *MockInterface) NodeGetInfo(arg0 context.Context, arg1 *csi.NodeGetInfoRequest) (*csi.NodeGetInfoResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NodeGetInfo", arg0, arg1) + ret0, _ := ret[0].(*csi.NodeGetInfoResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// NodeGetInfo indicates an expected call of NodeGetInfo. +func (mr *MockInterfaceMockRecorder) NodeGetInfo(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NodeGetInfo", reflect.TypeOf((*MockInterface)(nil).NodeGetInfo), arg0, arg1) +} + +// NodeGetVolumeStats mocks base method. +func (m *MockInterface) NodeGetVolumeStats(arg0 context.Context, arg1 *csi.NodeGetVolumeStatsRequest) (*csi.NodeGetVolumeStatsResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NodeGetVolumeStats", arg0, arg1) + ret0, _ := ret[0].(*csi.NodeGetVolumeStatsResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// NodeGetVolumeStats indicates an expected call of NodeGetVolumeStats. +func (mr *MockInterfaceMockRecorder) NodeGetVolumeStats(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NodeGetVolumeStats", reflect.TypeOf((*MockInterface)(nil).NodeGetVolumeStats), arg0, arg1) +} + +// NodePublishVolume mocks base method. +func (m *MockInterface) NodePublishVolume(arg0 context.Context, arg1 *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NodePublishVolume", arg0, arg1) + ret0, _ := ret[0].(*csi.NodePublishVolumeResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// NodePublishVolume indicates an expected call of NodePublishVolume. +func (mr *MockInterfaceMockRecorder) NodePublishVolume(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NodePublishVolume", reflect.TypeOf((*MockInterface)(nil).NodePublishVolume), arg0, arg1) +} + +// NodeStageVolume mocks base method. +func (m *MockInterface) NodeStageVolume(arg0 context.Context, arg1 *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NodeStageVolume", arg0, arg1) + ret0, _ := ret[0].(*csi.NodeStageVolumeResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// NodeStageVolume indicates an expected call of NodeStageVolume. +func (mr *MockInterfaceMockRecorder) NodeStageVolume(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NodeStageVolume", reflect.TypeOf((*MockInterface)(nil).NodeStageVolume), arg0, arg1) +} + +// NodeUnpublishVolume mocks base method. +func (m *MockInterface) NodeUnpublishVolume(arg0 context.Context, arg1 *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NodeUnpublishVolume", arg0, arg1) + ret0, _ := ret[0].(*csi.NodeUnpublishVolumeResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// NodeUnpublishVolume indicates an expected call of NodeUnpublishVolume. +func (mr *MockInterfaceMockRecorder) NodeUnpublishVolume(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NodeUnpublishVolume", reflect.TypeOf((*MockInterface)(nil).NodeUnpublishVolume), arg0, arg1) +} + +// NodeUnstageVolume mocks base method. +func (m *MockInterface) NodeUnstageVolume(arg0 context.Context, arg1 *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NodeUnstageVolume", arg0, arg1) + ret0, _ := ret[0].(*csi.NodeUnstageVolumeResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// NodeUnstageVolume indicates an expected call of NodeUnstageVolume. +func (mr *MockInterfaceMockRecorder) NodeUnstageVolume(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NodeUnstageVolume", reflect.TypeOf((*MockInterface)(nil).NodeUnstageVolume), arg0, arg1) +} + +// SetArrays mocks base method. +func (m *MockInterface) SetArrays(arg0 map[string]*array.PowerStoreArray) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "SetArrays", arg0) +} + +// SetArrays indicates an expected call of SetArrays. +func (mr *MockInterfaceMockRecorder) SetArrays(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetArrays", reflect.TypeOf((*MockInterface)(nil).SetArrays), arg0) +} + +// SetDefaultArray mocks base method. +func (m *MockInterface) SetDefaultArray(arg0 *array.PowerStoreArray) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "SetDefaultArray", arg0) +} + +// SetDefaultArray indicates an expected call of SetDefaultArray. +func (mr *MockInterfaceMockRecorder) SetDefaultArray(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetDefaultArray", reflect.TypeOf((*MockInterface)(nil).SetDefaultArray), arg0) +} + +// UpdateArrays mocks base method. +func (m *MockInterface) UpdateArrays(arg0 string, arg1 fs.Interface) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateArrays", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateArrays indicates an expected call of UpdateArrays. +func (mr *MockInterfaceMockRecorder) UpdateArrays(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateArrays", reflect.TypeOf((*MockInterface)(nil).UpdateArrays), arg0, arg1) +} diff --git a/mocks/NodeLabelsRetriever.go b/mocks/NodeLabelsRetriever.go deleted file mode 100644 index 4dca0b74..00000000 --- a/mocks/NodeLabelsRetriever.go +++ /dev/null @@ -1,152 +0,0 @@ -/* - Copyright (c) 2023 Dell Inc, or its subsidiaries. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Code generated by mockery. DO NOT EDIT. - -package mocks - -import ( - context "context" - - kubernetes "k8s.io/client-go/kubernetes" - - mock "github.com/stretchr/testify/mock" - - rest "k8s.io/client-go/rest" -) - -// NodeLabelsRetrieverInterface is an autogenerated mock type for the NodeLabelsRetrieverInterface type -type NodeLabelsRetrieverInterface struct { - mock.Mock -} - -// BuildConfigFromFlags provides a mock function with given fields: masterURL, kubeconfig -func (_m *NodeLabelsRetrieverInterface) BuildConfigFromFlags(masterURL string, kubeconfig string) (*rest.Config, error) { - ret := _m.Called(masterURL, kubeconfig) - - var r0 *rest.Config - var r1 error - if rf, ok := ret.Get(0).(func(string, string) (*rest.Config, error)); ok { - return rf(masterURL, kubeconfig) - } - if rf, ok := ret.Get(0).(func(string, string) *rest.Config); ok { - r0 = rf(masterURL, kubeconfig) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*rest.Config) - } - } - - if rf, ok := ret.Get(1).(func(string, string) error); ok { - r1 = rf(masterURL, kubeconfig) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetNodeLabels provides a mock function with given fields: ctx, k8sclientset, kubeNodeName -func (_m *NodeLabelsRetrieverInterface) GetNodeLabels(ctx context.Context, k8sclientset *kubernetes.Clientset, kubeNodeName string) (map[string]string, error) { - ret := _m.Called(k8sclientset, ctx, kubeNodeName) - - var r0 map[string]string - var r1 error - if rf, ok := ret.Get(0).(func(*kubernetes.Clientset, context.Context, string) (map[string]string, error)); ok { - return rf(k8sclientset, ctx, kubeNodeName) - } - if rf, ok := ret.Get(0).(func(*kubernetes.Clientset, context.Context, string) map[string]string); ok { - r0 = rf(k8sclientset, ctx, kubeNodeName) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(map[string]string) - } - } - - if rf, ok := ret.Get(1).(func(*kubernetes.Clientset, context.Context, string) error); ok { - r1 = rf(k8sclientset, ctx, kubeNodeName) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// InClusterConfig provides a mock function with given fields: -func (_m *NodeLabelsRetrieverInterface) InClusterConfig() (*rest.Config, error) { - ret := _m.Called() - - var r0 *rest.Config - var r1 error - if rf, ok := ret.Get(0).(func() (*rest.Config, error)); ok { - return rf() - } - if rf, ok := ret.Get(0).(func() *rest.Config); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*rest.Config) - } - } - - if rf, ok := ret.Get(1).(func() error); ok { - r1 = rf() - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// NewForConfig provides a mock function with given fields: config -func (_m *NodeLabelsRetrieverInterface) NewForConfig(config *rest.Config) (*kubernetes.Clientset, error) { - ret := _m.Called(config) - - var r0 *kubernetes.Clientset - var r1 error - if rf, ok := ret.Get(0).(func(*rest.Config) (*kubernetes.Clientset, error)); ok { - return rf(config) - } - if rf, ok := ret.Get(0).(func(*rest.Config) *kubernetes.Clientset); ok { - r0 = rf(config) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*kubernetes.Clientset) - } - } - - if rf, ok := ret.Get(1).(func(*rest.Config) error); ok { - r1 = rf(config) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// NewNodeLabelsRetrieverInterface creates a new instance of NodeLabelsRetrieverInterface. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewNodeLabelsRetrieverInterface(t interface { - mock.TestingT - Cleanup(func()) -}) *NodeLabelsRetrieverInterface { - mock := &NodeLabelsRetrieverInterface{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/mocks/NodeVolumePublisher.go b/mocks/NodeVolumePublisher.go index 57a6d4bf..3ac208b0 100644 --- a/mocks/NodeVolumePublisher.go +++ b/mocks/NodeVolumePublisher.go @@ -21,10 +21,9 @@ package mocks import ( context "context" + fs "github.com/dell/csi-powerstore/v2/pkg/identifiers/fs" + "github.com/dell/csmlog" csi "github.com/container-storage-interface/spec/lib/go/csi" - fs "github.com/dell/csi-powerstore/v2/pkg/common/fs" - - logrus "github.com/sirupsen/logrus" mock "github.com/stretchr/testify/mock" ) @@ -35,11 +34,11 @@ type NodeVolumePublisher struct { } // Publish provides a mock function with given fields: ctx, logFields, _a2, cap, isRO, targetPath, stagingPath -func (_m *NodeVolumePublisher) Publish(ctx context.Context, logFields logrus.Fields, _a2 fs.Interface, cap *csi.VolumeCapability, isRO bool, targetPath string, stagingPath string) (*csi.NodePublishVolumeResponse, error) { +func (_m *NodeVolumePublisher) Publish(ctx context.Context, logFields csmlog.Fields, _a2 fs.Interface, cap *csi.VolumeCapability, isRO bool, targetPath string, stagingPath string) (*csi.NodePublishVolumeResponse, error) { ret := _m.Called(ctx, logFields, _a2, cap, isRO, targetPath, stagingPath) var r0 *csi.NodePublishVolumeResponse - if rf, ok := ret.Get(0).(func(context.Context, logrus.Fields, fs.Interface, *csi.VolumeCapability, bool, string, string) *csi.NodePublishVolumeResponse); ok { + if rf, ok := ret.Get(0).(func(context.Context, csmlog.Fields, fs.Interface, *csi.VolumeCapability, bool, string, string) *csi.NodePublishVolumeResponse); ok { r0 = rf(ctx, logFields, _a2, cap, isRO, targetPath, stagingPath) } else { if ret.Get(0) != nil { @@ -48,7 +47,7 @@ func (_m *NodeVolumePublisher) Publish(ctx context.Context, logFields logrus.Fie } var r1 error - if rf, ok := ret.Get(1).(func(context.Context, logrus.Fields, fs.Interface, *csi.VolumeCapability, bool, string, string) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, csmlog.Fields, fs.Interface, *csi.VolumeCapability, bool, string, string) error); ok { r1 = rf(ctx, logFields, _a2, cap, isRO, targetPath, stagingPath) } else { r1 = ret.Error(1) diff --git a/mocks/NodeVolumeStager.go b/mocks/NodeVolumeStager.go index c3e8056e..c1304de6 100644 --- a/mocks/NodeVolumeStager.go +++ b/mocks/NodeVolumeStager.go @@ -21,10 +21,9 @@ package mocks import ( context "context" + fs "github.com/dell/csi-powerstore/v2/pkg/identifiers/fs" + "github.com/dell/csmlog" csi "github.com/container-storage-interface/spec/lib/go/csi" - fs "github.com/dell/csi-powerstore/v2/pkg/common/fs" - - logrus "github.com/sirupsen/logrus" mock "github.com/stretchr/testify/mock" ) @@ -35,11 +34,11 @@ type NodeVolumeStager struct { } // Stage provides a mock function with given fields: ctx, req, logFields, _a3, id -func (_m *NodeVolumeStager) Stage(ctx context.Context, req *csi.NodeStageVolumeRequest, logFields logrus.Fields, _a3 fs.Interface, id string) (*csi.NodeStageVolumeResponse, error) { +func (_m *NodeVolumeStager) Stage(ctx context.Context, req *csi.NodeStageVolumeRequest, logFields csmlog.Fields, _a3 fs.Interface, id string) (*csi.NodeStageVolumeResponse, error) { ret := _m.Called(ctx, req, logFields, _a3, id) var r0 *csi.NodeStageVolumeResponse - if rf, ok := ret.Get(0).(func(context.Context, *csi.NodeStageVolumeRequest, logrus.Fields, fs.Interface, string) *csi.NodeStageVolumeResponse); ok { + if rf, ok := ret.Get(0).(func(context.Context, *csi.NodeStageVolumeRequest, csmlog.Fields, fs.Interface, string) *csi.NodeStageVolumeResponse); ok { r0 = rf(ctx, req, logFields, _a3, id) } else { if ret.Get(0) != nil { @@ -48,7 +47,7 @@ func (_m *NodeVolumeStager) Stage(ctx context.Context, req *csi.NodeStageVolumeR } var r1 error - if rf, ok := ret.Get(1).(func(context.Context, *csi.NodeStageVolumeRequest, logrus.Fields, fs.Interface, string) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, *csi.NodeStageVolumeRequest, csmlog.Fields, fs.Interface, string) error); ok { r1 = rf(ctx, req, logFields, _a3, id) } else { r1 = ret.Error(1) diff --git a/mocks/UtilInterface.go b/mocks/UtilInterface.go index 337dc8e6..a732ac7b 100644 --- a/mocks/UtilInterface.go +++ b/mocks/UtilInterface.go @@ -1,20 +1,4 @@ -/* - * - * Copyright © 2021-2022 Dell Inc. or its subsidiaries. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -// Code generated by mockery v2.14.0. DO NOT EDIT. +// Code generated by mockery. DO NOT EDIT. package mocks @@ -44,6 +28,10 @@ func (_m *UtilInterface) BindMount(ctx context.Context, source string, target st _ca = append(_ca, _va...) ret := _m.Called(_ca...) + if len(ret) == 0 { + panic("no return value specified for BindMount") + } + var r0 error if rf, ok := ret.Get(0).(func(context.Context, string, string, ...string) error); ok { r0 = rf(ctx, source, target, options...) @@ -58,6 +46,10 @@ func (_m *UtilInterface) BindMount(ctx context.Context, source string, target st func (_m *UtilInterface) DeviceRescan(ctx context.Context, devicePath string) error { ret := _m.Called(ctx, devicePath) + if len(ret) == 0 { + panic("no return value specified for DeviceRescan") + } + var r0 error if rf, ok := ret.Get(0).(func(context.Context, string) error); ok { r0 = rf(ctx, devicePath) @@ -72,14 +64,21 @@ func (_m *UtilInterface) DeviceRescan(ctx context.Context, devicePath string) er func (_m *UtilInterface) FindFSType(ctx context.Context, mountpoint string) (string, error) { ret := _m.Called(ctx, mountpoint) + if len(ret) == 0 { + panic("no return value specified for FindFSType") + } + var r0 string + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string) (string, error)); ok { + return rf(ctx, mountpoint) + } if rf, ok := ret.Get(0).(func(context.Context, string) string); ok { r0 = rf(ctx, mountpoint) } else { r0 = ret.Get(0).(string) } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { r1 = rf(ctx, mountpoint) } else { @@ -100,6 +99,10 @@ func (_m *UtilInterface) Format(ctx context.Context, source string, target strin _ca = append(_ca, _va...) ret := _m.Called(_ca...) + if len(ret) == 0 { + panic("no return value specified for Format") + } + var r0 error if rf, ok := ret.Get(0).(func(context.Context, string, string, string, ...string) error); ok { r0 = rf(ctx, source, target, fsType, options...) @@ -121,6 +124,10 @@ func (_m *UtilInterface) FormatAndMount(ctx context.Context, source string, targ _ca = append(_ca, _va...) ret := _m.Called(_ca...) + if len(ret) == 0 { + panic("no return value specified for FormatAndMount") + } + var r0 error if rf, ok := ret.Get(0).(func(context.Context, string, string, string, ...string) error); ok { r0 = rf(ctx, source, target, fsType, options...) @@ -135,7 +142,15 @@ func (_m *UtilInterface) FormatAndMount(ctx context.Context, source string, targ func (_m *UtilInterface) GetDevMounts(ctx context.Context, dev string) ([]gofsutil.Info, error) { ret := _m.Called(ctx, dev) + if len(ret) == 0 { + panic("no return value specified for GetDevMounts") + } + var r0 []gofsutil.Info + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string) ([]gofsutil.Info, error)); ok { + return rf(ctx, dev) + } if rf, ok := ret.Get(0).(func(context.Context, string) []gofsutil.Info); ok { r0 = rf(ctx, dev) } else { @@ -144,7 +159,6 @@ func (_m *UtilInterface) GetDevMounts(ctx context.Context, dev string) ([]gofsut } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { r1 = rf(ctx, dev) } else { @@ -158,14 +172,21 @@ func (_m *UtilInterface) GetDevMounts(ctx context.Context, dev string) ([]gofsut func (_m *UtilInterface) GetDiskFormat(ctx context.Context, disk string) (string, error) { ret := _m.Called(ctx, disk) + if len(ret) == 0 { + panic("no return value specified for GetDiskFormat") + } + var r0 string + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string) (string, error)); ok { + return rf(ctx, disk) + } if rf, ok := ret.Get(0).(func(context.Context, string) string); ok { r0 = rf(ctx, disk) } else { r0 = ret.Get(0).(string) } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { r1 = rf(ctx, disk) } else { @@ -179,7 +200,15 @@ func (_m *UtilInterface) GetDiskFormat(ctx context.Context, disk string) (string func (_m *UtilInterface) GetFCHostPortWWNs(ctx context.Context) ([]string, error) { ret := _m.Called(ctx) + if len(ret) == 0 { + panic("no return value specified for GetFCHostPortWWNs") + } + var r0 []string + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) ([]string, error)); ok { + return rf(ctx) + } if rf, ok := ret.Get(0).(func(context.Context) []string); ok { r0 = rf(ctx) } else { @@ -188,7 +217,6 @@ func (_m *UtilInterface) GetFCHostPortWWNs(ctx context.Context) ([]string, error } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context) error); ok { r1 = rf(ctx) } else { @@ -202,7 +230,15 @@ func (_m *UtilInterface) GetFCHostPortWWNs(ctx context.Context) ([]string, error func (_m *UtilInterface) GetMountInfoFromDevice(ctx context.Context, devID string) (*gofsutil.DeviceMountInfo, error) { ret := _m.Called(ctx, devID) + if len(ret) == 0 { + panic("no return value specified for GetMountInfoFromDevice") + } + var r0 *gofsutil.DeviceMountInfo + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string) (*gofsutil.DeviceMountInfo, error)); ok { + return rf(ctx, devID) + } if rf, ok := ret.Get(0).(func(context.Context, string) *gofsutil.DeviceMountInfo); ok { r0 = rf(ctx, devID) } else { @@ -211,7 +247,6 @@ func (_m *UtilInterface) GetMountInfoFromDevice(ctx context.Context, devID strin } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { r1 = rf(ctx, devID) } else { @@ -225,7 +260,15 @@ func (_m *UtilInterface) GetMountInfoFromDevice(ctx context.Context, devID strin func (_m *UtilInterface) GetMounts(ctx context.Context) ([]gofsutil.Info, error) { ret := _m.Called(ctx) + if len(ret) == 0 { + panic("no return value specified for GetMounts") + } + var r0 []gofsutil.Info + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) ([]gofsutil.Info, error)); ok { + return rf(ctx) + } if rf, ok := ret.Get(0).(func(context.Context) []gofsutil.Info); ok { r0 = rf(ctx) } else { @@ -234,7 +277,6 @@ func (_m *UtilInterface) GetMounts(ctx context.Context) ([]gofsutil.Info, error) } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context) error); ok { r1 = rf(ctx) } else { @@ -248,14 +290,21 @@ func (_m *UtilInterface) GetMounts(ctx context.Context) ([]gofsutil.Info, error) func (_m *UtilInterface) GetMpathNameFromDevice(ctx context.Context, device string) (string, error) { ret := _m.Called(ctx, device) + if len(ret) == 0 { + panic("no return value specified for GetMpathNameFromDevice") + } + var r0 string + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string) (string, error)); ok { + return rf(ctx, device) + } if rf, ok := ret.Get(0).(func(context.Context, string) string); ok { r0 = rf(ctx, device) } else { r0 = ret.Get(0).(string) } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { r1 = rf(ctx, device) } else { @@ -265,11 +314,47 @@ func (_m *UtilInterface) GetMpathNameFromDevice(ctx context.Context, device stri return r0, r1 } +// GetNVMeController provides a mock function with given fields: device +func (_m *UtilInterface) GetNVMeController(device string) (string, error) { + ret := _m.Called(device) + + if len(ret) == 0 { + panic("no return value specified for GetNVMeController") + } + + var r0 string + var r1 error + if rf, ok := ret.Get(0).(func(string) (string, error)); ok { + return rf(device) + } + if rf, ok := ret.Get(0).(func(string) string); ok { + r0 = rf(device) + } else { + r0 = ret.Get(0).(string) + } + + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(device) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetSysBlockDevicesForVolumeWWN provides a mock function with given fields: ctx, volumeWWN func (_m *UtilInterface) GetSysBlockDevicesForVolumeWWN(ctx context.Context, volumeWWN string) ([]string, error) { ret := _m.Called(ctx, volumeWWN) + if len(ret) == 0 { + panic("no return value specified for GetSysBlockDevicesForVolumeWWN") + } + var r0 []string + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string) ([]string, error)); ok { + return rf(ctx, volumeWWN) + } if rf, ok := ret.Get(0).(func(context.Context, string) []string); ok { r0 = rf(ctx, volumeWWN) } else { @@ -278,7 +363,6 @@ func (_m *UtilInterface) GetSysBlockDevicesForVolumeWWN(ctx context.Context, vol } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { r1 = rf(ctx, volumeWWN) } else { @@ -292,6 +376,10 @@ func (_m *UtilInterface) GetSysBlockDevicesForVolumeWWN(ctx context.Context, vol func (_m *UtilInterface) IssueLIPToAllFCHosts(ctx context.Context) error { ret := _m.Called(ctx) + if len(ret) == 0 { + panic("no return value specified for IssueLIPToAllFCHosts") + } + var r0 error if rf, ok := ret.Get(0).(func(context.Context) error); ok { r0 = rf(ctx) @@ -313,6 +401,10 @@ func (_m *UtilInterface) Mount(ctx context.Context, source string, target string _ca = append(_ca, _va...) ret := _m.Called(_ca...) + if len(ret) == 0 { + panic("no return value specified for Mount") + } + var r0 error if rf, ok := ret.Get(0).(func(context.Context, string, string, string, ...string) error); ok { r0 = rf(ctx, source, target, fsType, options...) @@ -334,7 +426,15 @@ func (_m *UtilInterface) MultipathCommand(ctx context.Context, timeout time.Dura _ca = append(_ca, _va...) ret := _m.Called(_ca...) + if len(ret) == 0 { + panic("no return value specified for MultipathCommand") + } + var r0 []byte + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, time.Duration, string, ...string) ([]byte, error)); ok { + return rf(ctx, timeout, chroot, arguments...) + } if rf, ok := ret.Get(0).(func(context.Context, time.Duration, string, ...string) []byte); ok { r0 = rf(ctx, timeout, chroot, arguments...) } else { @@ -343,7 +443,6 @@ func (_m *UtilInterface) MultipathCommand(ctx context.Context, timeout time.Dura } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, time.Duration, string, ...string) error); ok { r1 = rf(ctx, timeout, chroot, arguments...) } else { @@ -357,6 +456,10 @@ func (_m *UtilInterface) MultipathCommand(ctx context.Context, timeout time.Dura func (_m *UtilInterface) RemoveBlockDevice(ctx context.Context, blockDevicePath string) error { ret := _m.Called(ctx, blockDevicePath) + if len(ret) == 0 { + panic("no return value specified for RemoveBlockDevice") + } + var r0 error if rf, ok := ret.Get(0).(func(context.Context, string) error); ok { r0 = rf(ctx, blockDevicePath) @@ -371,6 +474,10 @@ func (_m *UtilInterface) RemoveBlockDevice(ctx context.Context, blockDevicePath func (_m *UtilInterface) RescanSCSIHost(ctx context.Context, targets []string, lun string) error { ret := _m.Called(ctx, targets, lun) + if len(ret) == 0 { + panic("no return value specified for RescanSCSIHost") + } + var r0 error if rf, ok := ret.Get(0).(func(context.Context, []string, string) error); ok { r0 = rf(ctx, targets, lun) @@ -385,6 +492,10 @@ func (_m *UtilInterface) RescanSCSIHost(ctx context.Context, targets []string, l func (_m *UtilInterface) ResizeFS(ctx context.Context, volumePath string, devicePath string, ppathDevice string, mpathDevice string, fsType string) error { ret := _m.Called(ctx, volumePath, devicePath, ppathDevice, mpathDevice, fsType) + if len(ret) == 0 { + panic("no return value specified for ResizeFS") + } + var r0 error if rf, ok := ret.Get(0).(func(context.Context, string, string, string, string, string) error); ok { r0 = rf(ctx, volumePath, devicePath, ppathDevice, mpathDevice, fsType) @@ -399,6 +510,10 @@ func (_m *UtilInterface) ResizeFS(ctx context.Context, volumePath string, device func (_m *UtilInterface) ResizeMultipath(ctx context.Context, deviceName string) error { ret := _m.Called(ctx, deviceName) + if len(ret) == 0 { + panic("no return value specified for ResizeMultipath") + } + var r0 error if rf, ok := ret.Get(0).(func(context.Context, string) error); ok { r0 = rf(ctx, deviceName) @@ -413,7 +528,15 @@ func (_m *UtilInterface) ResizeMultipath(ctx context.Context, deviceName string) func (_m *UtilInterface) TargetIPLUNToDevicePath(ctx context.Context, targetIP string, lunID int) (map[string]string, error) { ret := _m.Called(ctx, targetIP, lunID) + if len(ret) == 0 { + panic("no return value specified for TargetIPLUNToDevicePath") + } + var r0 map[string]string + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, int) (map[string]string, error)); ok { + return rf(ctx, targetIP, lunID) + } if rf, ok := ret.Get(0).(func(context.Context, string, int) map[string]string); ok { r0 = rf(ctx, targetIP, lunID) } else { @@ -422,7 +545,6 @@ func (_m *UtilInterface) TargetIPLUNToDevicePath(ctx context.Context, targetIP s } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, string, int) error); ok { r1 = rf(ctx, targetIP, lunID) } else { @@ -436,6 +558,10 @@ func (_m *UtilInterface) TargetIPLUNToDevicePath(ctx context.Context, targetIP s func (_m *UtilInterface) Unmount(ctx context.Context, target string) error { ret := _m.Called(ctx, target) + if len(ret) == 0 { + panic("no return value specified for Unmount") + } + var r0 error if rf, ok := ret.Get(0).(func(context.Context, string) error); ok { r0 = rf(ctx, target) @@ -450,14 +576,21 @@ func (_m *UtilInterface) Unmount(ctx context.Context, target string) error { func (_m *UtilInterface) ValidateDevice(ctx context.Context, source string) (string, error) { ret := _m.Called(ctx, source) + if len(ret) == 0 { + panic("no return value specified for ValidateDevice") + } + var r0 string + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string) (string, error)); ok { + return rf(ctx, source) + } if rf, ok := ret.Get(0).(func(context.Context, string) string); ok { r0 = rf(ctx, source) } else { r0 = ret.Get(0).(string) } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { r1 = rf(ctx, source) } else { @@ -471,21 +604,28 @@ func (_m *UtilInterface) ValidateDevice(ctx context.Context, source string) (str func (_m *UtilInterface) WWNToDevicePath(ctx context.Context, wwn string) (string, string, error) { ret := _m.Called(ctx, wwn) + if len(ret) == 0 { + panic("no return value specified for WWNToDevicePath") + } + var r0 string + var r1 string + var r2 error + if rf, ok := ret.Get(0).(func(context.Context, string) (string, string, error)); ok { + return rf(ctx, wwn) + } if rf, ok := ret.Get(0).(func(context.Context, string) string); ok { r0 = rf(ctx, wwn) } else { r0 = ret.Get(0).(string) } - var r1 string if rf, ok := ret.Get(1).(func(context.Context, string) string); ok { r1 = rf(ctx, wwn) } else { r1 = ret.Get(1).(string) } - var r2 error if rf, ok := ret.Get(2).(func(context.Context, string) error); ok { r2 = rf(ctx, wwn) } else { @@ -495,13 +635,12 @@ func (_m *UtilInterface) WWNToDevicePath(ctx context.Context, wwn string) (strin return r0, r1, r2 } -type mockConstructorTestingTNewUtilInterface interface { +// NewUtilInterface creates a new instance of UtilInterface. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewUtilInterface(t interface { mock.TestingT Cleanup(func()) -} - -// NewUtilInterface creates a new instance of UtilInterface. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewUtilInterface(t mockConstructorTestingTNewUtilInterface) *UtilInterface { +}) *UtilInterface { mock := &UtilInterface{} mock.Mock.Test(t) diff --git a/mocks/VolumeCreator.go b/mocks/VolumeCreator.go index 75e50069..cbf5c650 100644 --- a/mocks/VolumeCreator.go +++ b/mocks/VolumeCreator.go @@ -1,19 +1,3 @@ -/* - * - * Copyright © 2021-2022 Dell Inc. or its subsidiaries. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - // Code generated by mockery. DO NOT EDIT. package mocks @@ -37,7 +21,15 @@ type VolumeCreator struct { func (_m *VolumeCreator) CheckIfAlreadyExists(ctx context.Context, name string, sizeInBytes int64, client gopowerstore.Client) (*csi.Volume, error) { ret := _m.Called(ctx, name, sizeInBytes, client) + if len(ret) == 0 { + panic("no return value specified for CheckIfAlreadyExists") + } + var r0 *csi.Volume + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, int64, gopowerstore.Client) (*csi.Volume, error)); ok { + return rf(ctx, name, sizeInBytes, client) + } if rf, ok := ret.Get(0).(func(context.Context, string, int64, gopowerstore.Client) *csi.Volume); ok { r0 = rf(ctx, name, sizeInBytes, client) } else { @@ -46,7 +38,6 @@ func (_m *VolumeCreator) CheckIfAlreadyExists(ctx context.Context, name string, } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, string, int64, gopowerstore.Client) error); ok { r1 = rf(ctx, name, sizeInBytes, client) } else { @@ -60,6 +51,10 @@ func (_m *VolumeCreator) CheckIfAlreadyExists(ctx context.Context, name string, func (_m *VolumeCreator) CheckName(ctx context.Context, name string) error { ret := _m.Called(ctx, name) + if len(ret) == 0 { + panic("no return value specified for CheckName") + } + var r0 error if rf, ok := ret.Get(0).(func(context.Context, string) error); ok { r0 = rf(ctx, name) @@ -70,20 +65,27 @@ func (_m *VolumeCreator) CheckName(ctx context.Context, name string) error { return r0 } -// CheckSize provides a mock function with given fields: ctx, cr -func (_m *VolumeCreator) CheckSize(ctx context.Context, cr *csi.CapacityRange) (int64, error) { - ret := _m.Called(ctx, cr) +// CheckSize provides a mock function with given fields: ctx, cr, isAutoRoundOffFsSizeEnabled +func (_m *VolumeCreator) CheckSize(ctx context.Context, cr *csi.CapacityRange, isAutoRoundOffFsSizeEnabled bool) (int64, error) { + ret := _m.Called(ctx, cr, isAutoRoundOffFsSizeEnabled) + + if len(ret) == 0 { + panic("no return value specified for CheckSize") + } var r0 int64 - if rf, ok := ret.Get(0).(func(context.Context, *csi.CapacityRange) int64); ok { - r0 = rf(ctx, cr) + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *csi.CapacityRange, bool) (int64, error)); ok { + return rf(ctx, cr, isAutoRoundOffFsSizeEnabled) + } + if rf, ok := ret.Get(0).(func(context.Context, *csi.CapacityRange, bool) int64); ok { + r0 = rf(ctx, cr, isAutoRoundOffFsSizeEnabled) } else { r0 = ret.Get(0).(int64) } - var r1 error - if rf, ok := ret.Get(1).(func(context.Context, *csi.CapacityRange) error); ok { - r1 = rf(ctx, cr) + if rf, ok := ret.Get(1).(func(context.Context, *csi.CapacityRange, bool) error); ok { + r1 = rf(ctx, cr, isAutoRoundOffFsSizeEnabled) } else { r1 = ret.Error(1) } @@ -95,7 +97,15 @@ func (_m *VolumeCreator) CheckSize(ctx context.Context, cr *csi.CapacityRange) ( func (_m *VolumeCreator) Clone(ctx context.Context, volumeSource *csi.VolumeContentSource_VolumeSource, volumeName string, sizeInBytes int64, parameters map[string]string, client gopowerstore.Client) (*csi.Volume, error) { ret := _m.Called(ctx, volumeSource, volumeName, sizeInBytes, parameters, client) + if len(ret) == 0 { + panic("no return value specified for Clone") + } + var r0 *csi.Volume + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *csi.VolumeContentSource_VolumeSource, string, int64, map[string]string, gopowerstore.Client) (*csi.Volume, error)); ok { + return rf(ctx, volumeSource, volumeName, sizeInBytes, parameters, client) + } if rf, ok := ret.Get(0).(func(context.Context, *csi.VolumeContentSource_VolumeSource, string, int64, map[string]string, gopowerstore.Client) *csi.Volume); ok { r0 = rf(ctx, volumeSource, volumeName, sizeInBytes, parameters, client) } else { @@ -104,7 +114,6 @@ func (_m *VolumeCreator) Clone(ctx context.Context, volumeSource *csi.VolumeCont } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, *csi.VolumeContentSource_VolumeSource, string, int64, map[string]string, gopowerstore.Client) error); ok { r1 = rf(ctx, volumeSource, volumeName, sizeInBytes, parameters, client) } else { @@ -118,14 +127,21 @@ func (_m *VolumeCreator) Clone(ctx context.Context, volumeSource *csi.VolumeCont func (_m *VolumeCreator) Create(ctx context.Context, req *csi.CreateVolumeRequest, sizeInBytes int64, client gopowerstore.Client) (gopowerstore.CreateResponse, error) { ret := _m.Called(ctx, req, sizeInBytes, client) + if len(ret) == 0 { + panic("no return value specified for Create") + } + var r0 gopowerstore.CreateResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *csi.CreateVolumeRequest, int64, gopowerstore.Client) (gopowerstore.CreateResponse, error)); ok { + return rf(ctx, req, sizeInBytes, client) + } if rf, ok := ret.Get(0).(func(context.Context, *csi.CreateVolumeRequest, int64, gopowerstore.Client) gopowerstore.CreateResponse); ok { r0 = rf(ctx, req, sizeInBytes, client) } else { r0 = ret.Get(0).(gopowerstore.CreateResponse) } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, *csi.CreateVolumeRequest, int64, gopowerstore.Client) error); ok { r1 = rf(ctx, req, sizeInBytes, client) } else { @@ -139,7 +155,15 @@ func (_m *VolumeCreator) Create(ctx context.Context, req *csi.CreateVolumeReques func (_m *VolumeCreator) CreateVolumeFromSnapshot(ctx context.Context, snapshotSource *csi.VolumeContentSource_SnapshotSource, volumeName string, sizeInBytes int64, parameters map[string]string, client gopowerstore.Client) (*csi.Volume, error) { ret := _m.Called(ctx, snapshotSource, volumeName, sizeInBytes, parameters, client) + if len(ret) == 0 { + panic("no return value specified for CreateVolumeFromSnapshot") + } + var r0 *csi.Volume + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *csi.VolumeContentSource_SnapshotSource, string, int64, map[string]string, gopowerstore.Client) (*csi.Volume, error)); ok { + return rf(ctx, snapshotSource, volumeName, sizeInBytes, parameters, client) + } if rf, ok := ret.Get(0).(func(context.Context, *csi.VolumeContentSource_SnapshotSource, string, int64, map[string]string, gopowerstore.Client) *csi.Volume); ok { r0 = rf(ctx, snapshotSource, volumeName, sizeInBytes, parameters, client) } else { @@ -148,7 +172,6 @@ func (_m *VolumeCreator) CreateVolumeFromSnapshot(ctx context.Context, snapshotS } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, *csi.VolumeContentSource_SnapshotSource, string, int64, map[string]string, gopowerstore.Client) error); ok { r1 = rf(ctx, snapshotSource, volumeName, sizeInBytes, parameters, client) } else { @@ -157,3 +180,17 @@ func (_m *VolumeCreator) CreateVolumeFromSnapshot(ctx context.Context, snapshotS return r0, r1 } + +// NewVolumeCreator creates a new instance of VolumeCreator. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewVolumeCreator(t interface { + mock.TestingT + Cleanup(func()) +}) *VolumeCreator { + mock := &VolumeCreator{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/mocks/VolumePublisher.go b/mocks/VolumePublisher.go index a7253c80..20f60f1e 100644 --- a/mocks/VolumePublisher.go +++ b/mocks/VolumePublisher.go @@ -1,19 +1,3 @@ -/* - * - * Copyright © 2021-2022 Dell Inc. or its subsidiaries. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - // Code generated by mockery. DO NOT EDIT. package mocks @@ -37,6 +21,10 @@ type VolumePublisher struct { func (_m *VolumePublisher) CheckIfVolumeExists(ctx context.Context, client gopowerstore.Client, volID string) error { ret := _m.Called(ctx, client, volID) + if len(ret) == 0 { + panic("no return value specified for CheckIfVolumeExists") + } + var r0 error if rf, ok := ret.Get(0).(func(context.Context, gopowerstore.Client, string) error); ok { r0 = rf(ctx, client, volID) @@ -47,25 +35,46 @@ func (_m *VolumePublisher) CheckIfVolumeExists(ctx context.Context, client gopow return r0 } -// Publish provides a mock function with given fields: ctx, req, client, kubeNodeID, volumeID -func (_m *VolumePublisher) Publish(ctx context.Context, req *csi.ControllerPublishVolumeRequest, client gopowerstore.Client, kubeNodeID string, volumeID string) (*csi.ControllerPublishVolumeResponse, error) { - ret := _m.Called(ctx, req, client, kubeNodeID, volumeID) +// Publish provides a mock function with given fields: ctx, publishContext, req, client, kubeNodeID, volumeID, isRemote +func (_m *VolumePublisher) Publish(ctx context.Context, publishContext map[string]string, req *csi.ControllerPublishVolumeRequest, client gopowerstore.Client, kubeNodeID string, volumeID string, isRemote bool) (*csi.ControllerPublishVolumeResponse, error) { + ret := _m.Called(ctx, publishContext, req, client, kubeNodeID, volumeID, isRemote) + + if len(ret) == 0 { + panic("no return value specified for Publish") + } var r0 *csi.ControllerPublishVolumeResponse - if rf, ok := ret.Get(0).(func(context.Context, *csi.ControllerPublishVolumeRequest, gopowerstore.Client, string, string) *csi.ControllerPublishVolumeResponse); ok { - r0 = rf(ctx, req, client, kubeNodeID, volumeID) + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, map[string]string, *csi.ControllerPublishVolumeRequest, gopowerstore.Client, string, string, bool) (*csi.ControllerPublishVolumeResponse, error)); ok { + return rf(ctx, publishContext, req, client, kubeNodeID, volumeID, isRemote) + } + if rf, ok := ret.Get(0).(func(context.Context, map[string]string, *csi.ControllerPublishVolumeRequest, gopowerstore.Client, string, string, bool) *csi.ControllerPublishVolumeResponse); ok { + r0 = rf(ctx, publishContext, req, client, kubeNodeID, volumeID, isRemote) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*csi.ControllerPublishVolumeResponse) } } - var r1 error - if rf, ok := ret.Get(1).(func(context.Context, *csi.ControllerPublishVolumeRequest, gopowerstore.Client, string, string) error); ok { - r1 = rf(ctx, req, client, kubeNodeID, volumeID) + if rf, ok := ret.Get(1).(func(context.Context, map[string]string, *csi.ControllerPublishVolumeRequest, gopowerstore.Client, string, string, bool) error); ok { + r1 = rf(ctx, publishContext, req, client, kubeNodeID, volumeID, isRemote) } else { r1 = ret.Error(1) } return r0, r1 } + +// NewVolumePublisher creates a new instance of VolumePublisher. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewVolumePublisher(t interface { + mock.TestingT + Cleanup(func()) +}) *VolumePublisher { + mock := &VolumePublisher{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/mocks/VolumeSnapshotter.go b/mocks/VolumeSnapshotter.go deleted file mode 100644 index 8711a3c4..00000000 --- a/mocks/VolumeSnapshotter.go +++ /dev/null @@ -1,77 +0,0 @@ -/* - * - * Copyright © 2021-2023 Dell Inc. or its subsidiaries. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -// Code generated by mockery. DO NOT EDIT. - -package mocks - -import ( - context "context" - - controller "github.com/dell/csi-powerstore/v2/pkg/controller" - gopowerstore "github.com/dell/gopowerstore" - - mock "github.com/stretchr/testify/mock" -) - -// VolumeSnapshotter is an autogenerated mock type for the VolumeSnapshotter type -type VolumeSnapshotter struct { - mock.Mock -} - -// Create provides a mock function with given fields: _a0, _a1, _a2, _a3 -func (_m *VolumeSnapshotter) Create(_a0 context.Context, _a1 string, _a2 string, _a3 gopowerstore.Client) (gopowerstore.CreateResponse, error) { - ret := _m.Called(_a0, _a1, _a2, _a3) - - var r0 gopowerstore.CreateResponse - if rf, ok := ret.Get(0).(func(context.Context, string, string, gopowerstore.Client) gopowerstore.CreateResponse); ok { - r0 = rf(_a0, _a1, _a2, _a3) - } else { - r0 = ret.Get(0).(gopowerstore.CreateResponse) - } - - var r1 error - if rf, ok := ret.Get(1).(func(context.Context, string, string, gopowerstore.Client) error); ok { - r1 = rf(_a0, _a1, _a2, _a3) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetExistingSnapshot provides a mock function with given fields: _a0, _a1, _a2 -func (_m *VolumeSnapshotter) GetExistingSnapshot(_a0 context.Context, _a1 string, _a2 gopowerstore.Client) (controller.GeneralSnapshot, error) { - ret := _m.Called(_a0, _a1, _a2) - - var r0 controller.GeneralSnapshot - if rf, ok := ret.Get(0).(func(context.Context, string, gopowerstore.Client) controller.GeneralSnapshot); ok { - r0 = rf(_a0, _a1, _a2) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(controller.GeneralSnapshot) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(context.Context, string, gopowerstore.Client) error); ok { - r1 = rf(_a0, _a1, _a2) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} diff --git a/mocks/VolumeStager.go b/mocks/VolumeStager.go new file mode 100644 index 00000000..7e0fc14e --- /dev/null +++ b/mocks/VolumeStager.go @@ -0,0 +1,62 @@ +// Code generated by mockery. DO NOT EDIT. + +package mocks + +import ( + context "context" + + fs "github.com/dell/csi-powerstore/v2/pkg/identifiers/fs" + "github.com/dell/csmlog" + csi "github.com/container-storage-interface/spec/lib/go/csi" + + mock "github.com/stretchr/testify/mock" +) + +// VolumeStager is an autogenerated mock type for the VolumeStager type +type VolumeStager struct { + mock.Mock +} + +// Stage provides a mock function with given fields: ctx, req, logFields, _a3, id, isRemote +func (_m *VolumeStager) Stage(ctx context.Context, req *csi.NodeStageVolumeRequest, logFields csmlog.Fields, _a3 fs.Interface, id string, isRemote bool) (*csi.NodeStageVolumeResponse, error) { + ret := _m.Called(ctx, req, logFields, _a3, id, isRemote) + + if len(ret) == 0 { + panic("no return value specified for Stage") + } + + var r0 *csi.NodeStageVolumeResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *csi.NodeStageVolumeRequest, csmlog.Fields, fs.Interface, string, bool) (*csi.NodeStageVolumeResponse, error)); ok { + return rf(ctx, req, logFields, _a3, id, isRemote) + } + if rf, ok := ret.Get(0).(func(context.Context, *csi.NodeStageVolumeRequest, csmlog.Fields, fs.Interface, string, bool) *csi.NodeStageVolumeResponse); ok { + r0 = rf(ctx, req, logFields, _a3, id, isRemote) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*csi.NodeStageVolumeResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *csi.NodeStageVolumeRequest, csmlog.Fields, fs.Interface, string, bool) error); ok { + r1 = rf(ctx, req, logFields, _a3, id, isRemote) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// NewVolumeStager creates a new instance of VolumeStager. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewVolumeStager(t interface { + mock.TestingT + Cleanup(func()) +}) *VolumeStager { + mock := &VolumeStager{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/overrides.mk b/overrides.mk new file mode 100644 index 00000000..517d964f --- /dev/null +++ b/overrides.mk @@ -0,0 +1,29 @@ +# Copyright © 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Dell Technologies, Dell and other trademarks are trademarks of Dell Inc. +# or its subsidiaries. Other trademarks may be trademarks of their respective +# owners. + +IMAGE_REGISTRY?="sample_registry" +IMAGE_NAME="csi-powerstore" +IMAGE_TAG?=$(shell date +%Y%m%d%H%M%S) + +# figure out if podman or docker should be used (use podman if found) +ifneq (, $(shell which podman 2>/dev/null)) +export BUILDER=podman +else +export BUILDER=docker +endif + +# target to print some help regarding these overrides and how to use them +overrides-help: + @echo + @echo "The following environment variables can be set to control the build" + @echo + @echo "IMAGE_REGISTRY - The registry to push images to." + @echo " Current setting is: $(IMAGE_REGISTRY)" + @echo "IMAGE_NAME - The image name to be built." + @echo " Current setting is: $(IMAGE_NAME)" + @echo "IMAGE_TAG - The image tag to be built, default is the current date." + @echo " Current setting is: $(IMAGE_TAG)" + @echo diff --git a/pkg/array/array.go b/pkg/array/array.go index d0624bb1..47581dec 100644 --- a/pkg/array/array.go +++ b/pkg/array/array.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2021-2023 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2021-2026 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,25 +26,34 @@ import ( "net/http" "path/filepath" "regexp" + "sort" "strconv" "strings" "sync" + "time" - "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/dell/csi-powerstore/v2/core" - "github.com/dell/csi-powerstore/v2/pkg/common" - "github.com/dell/csi-powerstore/v2/pkg/common/fs" + "github.com/dell/csi-powerstore/v2/pkg/identifiers" + "github.com/dell/csi-powerstore/v2/pkg/identifiers/fs" + "github.com/dell/csi-powerstore/v2/pkg/identifiers/k8sutils" + "github.com/dell/csm-dr/pkg/storage" + "github.com/dell/csmlog" csictx "github.com/dell/gocsi/context" "github.com/dell/gopowerstore" - log "github.com/sirupsen/logrus" + "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "gopkg.in/yaml.v3" + k8score "k8s.io/api/core/v1" + "k8s.io/component-helpers/scheduling/corev1/nodeaffinity" + "sigs.k8s.io/yaml" ) var ( // IPToArray - Store Array IPs - IPToArray map[string]string + IPToArray map[string]string + ipToArrayMux sync.Mutex + defaultMultiNasThreshold = 5 + defaultMultiNasCooldown = 5 * time.Minute + log = csmlog.GetLogger() ) // Consumer provides methods for safe management of arrays @@ -54,7 +63,6 @@ type Consumer interface { DefaultArray() *PowerStoreArray SetDefaultArray(*PowerStoreArray) UpdateArrays(string, fs.Interface) error - RegisterK8sCluster(fs.Interface) error } // Locker provides implementation for safe management of arrays @@ -65,6 +73,30 @@ type Locker struct { defaultArray *PowerStoreArray } +// Get returns a PowerStoreArray associated with the globalID +// if one exists in the Locker. +// +// Satisfies the ArrayLister interface for csm-dr +func (s *Locker) Get(globalID string) (storage.HealthChecker, error) { + log.Debugf("getting array for global ID %q", globalID) + return s.GetOneArray(globalID) +} + +// IsOnline returns true if the PowerStoreArray is online; false otherwise. +// Array status is determined by submitting a REST request for the array cluster +// using the gopowerstore API. +// +// Satisfies the HealthChecker interface for csm-dr. +func (psa *PowerStoreArray) IsOnline(ctx context.Context) bool { + log.Debugf("checking if array %q is online", psa.GlobalID) + _, err := psa.GetClient().GetCluster(ctx) + if err != nil { + log.Errorf("array %q is offline: %v", psa.GlobalID, err) + } + + return err == nil +} + // Arrays is a getter for list of arrays func (s *Locker) Arrays() map[string]*PowerStoreArray { s.arraysLock.Lock() @@ -104,6 +136,13 @@ func (s *Locker) SetDefaultArray(array *PowerStoreArray) { s.defaultArray = array } +// setIPToArray safely updates the IPToArray matcher. +func setIPToArray(matcher map[string]string) { + ipToArrayMux.Lock() + defer ipToArrayMux.Unlock() + IPToArray = matcher +} + // UpdateArrays updates array info func (s *Locker) UpdateArrays(configPath string, fs fs.Interface) error { log.Info("updating array info") @@ -112,27 +151,180 @@ func (s *Locker) UpdateArrays(configPath string, fs fs.Interface) error { return fmt.Errorf("can't get config for arrays: %s", err.Error()) } s.SetArrays(arrays) - IPToArray = matcher + setIPToArray(matcher) s.SetDefaultArray(defaultArray) return nil } +type NASCooldownTracker interface { + MarkFailure(nas string) + IsInCooldown(nas string) bool + ResetFailure(nas string) + FallbackRetry(nasList []string) string +} + +type NASStatus struct { + Failures int + CooldownUntil time.Time +} + +type NASCooldown struct { + statusMap map[string]*NASStatus + cooldownPeriod time.Duration + threshold int + mu sync.Mutex +} + +// NewNASCooldown returns a new instance of NASCooldown. +func NewNASCooldown(cooldownPeriod time.Duration, threshold int) *NASCooldown { + return &NASCooldown{ + statusMap: make(map[string]*NASStatus), + cooldownPeriod: cooldownPeriod, + threshold: threshold, + mu: sync.Mutex{}, + } +} + +// GetStatusMap is a getter for statusMap +func (n *NASCooldown) GetStatusMap() map[string]*NASStatus { + n.mu.Lock() + defer n.mu.Unlock() + // Return a copy of statusMap so that the original statusMap cannot be updated by the caller + statusMapCopy := make(map[string]*NASStatus) + for key, value := range n.statusMap { + statusMapCopy[key] = value + } + return statusMapCopy +} + +// GetCooldownPeriod is a getter for cooldownPeriod +func (n *NASCooldown) GetCooldownPeriod() time.Duration { + n.mu.Lock() + defer n.mu.Unlock() + return n.cooldownPeriod +} + +// GetThreshold is a getter for threshold +func (n *NASCooldown) GetThreshold() int { + n.mu.Lock() + defer n.mu.Unlock() + return n.threshold +} + +// Mark NAS as failed; only enter cooldown if threshold exceeded +func (n *NASCooldown) MarkFailure(nas string) { + n.mu.Lock() + defer n.mu.Unlock() + + status, exists := n.statusMap[nas] + if !exists { + status = &NASStatus{} + n.statusMap[nas] = status + } + + status.Failures++ + if status.Failures >= n.threshold { + status.CooldownUntil = time.Now().Add(n.cooldownPeriod) + } +} + +// Check if NAS is in cooldown +func (n *NASCooldown) IsInCooldown(nas string) bool { + n.mu.Lock() + defer n.mu.Unlock() + + if status, exists := n.statusMap[nas]; exists { + return time.Now().Before(status.CooldownUntil) + } + return false +} + +// Reset failure count on successful FS creation +func (n *NASCooldown) ResetFailure(nas string) { + n.mu.Lock() + defer n.mu.Unlock() + + delete(n.statusMap, nas) +} + +// Fallback logic - Retry all NAS servers, prioritizing least failed ones +func (n *NASCooldown) FallbackRetry(nasList []string) string { + n.mu.Lock() + defer n.mu.Unlock() + + sort.Slice(nasList, func(i, j int) bool { + if n.statusMap[nasList[i]] == nil { + return true + } else if n.statusMap[nasList[j]] == nil { + return false + } + return n.statusMap[nasList[i]].Failures < n.statusMap[nasList[j]].Failures + }) + + return nasList[0] // Pick NAS with least failures +} + // PowerStoreArray is a struct that stores all PowerStore connection information. // It stores gopowerstore client that can be directly used to invoke PowerStore API calls. // This structure is supposed to be parsed from config and mainly is created by GetPowerStoreArrays function. type PowerStoreArray struct { - Endpoint string `yaml:"endpoint"` - GlobalID string `yaml:"globalID"` - Username string `yaml:"username"` - Password string `yaml:"password"` - NasName string `yaml:"nasName"` - BlockProtocol common.TransportType `yaml:"blockProtocol"` - Insecure bool `yaml:"skipCertificateValidation"` - IsDefault bool `yaml:"isDefault"` - NfsAcls string `yaml:"nfsAcls"` + Endpoint string `yaml:"endpoint" json:"endpoint"` + GlobalID string `yaml:"globalID" json:"globalID"` + Username string `yaml:"username" json:"username"` + Password string `yaml:"password" json:"password"` + NasName string `yaml:"nasName" json:"nasName"` + BlockProtocol identifiers.TransportType `yaml:"blockProtocol" json:"blockProtocol"` + Insecure bool `yaml:"skipCertificateValidation" json:"skipCertificateValidation"` + IsDefault bool `yaml:"isDefault" json:"isDefault"` + NfsAcls string `yaml:"nfsAcls" json:"nfsAcls"` + + // MetroTopology describes the desired topology of the hosts configured for + // metro replication. Accepted values are "Uniform" and "Non-Uniform", with + // "Non-Uniform" not yet being implemented. + // + // Deprecated: MetroTopology is deprecated and remains only for purposes of + // backward compatibility. Use HostConnectivity instead. + MetroTopology string `yaml:"metroTopology" json:"metroTopology"` + // Labels is a set of labels, ANDed to build a node selector query. Used in + // conjunction with MetroTopology to identify nodes that should register + // metro hosts in this PowerStore system. + Labels map[string]string `yaml:"labels" json:"labels"` + + // HostConnectivity describes how nodes should register their hosts + // for this PowerStore system. + HostConnectivity *HostConnectivity `yaml:"hostConnectivity" json:"hostConnectivity"` + + Client gopowerstore.Client + IP string + NASCooldownTracker NASCooldownTracker +} + +// HostConnectivity provides two host connectivity options for nodes that will +// register hosts in a PowerStore system -- Local and Metro. +type HostConnectivity struct { + // Local contains a NodeSelector term used to identify nodes that are locally + // connected to the current PowerStore array and should register their hosts locally. + Local k8score.NodeSelector `yaml:"local" json:"local"` + // Metro contains more-specific options for how to register a host for + // use with uniform metro replication. For non-uniform metro replication, + // hosts should be registered as Local. + Metro MetroConnectivityOptions `yaml:"metro" json:"metro"` +} - Client gopowerstore.Client - IP string +// MetroConnectivityOptions provides options for how a host should be registered +// to optimize the connection for uniform metro replication. +type MetroConnectivityOptions struct { + // ColocatedLocal contains a NodeSelector term used to identify nodes that are + // colocated with the current PowerStore array and should register a host with the system. + ColocatedLocal k8score.NodeSelector `yaml:"colocatedLocal" json:"colocatedLocal"` + // ColocatedRemote contains a NodeSelector term used to identify nodes that are + // NOT colocated with the current PowerStore array, but are instead colocated with the + // metro replication target and should still register a host with this system as a secondary path. + ColocatedRemote k8score.NodeSelector `yaml:"colocatedRemote" json:"colocatedRemote"` + // ColocatedBoth contains a NodeSelector term used to identify nodes that are + // colocated with both the current PowerStore array and the metro replication target PowerStore + // array and should register a host with the current system. + ColocatedBoth k8score.NodeSelector `yaml:"colocatedBoth" json:"colocatedBoth"` } // GetNasName is a getter that returns name of configured NAS @@ -198,14 +390,18 @@ func GetPowerStoreArrays(fs fs.Interface, filePath string) (map[string]*PowerSto return nil, nil, nil, errors.New("no GlobalID field found in config.yaml - update config.yaml according to the documentation") } clientOptions := gopowerstore.NewClientOptions() + log.Debugf("PowerStore REST API timeout set to %s", identifiers.PowerstoreRESTApiTimeout) + clientOptions.SetDefaultTimeout(identifiers.PowerstoreRESTApiTimeout) clientOptions.SetInsecure(array.Insecure) - if throttlingRateLimit, ok := csictx.LookupEnv(context.Background(), common.EnvThrottlingRateLimit); ok { + if throttlingRateLimit, ok := csictx.LookupEnv(context.Background(), identifiers.EnvThrottlingRateLimit); ok { rateLimit, err := strconv.Atoi(throttlingRateLimit) if err != nil { log.Errorf("can't get throttling rate limit, using default") + } else if rateLimit < 0 { + log.Errorf("throttling rate limit is negative, using default") } else { - clientOptions.SetRateLimit(uint64(rateLimit)) + clientOptions.SetRateLimit(rateLimit) } } @@ -216,17 +412,18 @@ func GetPowerStoreArrays(fs fs.Interface, filePath string) (map[string]*PowerSto "unable to create PowerStore client: %s", err.Error()) } c.SetCustomHTTPHeaders(http.Header{ - "Application-Type": {fmt.Sprintf("%s/%s", common.VerboseName, core.SemVer)}}) + "Application-Type": {fmt.Sprintf("%s/%s", identifiers.VerboseName, identifiers.ManifestSemver)}, + }) - c.SetLogger(&common.CustomLogger{}) + c.SetLogger(&identifiers.CustomLogger{}) array.Client = c if array.BlockProtocol == "" { - array.BlockProtocol = common.AutoDetectTransport + array.BlockProtocol = identifiers.AutoDetectTransport } - array.BlockProtocol = common.TransportType(strings.ToUpper(string(array.BlockProtocol))) + array.BlockProtocol = identifiers.TransportType(strings.ToUpper(string(array.BlockProtocol))) var ip string - ips := common.GetIPListFromString(array.Endpoint) + ips := identifiers.GetIPListFromString(array.Endpoint) if ips == nil { log.Warnf("didn't found an IP from the provided endPoint, it could be a FQDN. Please make sure to enter a valid FQDN in https://abc.com/api/rest format") sub := strings.Split(array.Endpoint, "/") @@ -249,112 +446,328 @@ func GetPowerStoreArrays(fs fs.Interface, filePath string) (map[string]*PowerSto defaultArray = array foundDefault = true } + failureThreshold := defaultMultiNasThreshold + if threshold, ok := csictx.LookupEnv(context.Background(), identifiers.EnvMultiNASFailureThreshold); ok { + if thresholdInt, err := strconv.Atoi(threshold); err != nil { + log.Warnf("can't parse multi NAS failure threshold, using default %d", failureThreshold) + } else if thresholdInt <= 0 { + log.Warnf("multi NAS filure threshold is 0 or negative, using default %d", failureThreshold) + } else { + log.Debugf("use multi NAS failure threshold as %d", thresholdInt) + failureThreshold = thresholdInt + } + } + cooldownPeriod := defaultMultiNasCooldown + if cp, ok := csictx.LookupEnv(context.Background(), identifiers.EnvMultiNASCooldownPeriod); ok { + if duration, err := time.ParseDuration(cp); err != nil { + log.Warnf("can't parse multi NAS cooldown period, using default %v", cooldownPeriod) + } else if duration <= 0 { + log.Warnf("multi NAS cooldown period 0 or negative, using default %d", failureThreshold) + } else { + log.Debugf("use multi NAS cooldown period as %v", duration) + cooldownPeriod = duration + } + } + array.NASCooldownTracker = NewNASCooldown(cooldownPeriod, failureThreshold) } return arrayMap, mapper, defaultArray, nil } -// ParseVolumeID parses volume id in from CO (Kubernetes) and tries to understand what in it are PowerStore volume id, and what is ip, protocol. -// "/" is used as a delimiter. +// VolumeHandle represents the components of a unique csi-powerstore volume identifier and any remote +// volumes associated with the volume via data replication. +type VolumeHandle struct { + // The UUID of a volume provisioned by a PowerStore system that is locally managed by this driver. + LocalUUID string + // The Global ID of the PowerStore system that is locally managed by this driver. The Global ID + // can be found in the PowerStore UI under Settings > Properties + LocalArrayGlobalID string + // The UUID of a volume provisioned by a PowerStore system that is paired for replication with the + // PowerStore system managed by this driver. Currently only used for Metro replicated volume handles. + RemoteUUID string + // The Global ID of the PowerStore system that is paired for replication with the PowerStore system + // managed by this driver. Currently only used for Metro replicated volume handles. + // The Global ID can be found in the PowerStore UI under Settings > Properties + RemoteArrayGlobalID string + // One of "scsi" or "nfs" + Protocol string +} + +// ParseVolumeID parses a volume id from the CO (Kubernetes) and tries to extract local and remote PowerStore volume UUID, Global ID, and protocol. +// +// Example: +// +// ParseVolumeID("1cd254s/192.168.0.1/scsi") assuming 192.168.0.1 is the IP array PSabc0123def will return +// VolumeHandle{ +// LocalUUID: "1cd254s", +// LocalArrayGlobalID: "PSabc0123def", +// RemoteUUID: "", +// RemoteArrayGlobalID: "", +// Protocol: "scsi", +// }, nil // // Example: // -// ParseVolumeID("1cd254s/192.168.0.1/scsi") will return -// id = "1cd254s" -// ip = "192.168.0.1" -// protocol = "scsi" +// ParseVolumeID("9f840c56-96e6-4de9-b5a3-27e7c20eaa77/PSabcdef0123/scsi:9f840c56-96e6-4de9-b5a3-27e7c20eaa77/PS0123abcdef") returns +// VolumeHandle{ +// LocalUUID: "9f840c56-96e6-4de9-b5a3-27e7c20eaa77", +// LocalArrayGlobalID: "PSabcdef0123", +// RemoteUUID: "9f840c56-96e6-4de9-b5a3-27e7c20eaa77", +// RemoteArrayGlobalID: "PS0123abcdef", +// Protocol: "scsi", +// }, nil // // This function is backwards compatible and will try to understand volume protocol even if there is no such information in volume id. // It will do that by querying default powerstore array passed as one of the arguments -func ParseVolumeID(ctx context.Context, volumeID string, defaultArray *PowerStoreArray /*optional*/, cap *csi.VolumeCapability) (id string, arrayID string, protocol string, e error) { - if volumeID == "" { - return "", "", "", status.Errorf(codes.FailedPrecondition, - "incorrect volume id ") - } - volID := strings.Split(volumeID, "/") - id = volID[0] - log.Infof("volid %s", volID) - if len(volID) == 1 { - // We've got volume from previous version +func ParseVolumeID(ctx context.Context, volumeHandleRaw string, + defaultArray *PowerStoreArray, /*legacy support*/ + vc *csi.VolumeCapability, /*legacy support*/ +) (volumeHandle VolumeHandle, err error) { + log = log.WithContext(ctx) + log.Debugf("ParseVolumeID: parsing volume handle %s", volumeHandleRaw) + + if volumeHandleRaw == "" { + return volumeHandle, status.Errorf(codes.FailedPrecondition, + "unable to parse volume handle. volumeHandle is empty") + } + + // metro volume handles will have a colon separating the local + // volume handle and remote volume handle + // e.g. 9f840c56-96e6-4de9-b5a3-27e7c20eaa77/PSabcdef0123/scsi:9f840c56-96e6-4de9-b5a3-27e7c20eaa77/PS0123abcdef + volumeHandles := strings.Split(volumeHandleRaw, ":") + + // parse the first (potentially only) volume handle + localVolumeHandle := strings.Split(volumeHandles[0], "/") + volumeHandle.LocalUUID = localVolumeHandle[0] + log.Debugf("ParseVolumeID: local volume handle: %s", localVolumeHandle) + + if len(localVolumeHandle) == 1 { + // Legacy support where the volume name consists of only the volume ID. + + // We've got a volume from previous version // We assume that we should use default array for that - // Try to understand whether it is a nfs or scsi based volume + // Try to understand whether it is an nfs or scsi based volume + + volumeHandle.LocalArrayGlobalID = defaultArray.GetGlobalID() // If we have volume capability in request we can check FsType - if cap != nil && cap.GetMount() != nil { - if cap.GetMount().GetFsType() == "nfs" { - protocol = "nfs" + if vc != nil && vc.GetMount() != nil { + if vc.GetMount().GetFsType() == "nfs" { + volumeHandle.Protocol = "nfs" } else { - protocol = "scsi" + volumeHandle.Protocol = "scsi" } - arrayID = defaultArray.GetGlobalID() - return id, arrayID, protocol, nil - } - - // Try to just find out volume type by querying it's id from array - _, err := defaultArray.GetClient().GetVolume(ctx, id) - if err == nil { - protocol = "scsi" } else { - _, err := defaultArray.GetClient().GetFS(ctx, id) + // Try to just find out volume type by querying it's id from array + _, err := defaultArray.GetClient().GetVolume(ctx, volumeHandle.LocalUUID) if err == nil { - protocol = "nfs" + volumeHandle.Protocol = "scsi" } else { - if apiError, ok := err.(gopowerstore.APIError); ok && apiError.NotFound() { - return id, arrayID, protocol, apiError + _, err := defaultArray.GetClient().GetFS(ctx, volumeHandle.LocalUUID) + if err == nil { + volumeHandle.Protocol = "nfs" + } else { + if apiError, ok := err.(gopowerstore.APIError); ok && apiError.NotFound() { + return volumeHandle, apiError + } + return volumeHandle, status.Errorf(codes.Unknown, "failure checking volume status: %s", err.Error()) } - return id, arrayID, protocol, status.Errorf(codes.Unknown, - "failure checking volume status: %s", err.Error()) } } - arrayID = defaultArray.GetGlobalID() } else { - if ips := common.GetIPListFromString(volID[1]); ips != nil { - arrayID = IPToArray[ips[0]] + if ips := identifiers.GetIPListFromString(localVolumeHandle[1]); ips != nil { + // Legacy support where IP is used in the volume name in place of a PowerStore Global ID. + volumeHandle.LocalArrayGlobalID = IPToArray[ips[0]] } else { - arrayID = volID[1] + volumeHandle.LocalArrayGlobalID = localVolumeHandle[1] } - protocol = volID[2] + volumeHandle.Protocol = localVolumeHandle[2] + } + + // Parse the second portion of a metro volume handle + if len(volumeHandles) > 1 { + remoteVolumeHandle := strings.Split(volumeHandles[1], "/") + log.Debugf("ParseVolumeID: remote volume handle: %s", remoteVolumeHandle) + + volumeHandle.RemoteUUID = remoteVolumeHandle[0] + volumeHandle.RemoteArrayGlobalID = remoteVolumeHandle[1] + } + + log.Debugf( + "ParseVolumeID: volumeID: %s, arrayID: %s, protocol: %s, remoteVolumeID: %s, remoteArrayID: %s", + volumeHandle.LocalUUID, volumeHandle.LocalArrayGlobalID, volumeHandle.Protocol, volumeHandle.RemoteUUID, volumeHandle.RemoteArrayGlobalID, + ) + return volumeHandle, nil +} + +// IsMetro determines if the volume handle belongs to a metro volume by checking +// if the RemoteUUID field is non-empty. +func (v *VolumeHandle) IsMetro() bool { + return v.RemoteUUID != "" && v.RemoteArrayGlobalID != "" +} + +// GetVolumeUUIDPrefix extracts the prefix, if any exists, from a volume ID with a UUID format. +// The prefix is assumed to be all characters preceding the volume UUID including separators/delimiters, +// e.g. '-'. If no prefix is found, or the volume ID is not of the UUID format, the function returns an +// empty string. +func GetVolumeUUIDPrefix(volumeID string) (prefix string) { + matchUUID := regexp.MustCompile(`[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}`) + + // if the ID does not contain a UUID, return as-is. + if matchUUID.FindString(volumeID) == "" { + return "" } - log.Infof("id %s arrayID %s proto %s", id, arrayID, protocol) - return id, arrayID, protocol, nil + + // get the index of the UUID in the volumeID and use that + // to extract the prefix. + i := matchUUID.FindStringIndex(volumeID) + // create a slice from the beginning of the volume ID up to, + // but excluding, the UUID. This is the prefix. + prefix = volumeID[:i[0]] + + return prefix } -// RegisterK8sCluster registers the k8s cluster with PowerStore arrays -func (s *Locker) RegisterK8sCluster(fs fs.Interface) error { - k8sClusters, err := getK8sClusterInfo(fs) +// GetLeastUsedActiveNAS finds the active NAS with the least FS count +func GetLeastUsedActiveNAS(ctx context.Context, arr *PowerStoreArray, nasServers []string) (string, error) { + log := log.WithContext(ctx) + nasList, err := arr.Client.GetNASServers(ctx) if err != nil { - return err + log.Errorf("Failed to fetch NAS servers: %v", err) + return "", err } - for _, array := range s.arrays { - if !isK8sClusterAPISupported(array.Client) { - continue + nasMap := createNASMap(nasServers) + leastUsedNAS := findLeastUsedActiveNAS(arr, nasList, nasMap) + + if leastUsedNAS == nil { + nasInCooldown := GetNASInCooldown(arr, nasServers) + if len(nasInCooldown) != 0 { + log.Debugf("some NAS servers are in cooldown, moving to fallback retry") + return arr.NASCooldownTracker.FallbackRetry(nasInCooldown), nil } + log.Warnf("all NAS servers are inactive/unhealthy") + return "", fmt.Errorf("no suitable NAS server found, please ensure the NAS is running and healthy") + } - for _, cluster := range k8sClusters { - resp, err := array.Client.RegisterK8sCluster(context.Background(), &gopowerstore.K8sCluster{ - Name: cluster.Name, - IPAddress: cluster.IPAddress, - Port: cluster.Port, - Token: cluster.Token, - }) + return leastUsedNAS.Name, nil +} - if err != nil { - log.Errorf("cannot register k8s cluster: %s with %s:%d to array: %s err: %s \n", - cluster.Name, cluster.IPAddress, cluster.Port, array.Endpoint, err.Error()) - continue - } +func createNASMap(nasServers []string) map[string]bool { + nasMap := make(map[string]bool) + for _, nasServer := range nasServers { + nasMap[nasServer] = true + } + return nasMap +} - if resp.ID == "" { - log.Errorf("cannot register k8s cluster: %s with %s:%d to array: %s response Id is empty string \n", - cluster.Name, cluster.IPAddress, cluster.Port, array.Endpoint) - continue - } +func findLeastUsedActiveNAS(arr *PowerStoreArray, nasList []gopowerstore.NAS, nasMap map[string]bool) *gopowerstore.NAS { + var leastUsedNAS *gopowerstore.NAS + for i := range nasList { + nas := &nasList[i] + if !isEligibleNAS(arr, nas, nasMap) { + continue + } + if leastUsedNAS == nil || IsLessUsed(nas, leastUsedNAS) { + leastUsedNAS = nas + } + } + return leastUsedNAS +} - log.Infof("Registered k8s cluster: %s with %s:%d to array: %s id: %s\n", - cluster.Name, cluster.IPAddress, cluster.Port, array.Endpoint, resp.ID) +func isEligibleNAS(arr *PowerStoreArray, nas *gopowerstore.NAS, nasMap map[string]bool) bool { + if !nasMap[nas.Name] { + return false + } + if arr.NASCooldownTracker.IsInCooldown(nas.Name) { + return false + } + if nas.OperationalStatus != gopowerstore.Started { + return false + } + if !(nas.HealthDetails.State == gopowerstore.Info || nas.HealthDetails.State == gopowerstore.None) { + return false + } + return true +} + +func IsLessUsed(nas, current *gopowerstore.NAS) bool { + if len(nas.FileSystems) < len(current.FileSystems) { + return true + } + if len(nas.FileSystems) == len(current.FileSystems) && nas.Name < current.Name { + return true + } + return false +} + +// GetNASInCooldown returns a list of NAS servers that are in cooldown +func GetNASInCooldown(arr *PowerStoreArray, nasServers []string) []string { + nasInCooldown := make([]string, 0) + for _, nas := range nasServers { + if arr.NASCooldownTracker.IsInCooldown(nas) { + nasInCooldown = append(nasInCooldown, nas) } } + return nasInCooldown +} - return nil +// checkConnectivity checks if kubeNode matches metro selector. +func (psa *PowerStoreArray) CheckConnectivity(ctx context.Context, kubeNodeID string) bool { + var err error + + // Check for backward compatibility + if psa.HostConnectivity == nil { + log.Warnf("HostConnectivity is not defined in secret for array %s", psa.Endpoint) + return true + } + node, err := k8sutils.GetNodeByCSINodeID(ctx, identifiers.Name, kubeNodeID, identifiers.KeyNodeID) + if err != nil { + log.Errorf("GetNodeByCSINodeID error %s kubeNodeID %s", err, kubeNodeID) + return false + } + log.Debugf("checking if kubeNode %s matches metro selector. volume for metro array %+v", kubeNodeID, *psa.HostConnectivity) + if psa.DoesNodeMatchMetroSelectors(node) { + log.Debugf("selector matched.volume for uniform array %s, on kubeNodeID %s", psa.Endpoint, kubeNodeID) + return true + } + localSelector, localErr := nodeaffinity.NewNodeSelector(&psa.HostConnectivity.Local) + if localErr == nil && localSelector.Match(node) { // Check if node matches any of the local selectors in array(for non-unform array) + log.Debugf("Local selector matched for non-uniform array %s, on kubeNodeID %s", psa.Endpoint, kubeNodeID) + return true + } + + return false +} + +// DoesNodeMatchMetroSelectors checks if the given node matches any of the provided metro selectors in secret +func (psa *PowerStoreArray) DoesNodeMatchMetroSelectors(node *k8score.Node) bool { + local, errLocal := nodeaffinity.NewNodeSelector(&psa.HostConnectivity.Metro.ColocatedLocal) + if errLocal != nil { + log.Debugf("Error creating local node selector: %v", errLocal) + } + + if errLocal == nil && local != nil && local.Match(node) { + return true + } + + remote, errRemote := nodeaffinity.NewNodeSelector(&psa.HostConnectivity.Metro.ColocatedRemote) + if errRemote != nil { + log.Debugf("Error creating remote node selector: %v", errRemote) + } + + if errRemote == nil && remote != nil && remote.Match(node) { + return true + } + + colocate, errColocate := nodeaffinity.NewNodeSelector(&psa.HostConnectivity.Metro.ColocatedBoth) + if errColocate != nil { + log.Debugf("Error creating colocated node selector: %v", errColocate) + } + + if errColocate == nil && colocate != nil && colocate.Match(node) { + return true + } + log.Debug("Node does not match any metro selectors") + return false } diff --git a/pkg/array/array_test.go b/pkg/array/array_test.go index 695e9b1e..dd650668 100644 --- a/pkg/array/array_test.go +++ b/pkg/array/array_test.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2021-2023 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2021-2025 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,26 +21,64 @@ package array_test import ( "context" "errors" + "net/http" "os" "testing" + "time" - "github.com/container-storage-interface/spec/lib/go/csi" "github.com/dell/csi-powerstore/v2/mocks" "github.com/dell/csi-powerstore/v2/pkg/array" - "github.com/dell/csi-powerstore/v2/pkg/common" - "github.com/dell/csi-powerstore/v2/pkg/common/fs" + "github.com/dell/csi-powerstore/v2/pkg/identifiers" + "github.com/dell/csi-powerstore/v2/pkg/identifiers/fs" + "github.com/dell/csm-dr/pkg/storage" "github.com/dell/gofsutil" "github.com/dell/gopowerstore" + "github.com/dell/gopowerstore/api" gopowerstoremock "github.com/dell/gopowerstore/mocks" + "github.com/container-storage-interface/spec/lib/go/csi" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" + k8score "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +const ( + validPowerStoreIP = "127.0.0.1" + + validBlockVolumeUUID = "39bb1b5f-5624-490d-9ece-18f7b28a904e" + validRemoteBlockVolumeUUID = "9f840c56-96e6-4de9-b5a3-27e7c20eaa77" + + validFileSystemUUID = "66d815e3-52c2-126f-e29f-3e95021dcb9b" + + validGlobalID = "globalvolid1" + validRemoteGlobalID = "globalvolid2" + + scsi = "scsi" + nfs = "nfs" +) + +var ( + validBlockVolumeNameSCSI = buildVolumeName(validBlockVolumeUUID, validGlobalID, scsi) + validMetroBlockVolumeNameSCSI = buildMetroVolumeName(validBlockVolumeUUID, validGlobalID, scsi, validRemoteBlockVolumeUUID, validRemoteGlobalID) +) + +func buildVolumeName(uuid, globalID, transport string) string { + return uuid + "/" + globalID + "/" + transport +} + +func buildMetroVolumeName(uuid, globalID, transport, remoteUUID, remoteGlobalID string) string { + return buildVolumeName(uuid, globalID, transport) + ":" + remoteUUID + "/" + remoteGlobalID +} + func TestGetPowerStoreArrays(t *testing.T) { type args struct { fs fs.Interface data string } - _ = os.Setenv(common.EnvThrottlingRateLimit, "1000") + _ = os.Setenv(identifiers.EnvThrottlingRateLimit, "1000") + _ = os.Setenv(identifiers.EnvMultiNASFailureThreshold, "10") + _ = os.Setenv(identifiers.EnvMultiNASCooldownPeriod, "2m") tests := []struct { name string @@ -59,7 +97,7 @@ func TestGetPowerStoreArrays(t *testing.T) { Password: "password", Insecure: true, IsDefault: true, - BlockProtocol: common.ISCSITransport, + BlockProtocol: identifiers.ISCSITransport, }, "gid2": { Endpoint: "https://127.0.0.2/api/rest", @@ -67,7 +105,7 @@ func TestGetPowerStoreArrays(t *testing.T) { Password: "password", Insecure: true, IsDefault: false, - BlockProtocol: common.AutoDetectTransport, + BlockProtocol: identifiers.AutoDetectTransport, }, }, }, @@ -82,7 +120,7 @@ func TestGetPowerStoreArrays(t *testing.T) { Password: "password", Insecure: true, IsDefault: true, - BlockProtocol: common.AutoDetectTransport, + BlockProtocol: identifiers.AutoDetectTransport, }, }, }, @@ -109,6 +147,9 @@ func TestGetPowerStoreArrays(t *testing.T) { assert.Equal(t, v1.Insecure, got[k].Insecure) assert.Equal(t, v1.IsDefault, got[k].IsDefault) assert.Equal(t, v1.BlockProtocol, got[k].BlockProtocol) + got[k].NASCooldownTracker.(*array.NASCooldown).GetCooldownPeriod() + assert.Equal(t, 10, got[k].NASCooldownTracker.(*array.NASCooldown).GetThreshold()) + assert.Equal(t, 2*time.Minute, got[k].NASCooldownTracker.(*array.NASCooldown).GetCooldownPeriod()) } }) } @@ -155,324 +196,237 @@ func TestGetPowerStoreArrays(t *testing.T) { }) t.Run("incorrect throttling limit", func(t *testing.T) { - _ = os.Setenv(common.EnvThrottlingRateLimit, "abc") + _ = os.Setenv(identifiers.EnvThrottlingRateLimit, "abc") f := &fs.Fs{Util: &gofsutil.FS{}} _, _, _, err := array.GetPowerStoreArrays(f, "./testdata/one-arr.yaml") assert.NoError(t, err) }) -} -func TestParseVolumeID(t *testing.T) { - t.Run("incorrect volume id", func(t *testing.T) { - _, _, _, err := array.ParseVolumeID(context.Background(), "", nil, nil) - assert.Error(t, err) + t.Run("incorrect EnvMultiNASFailureThreshold & EnvMultiNASCooldownPeriod value", func(t *testing.T) { + _ = os.Setenv(identifiers.EnvMultiNASFailureThreshold, "0") + _ = os.Setenv(identifiers.EnvMultiNASCooldownPeriod, "0m") + f := &fs.Fs{Util: &gofsutil.FS{}} + got, _, _, err := array.GetPowerStoreArrays(f, "./testdata/one-arr.yaml") + assert.NoError(t, err) + assert.Equal(t, 5, got["gid1"].NASCooldownTracker.(*array.NASCooldown).GetThreshold()) + assert.Equal(t, 5*time.Minute, got["gid1"].NASCooldownTracker.(*array.NASCooldown).GetCooldownPeriod()) }) - t.Run("volume capability", func(t *testing.T) { - id := "1cd254s" - ip := "gid1" - getVolCap := func() *csi.VolumeCapability { - accessMode := new(csi.VolumeCapability_AccessMode) - accessMode.Mode = csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER - accessType := new(csi.VolumeCapability_Mount) - mountVolume := new(csi.VolumeCapability_MountVolume) - mountVolume.FsType = "nfs" - accessType.Mount = mountVolume - capability := new(csi.VolumeCapability) - capability.AccessMode = accessMode - capability.AccessType = accessType - return capability - } + t.Run("invalid format EnvMultiNASFailureThreshold & EnvMultiNASCooldownPeriod", func(t *testing.T) { + _ = os.Setenv(identifiers.EnvMultiNASFailureThreshold, "abc") + _ = os.Setenv(identifiers.EnvMultiNASCooldownPeriod, "abc") - volCap := getVolCap() - gotId, gotIp, protocol, err := array.ParseVolumeID(context.Background(), id, &array.PowerStoreArray{IP: ip, GlobalID: "gid1"}, volCap) + f := &fs.Fs{Util: &gofsutil.FS{}} + got, _, _, err := array.GetPowerStoreArrays(f, "./testdata/one-arr.yaml") assert.NoError(t, err) - assert.Equal(t, id, gotId) - assert.Equal(t, ip, gotIp) - assert.Equal(t, protocol, "nfs") + assert.Equal(t, 5, got["gid1"].NASCooldownTracker.(*array.NASCooldown).GetThreshold()) + assert.Equal(t, 5*time.Minute, got["gid1"].NASCooldownTracker.(*array.NASCooldown).GetCooldownPeriod()) }) } -func TestLocker_UpdateArrays(t *testing.T) { - lck := array.Locker{} - err := lck.UpdateArrays("./testdata/one-arr.yaml", &fs.Fs{Util: &gofsutil.FS{}}) - assert.NoError(t, err) - assert.Equal(t, lck.DefaultArray().Endpoint, "https://127.0.0.1/api/rest") +type LegacyParseVolumeTestSuite struct { + suite.Suite + + // Used to mock gopowerstore + mockAPI struct { + APIError gopowerstore.APIError + Client *gopowerstoremock.Client + + GetVolume *mock.Call + GetFS *mock.Call + } + + psArray *array.PowerStoreArray } -func TestLocker_RegisterK8sCluster_Success(t *testing.T) { - lck := array.Locker{} - lck.UpdateArrays("./testdata/one-arr.yaml", &fs.Fs{Util: &gofsutil.FS{}}) - - path := "/etc/kubernetes/kubelet.conf" - fsMock := new(mocks.FsInterface) - fsMock.On("ReadFile", path).Return([]byte(` -apiVersion: v1 -clusters: -- cluster: - certificate-authority-data: certficate-authority-data - server: https://127.0.0.1:6443 - name: kubernetes-cluster1 -contexts: -- context: - cluster: kubernetes-cluster1 - user: kubernetes-user - name: kubernetes-user@kubernetes -current-context: kubernetes-user@kubernetes -kind: Config -preferences: {} -users: -- name: kubernetes-user - user:`), nil) - - path = "/powerstore-visibility/token" - fsMock.On("ReadFile", path).Return([]byte("powerstore-visibility-token"), nil) - - gopowerstoreClientMock := new(gopowerstoremock.Client) - lck.Arrays()["gid1"].Client = gopowerstoreClientMock - gopowerstoreClientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(4.0), nil) - - gopowerstoreClientMock.On("RegisterK8sCluster", context.Background(), &gopowerstore.K8sCluster{ - Name: "kubernetes-cluster1", - IPAddress: "127.0.0.1", - Port: 6443, - Token: "powerstore-visibility-token", - }).Return(gopowerstore.CreateResponse{ID: "valid-cluster-id"}, nil) - - assert.Equal(t, lck.RegisterK8sCluster(fsMock), nil) +func TestLegacyParseVolumeSuite(t *testing.T) { + suite.Run(t, new(LegacyParseVolumeTestSuite)) } -func TestLocker_RegisterK8sCluster_3_0_Array(t *testing.T) { - lck := array.Locker{} - lck.UpdateArrays("./testdata/one-arr.yaml", &fs.Fs{Util: &gofsutil.FS{}}) - - path := "/etc/kubernetes/kubelet.conf" - fsMock := new(mocks.FsInterface) - fsMock.On("ReadFile", path).Return([]byte(` -apiVersion: v1 -clusters: -- cluster: - certificate-authority-data: certficate-authority-data - server: https://127.0.0.1:6443 - name: kubernetes-cluster1 -contexts: -- context: - cluster: kubernetes-cluster1 - user: kubernetes-user - name: kubernetes-user@kubernetes -current-context: kubernetes-user@kubernetes -kind: Config -preferences: {} -users: -- name: kubernetes-user - user:`), nil) - - path = "/powerstore-visibility/token" - fsMock.On("ReadFile", path).Return([]byte("powerstore-visibility-token"), nil) - - gopowerstoreClientMock := new(gopowerstoremock.Client) - lck.Arrays()["gid1"].Client = gopowerstoreClientMock - gopowerstoreClientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(3.0), nil) - - gopowerstoreClientMock.On("RegisterK8sCluster", context.Background(), &gopowerstore.K8sCluster{ - Name: "kubernetes-cluster1", - IPAddress: "127.0.0.1", - Port: 6443, - Token: "powerstore-visibility-token", - }).Return(gopowerstore.CreateResponse{ID: "valid-cluster-id"}, nil) - - assert.Equal(t, lck.RegisterK8sCluster(fsMock), nil) +func (s *LegacyParseVolumeTestSuite) SetupSuite() { + s.mockAPI.Client = new(gopowerstoremock.Client) + s.psArray = &array.PowerStoreArray{Client: s.mockAPI.Client, IP: validPowerStoreIP, GlobalID: validGlobalID} } -func TestLocker_RegisterK8sCluster_Success_2(t *testing.T) { - lck := array.Locker{} - lck.UpdateArrays("./testdata/one-arr.yaml", &fs.Fs{Util: &gofsutil.FS{}}) - - path := "/etc/kubernetes/kubelet.conf" - fsMock := new(mocks.FsInterface) - fsMock.On("ReadFile", path).Return([]byte("invalid-yaml"), gopowerstore.NewAPIError()) - - path = "/etc/kubernetes/admin.conf" - fsMock.On("ReadFile", path).Return([]byte(` -apiVersion: v1 -clusters: -- cluster: - certificate-authority-data: certficate-authority-data - server: https://127.0.0.1:6443 - name: kubernetes-cluster1 -contexts: -- context: - cluster: kubernetes-cluster1 - user: kubernetes-user - name: kubernetes-user@kubernetes -current-context: kubernetes-user@kubernetes -kind: Config -preferences: {} -users: -- name: kubernetes-user - user:`), nil) - - path = "/powerstore-visibility/token" - fsMock.On("ReadFile", path).Return([]byte("powerstore-visibility-token"), nil) - - gopowerstoreClientMock := new(gopowerstoremock.Client) - lck.Arrays()["gid1"].Client = gopowerstoreClientMock - gopowerstoreClientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(4.0), nil) - - gopowerstoreClientMock.On("RegisterK8sCluster", context.Background(), &gopowerstore.K8sCluster{ - Name: "kubernetes-cluster1", - IPAddress: "127.0.0.1", - Port: 6443, - Token: "powerstore-visibility-token", - }).Return(gopowerstore.CreateResponse{ID: "valid-cluster-id"}, nil) - - assert.Equal(t, lck.RegisterK8sCluster(fsMock), nil) +func (s *LegacyParseVolumeTestSuite) SetupTest() { + // A standard setup for mocking these API functions for these tests. + // Functions can be modified in the test implementation if needed. + s.mockAPI.GetVolume = s.mockAPI.Client.On("GetVolume", mock.Anything, mock.Anything) + s.mockAPI.GetFS = s.mockAPI.Client.On("GetFS", mock.Anything, mock.Anything) } -func TestLocker_RegisterK8sCluster_Failure(t *testing.T) { - lck := array.Locker{} - lck.UpdateArrays("./testdata/one-arr.yaml", &fs.Fs{Util: &gofsutil.FS{}}) - - path := "/etc/kubernetes/kubelet.conf" - fsMock := new(mocks.FsInterface) - fsMock.On("ReadFile", path).Return([]byte("invalid-yaml-content"), nil) - - gopowerstoreClientMock := new(gopowerstoremock.Client) - lck.Arrays()["gid1"].Client = gopowerstoreClientMock - gopowerstoreClientMock.On("RegisterK8sCluster", context.Background(), &gopowerstore.K8sCluster{ - Name: "kubernetes-cluster1", - IPAddress: "127.0.0.1", - Port: 6443, - Token: "powerstore-visibility-token", - }).Return(nil, nil) - - assert.NotEqual(t, lck.RegisterK8sCluster(fsMock), nil) +func (s *LegacyParseVolumeTestSuite) TearDownTest() { + // Unset any mocks that were configured during the test. + s.mockAPI.GetVolume.Unset() + s.mockAPI.GetFS.Unset() + + // Reset the API error for the next test run. + s.mockAPI.APIError = *gopowerstore.NewAPIError() } -func TestLocker_RegisterK8sCluster_Version_API_Failure(t *testing.T) { - lck := array.Locker{} - lck.UpdateArrays("./testdata/one-arr.yaml", &fs.Fs{Util: &gofsutil.FS{}}) - - path := "/etc/kubernetes/kubelet.conf" - fsMock := new(mocks.FsInterface) - fsMock.On("ReadFile", path).Return([]byte(` -apiVersion: v1 -clusters: -- cluster: - certificate-authority-data: certficate-authority-data - server: https://127.0.0.1:6443 - name: kubernetes-cluster1 -contexts: -- context: - cluster: kubernetes-cluster1 - user: kubernetes-user - name: kubernetes-user@kubernetes -current-context: kubernetes-user@kubernetes -kind: Config -preferences: {} -users: -- name: kubernetes-user - user:`), nil) - - path = "/powerstore-visibility/token" - fsMock.On("ReadFile", path).Return([]byte("powerstore-visibility-token"), nil) - - gopowerstoreClientMock := new(gopowerstoremock.Client) - lck.Arrays()["gid1"].Client = gopowerstoreClientMock - gopowerstoreClientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(3.0), gopowerstore.NewAPIError()) - - gopowerstoreClientMock.On("RegisterK8sCluster", context.Background(), &gopowerstore.K8sCluster{ - Name: "kubernetes-cluster1", - IPAddress: "127.0.0.1", - Port: 6443, - Token: "powerstore-visibility-token", - }).Return(nil, nil) - - assert.Equal(t, lck.RegisterK8sCluster(fsMock), nil) +func (s *LegacyParseVolumeTestSuite) TestVolumeCapabilityNFS() { + // When VolumeCapability (with mountVolume.FsType = "nfs") and default PowerStore array are passed to ParseVolumeID, + // use the capability to get the protocol and default array to get the PowerStore Global ID. + id := "1cd254s" + ip := "gid1" + getVolCap := func() *csi.VolumeCapability { + accessMode := new(csi.VolumeCapability_AccessMode) + accessMode.Mode = csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER + accessType := new(csi.VolumeCapability_Mount) + mountVolume := new(csi.VolumeCapability_MountVolume) + mountVolume.FsType = nfs + accessType.Mount = mountVolume + capability := new(csi.VolumeCapability) + capability.AccessMode = accessMode + capability.AccessType = accessType + return capability + } + + volCap := getVolCap() + gotID, err := array.ParseVolumeID(context.Background(), id, &array.PowerStoreArray{IP: ip, GlobalID: "gid1"}, volCap) + assert.NoError(s.T(), err) + assert.Equal(s.T(), id, gotID.LocalUUID) + assert.Equal(s.T(), ip, gotID.LocalArrayGlobalID) + assert.Equal(s.T(), "nfs", gotID.Protocol) } -func TestLocker_RegisterK8sCluster_Invalid_Yaml_Failure(t *testing.T) { - lck := array.Locker{} - lck.UpdateArrays("./testdata/one-arr.yaml", &fs.Fs{Util: &gofsutil.FS{}}) - - path := "/etc/kubernetes/kubelet.conf" - fsMock := new(mocks.FsInterface) - fsMock.On("ReadFile", path).Return([]byte(` -apiVersion: v1 -clusters: -- cluster: - certificate-authority-data: certficate-authority-data - server: https://127.0.0.1 - name: kubernetes-cluster1 -contexts: -- context: - cluster: kubernetes-cluster1 - user: kubernetes-user - name: kubernetes-user@kubernetes -current-context: kubernetes-user@kubernetes -kind: Config -preferences: {} -users: -- name: kubernetes-user - user:`), nil) - - path = "/powerstore-visibility/token" - fsMock.On("ReadFile", path).Return([]byte("powerstore-visibility-token"), nil) - - gopowerstoreClientMock := new(gopowerstoremock.Client) - lck.Arrays()["gid1"].Client = gopowerstoreClientMock - gopowerstoreClientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(3.0), gopowerstore.NewAPIError()) - - gopowerstoreClientMock.On("RegisterK8sCluster", context.Background(), &gopowerstore.K8sCluster{ - Name: "kubernetes-cluster1", - IPAddress: "127.0.0.1", - Port: 6443, - Token: "powerstore-visibility-token", - }).Return(nil, nil) - - assert.Equal(t, lck.RegisterK8sCluster(fsMock), nil) +func (s *LegacyParseVolumeTestSuite) TestVolumeCapabilitySCSI() { + // When VolumeCapability (with mountVolume.FsType = "scsi") and default PowerStore array are passed to ParseVolumeID, + // use the capability to get the protocol and default array to get the PowerStore Global ID. + id := validBlockVolumeUUID + ip := validPowerStoreIP + getVolCap := func() *csi.VolumeCapability { + accessMode := new(csi.VolumeCapability_AccessMode) + accessMode.Mode = csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER + accessType := new(csi.VolumeCapability_Mount) + mountVolume := new(csi.VolumeCapability_MountVolume) + mountVolume.FsType = scsi + accessType.Mount = mountVolume + capability := new(csi.VolumeCapability) + capability.AccessMode = accessMode + capability.AccessType = accessType + return capability + } + + volCap := getVolCap() + gotID, err := array.ParseVolumeID(context.Background(), id, &array.PowerStoreArray{IP: ip, GlobalID: validGlobalID}, volCap) + assert.NoError(s.T(), err) + assert.Equal(s.T(), id, gotID.LocalUUID) + assert.Equal(s.T(), validGlobalID, gotID.LocalArrayGlobalID) + assert.Equal(s.T(), scsi, gotID.Protocol) +} + +func (s *LegacyParseVolumeTestSuite) TestMissingSCSIProtocol() { + // When the protocol is not included in the volume name, + // if GetVolume returns without error, the protocol should be scsi. + s.mockAPI.GetVolume.Return(gopowerstore.Volume{ID: validBlockVolumeUUID}, nil) + + gotID, err := array.ParseVolumeID(context.Background(), validBlockVolumeUUID, s.psArray, nil) + assert.NoError(s.T(), err) + assert.Equal(s.T(), validBlockVolumeUUID, gotID.LocalUUID) + assert.Equal(s.T(), s.psArray.GlobalID, gotID.LocalArrayGlobalID) + assert.Equal(s.T(), scsi, gotID.Protocol) +} + +func (s *LegacyParseVolumeTestSuite) TestGetNFSProtocolFromAPIClient() { + // When the protocol is not included in the volume name, + // if GetVolume returns an error and GetFS returns without error, + // the protocol should be nfs. + s.mockAPI.GetVolume.Return(gopowerstore.Volume{}, errors.New("error")) + s.mockAPI.GetFS.Return(gopowerstore.FileSystem{ID: validFileSystemUUID}, nil) + + id, err := array.ParseVolumeID(context.Background(), validFileSystemUUID, s.psArray, nil) + assert.NoError(s.T(), err) + assert.Equal(s.T(), validFileSystemUUID, id.LocalUUID) + assert.Equal(s.T(), validGlobalID, id.LocalArrayGlobalID) + assert.Equal(s.T(), nfs, id.Protocol) +} + +func (s *LegacyParseVolumeTestSuite) TestVolumeNotFound() { + // When the protocol is not included in the volume name, + // and both GetVolume and GetFS return with errors, + // and the error returned by GetFS is http.StatusNotFound (404), + // then ParseVolumeID should return a related error. + s.mockAPI.GetVolume.Return(gopowerstore.Volume{}, errors.New("error")) + s.mockAPI.APIError = gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{ + StatusCode: http.StatusNotFound, + Message: "volume not found", + }, + } + + s.mockAPI.GetFS.Return(gopowerstore.FileSystem{}, error(s.mockAPI.APIError)) + + _, err := array.ParseVolumeID(context.Background(), validFileSystemUUID, s.psArray, nil) + assert.ErrorIs(s.T(), err, error(s.mockAPI.APIError)) +} + +func (s *LegacyParseVolumeTestSuite) TestVolumeUnknownError() { + // When the protocol is not included in the volume name, + // and both GetVolume and GetFS return with errors, + // and the error returned by GetFS is NOT http.StatusNotFound (404), + // then ParseVolumeID should return a related error. + s.mockAPI.GetVolume.Return(gopowerstore.Volume{}, errors.New("error")) + s.mockAPI.APIError = gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{ + StatusCode: http.StatusBadRequest, + Message: "bad request", + }, + } + + s.mockAPI.GetFS.Return(gopowerstore.FileSystem{}, error(s.mockAPI.APIError)) + + _, err := array.ParseVolumeID(context.Background(), validFileSystemUUID, s.psArray, nil) + assert.ErrorContains(s.T(), err, s.mockAPI.APIError.ErrorMsg.Message) +} + +func (s *LegacyParseVolumeTestSuite) TestIPAsArrayID() { + // When a volume name contains an IP as the second element delimited by a forward slash, + // ParseVolumeID should get the PowerStore Global ID from the IP. + + // Map the PowerStore IP to a valid Global ID + array.IPToArray = map[string]string{validPowerStoreIP: validGlobalID} + + // Build a volume ID using an IP in place of a PowerStore Global ID + volID := buildVolumeName(validBlockVolumeUUID, validPowerStoreIP, scsi) + + id, err := array.ParseVolumeID(context.Background(), volID, nil, nil) + assert.NoError(s.T(), err) + assert.Equal(s.T(), validBlockVolumeUUID, id.LocalUUID) + assert.Equal(s.T(), validGlobalID, id.LocalArrayGlobalID) + assert.Equal(s.T(), scsi, id.Protocol) +} + +func TestParseVolumeID(t *testing.T) { + t.Run("parse volume name", func(t *testing.T) { + id, err := array.ParseVolumeID(context.Background(), validBlockVolumeNameSCSI, nil, nil) + assert.NoError(t, err) + assert.Equal(t, validBlockVolumeUUID, id.LocalUUID) + assert.Equal(t, validGlobalID, id.LocalArrayGlobalID) + assert.Equal(t, scsi, id.Protocol) + }) + + t.Run("incorrect volume id", func(t *testing.T) { + _, err := array.ParseVolumeID(context.Background(), "", nil, nil) + assert.Error(t, err) + }) + + t.Run("parse metro volume name", func(t *testing.T) { + id, err := array.ParseVolumeID(context.Background(), validMetroBlockVolumeNameSCSI, nil, nil) + assert.NoError(t, err) + assert.Equal(t, validBlockVolumeUUID, id.LocalUUID) + assert.Equal(t, validRemoteBlockVolumeUUID, id.RemoteUUID) + assert.Equal(t, validGlobalID, id.LocalArrayGlobalID) + assert.Equal(t, validRemoteGlobalID, id.RemoteArrayGlobalID) + assert.Equal(t, scsi, id.Protocol) + }) } -func TestLocker_RegisterK8sCluster_Invalid_Port_Failure(t *testing.T) { +func TestLocker_UpdateArrays(t *testing.T) { lck := array.Locker{} - lck.UpdateArrays("./testdata/one-arr.yaml", &fs.Fs{Util: &gofsutil.FS{}}) - - path := "/etc/kubernetes/kubelet.conf" - fsMock := new(mocks.FsInterface) - fsMock.On("ReadFile", path).Return([]byte(` -apiVersion: v1 -clusters: -- cluster: - certificate-authority-data: certficate-authority-data - server: https://127.0.0.1:ab - name: kubernetes-cluster1 -contexts: -- context: - cluster: kubernetes-cluster1 - user: kubernetes-user - name: kubernetes-user@kubernetes -current-context: kubernetes-user@kubernetes -kind: Config -preferences: {} -users: -- name: kubernetes-user - user:`), nil) - - path = "/powerstore-visibility/token" - fsMock.On("ReadFile", path).Return([]byte("powerstore-visibility-token"), nil) - - gopowerstoreClientMock := new(gopowerstoremock.Client) - lck.Arrays()["gid1"].Client = gopowerstoreClientMock - gopowerstoreClientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(3.0), gopowerstore.NewAPIError()) - - gopowerstoreClientMock.On("RegisterK8sCluster", context.Background(), &gopowerstore.K8sCluster{ - Name: "kubernetes-cluster1", - IPAddress: "127.0.0.1", - Port: 6443, - Token: "powerstore-visibility-token", - }).Return(nil, nil) - - assert.Equal(t, lck.RegisterK8sCluster(fsMock), nil) + err := lck.UpdateArrays("./testdata/one-arr.yaml", &fs.Fs{Util: &gofsutil.FS{}}) + assert.NoError(t, err) + assert.Equal(t, lck.DefaultArray().Endpoint, "https://127.0.0.1/api/rest") } func TestLocker_GetOneArray(t *testing.T) { @@ -488,3 +442,743 @@ func TestLocker_GetOneArray(t *testing.T) { assert.Error(t, err) assert.NotEqual(t, fetched, array) } + +func TestGetLeastUsedActiveNAS(t *testing.T) { + ctx := context.Background() + clientMock := new(gopowerstoremock.Client) + + // Define NAS servers for different test cases + validNAS1 := gopowerstore.NAS{ + Name: "nasA", + OperationalStatus: gopowerstore.Started, + HealthDetails: gopowerstore.HealthDetails{State: gopowerstore.Info}, + FileSystems: make([]gopowerstore.FileSystem, 3), // 3 FS + } + + validNAS2 := gopowerstore.NAS{ + Name: "nasB", + OperationalStatus: gopowerstore.Started, + HealthDetails: gopowerstore.HealthDetails{State: gopowerstore.None}, + FileSystems: make([]gopowerstore.FileSystem, 2), // 2 FS (should be chosen) + } + + validNAS3 := gopowerstore.NAS{ + Name: "nasC", + OperationalStatus: gopowerstore.Started, + HealthDetails: gopowerstore.HealthDetails{State: gopowerstore.Info}, + FileSystems: make([]gopowerstore.FileSystem, 2), // 2 FS, but lexicographically larger + } + + validNAS4 := gopowerstore.NAS{ + Name: "nasD", + OperationalStatus: gopowerstore.Started, + HealthDetails: gopowerstore.HealthDetails{State: gopowerstore.Info}, + FileSystems: make([]gopowerstore.FileSystem, 1), + } + + invalidNAS1 := gopowerstore.NAS{ + Name: "nasX", + OperationalStatus: gopowerstore.Stopped, // Inactive NAS + HealthDetails: gopowerstore.HealthDetails{State: gopowerstore.Info}, + FileSystems: make([]gopowerstore.FileSystem, 1), + } + + invalidNAS2 := gopowerstore.NAS{ + Name: "nasY", + OperationalStatus: gopowerstore.Started, + HealthDetails: gopowerstore.HealthDetails{State: gopowerstore.Critical}, // Invalid state + FileSystems: make([]gopowerstore.FileSystem, 1), + } + + invalidNAS3 := gopowerstore.NAS{ + Name: "nasZ", + OperationalStatus: gopowerstore.Started, + HealthDetails: gopowerstore.HealthDetails{State: gopowerstore.Info}, + FileSystems: make([]gopowerstore.FileSystem, 1), + } + + tests := []struct { + name string + nasList []gopowerstore.NAS + expectedNAS *gopowerstore.NAS + markForFailure []string + nasServersInSc []string + + expectedErrMsg string + }{ + { + name: "Valid NAS selection (least FS count wins)", + nasList: []gopowerstore.NAS{validNAS1, validNAS2, validNAS3, validNAS4, invalidNAS1, invalidNAS2}, + expectedNAS: &validNAS4, // nasD has the least FS count (1) + nasServersInSc: []string{"nasA", "nasD"}, + }, + { + name: "NAS not in nasServers map", + nasList: []gopowerstore.NAS{invalidNAS3}, + expectedErrMsg: "no suitable NAS server found", + nasServersInSc: []string{"nasA", "nasD"}, + }, + { + name: "NAS not active", + nasList: []gopowerstore.NAS{invalidNAS1}, + expectedErrMsg: "no suitable NAS server found", + nasServersInSc: []string{"nasA", "nasD", "nasX"}, + }, + { + name: "NAS with invalid health state", + nasList: []gopowerstore.NAS{invalidNAS2}, + expectedErrMsg: "no suitable NAS server found", + nasServersInSc: []string{"nasA", "nasD", "nasY"}, + }, + { + name: "All NAS servers inactive or unhealthy", + nasList: []gopowerstore.NAS{invalidNAS1, invalidNAS2}, + expectedErrMsg: "no suitable NAS server found", + nasServersInSc: []string{"nasA", "nasB", "nasC", "nasD", "nasX", "nasY", "nasZ"}, + }, + { + name: "All NAS servers are in cooldown 1", + nasList: []gopowerstore.NAS{validNAS1, validNAS2, validNAS3, validNAS4, invalidNAS1, invalidNAS2}, + expectedNAS: &validNAS4, // nasD has the least Failure count (1) + markForFailure: []string{"nasA", "nasD"}, + nasServersInSc: []string{"nasD", "nasA"}, + }, + { + name: "All NAS servers are in cooldown 2", + nasList: []gopowerstore.NAS{validNAS1, validNAS2, validNAS3, validNAS4, invalidNAS1, invalidNAS2}, + expectedNAS: &validNAS1, // nasA has the least Failure count (1) + markForFailure: []string{"nasA", "nasD", "nasD"}, + nasServersInSc: []string{"nasD", "nasA"}, + }, + { + name: "Few NAS servers inactive or unhealthy and rest are in cooldown", + nasList: []gopowerstore.NAS{invalidNAS1, invalidNAS2, validNAS3, validNAS4}, + expectedNAS: &validNAS3, // nasC has the least Failure count (1) + markForFailure: []string{"nasC", "nasD", "nasD"}, + nasServersInSc: []string{"nasA", "nasB", "nasC", "nasD", "nasX", "nasY", "nasZ"}, + }, + { + name: "Empty NAS list", + nasList: []gopowerstore.NAS{}, + expectedErrMsg: "no suitable NAS server found", + }, + { + name: "Error fetching NAS servers", + nasList: nil, + expectedErrMsg: "failed to fetch NAS servers", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + // Setup mock expectations + if tc.nasList == nil { + clientMock.On("GetNASServers", ctx).Return(nil, errors.New("failed to fetch NAS servers")).Once() + } else { + clientMock.On("GetNASServers", ctx).Return(tc.nasList, nil).Once() + } + + arr := &array.PowerStoreArray{Client: clientMock, NASCooldownTracker: array.NewNASCooldown(30*time.Minute, 1)} + for _, nas := range tc.markForFailure { + arr.NASCooldownTracker.MarkFailure(nas) + } + + // Call the function + result, err := array.GetLeastUsedActiveNAS(ctx, arr, tc.nasServersInSc) + + // Assertions + if tc.expectedErrMsg != "" { + assert.Empty(t, result) + assert.Error(t, err) + assert.ErrorContains(t, err, tc.expectedErrMsg) + } else { + assert.NoError(t, err) + assert.NotNil(t, result) + assert.Equal(t, tc.expectedNAS.Name, result) + } + + clientMock.AssertExpectations(t) + }) + } +} + +func TestIsLessUsed(t *testing.T) { + makeFS := func(count int) []gopowerstore.FileSystem { + return make([]gopowerstore.FileSystem, count) + } + + tests := []struct { + name string + nas *gopowerstore.NAS + current *gopowerstore.NAS + expected bool + }{ + { + name: "NAS has fewer filesystems than current", + nas: &gopowerstore.NAS{Name: "nasA", FileSystems: makeFS(2)}, + current: &gopowerstore.NAS{Name: "nasB", FileSystems: makeFS(3)}, + expected: true, + }, + { + name: "NAS has more filesystems than current", + nas: &gopowerstore.NAS{Name: "nasA", FileSystems: makeFS(4)}, + current: &gopowerstore.NAS{Name: "nasB", FileSystems: makeFS(3)}, + expected: false, + }, + { + name: "NAS and current have same FS count, NAS name is lexicographically smaller", + nas: &gopowerstore.NAS{Name: "nasA", FileSystems: makeFS(2)}, + current: &gopowerstore.NAS{Name: "nasB", FileSystems: makeFS(2)}, + expected: true, + }, + { + name: "NAS and current have same FS count, NAS name is lexicographically larger", + nas: &gopowerstore.NAS{Name: "nasC", FileSystems: makeFS(2)}, + current: &gopowerstore.NAS{Name: "nasB", FileSystems: makeFS(2)}, + expected: false, + }, + { + name: "NAS and current are identical", + nas: &gopowerstore.NAS{Name: "nasA", FileSystems: makeFS(2)}, + current: &gopowerstore.NAS{Name: "nasA", FileSystems: makeFS(2)}, + expected: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + result := array.IsLessUsed(tc.nas, tc.current) + assert.Equal(t, tc.expected, result) + }) + } +} + +func TestResetFailure(t *testing.T) { + tests := []struct { + name string + nasList []string + markForFailure []string + expectedFailures int + expectedCooldown time.Duration + expectedMapLen int + }{ + { + name: "1 failure", + markForFailure: []string{"nas1"}, + expectedFailures: 1, + expectedCooldown: 1 * time.Minute, + }, + { + name: "2 failures", + markForFailure: []string{"nas1", "nas1"}, + expectedFailures: 2, + expectedCooldown: 1 * time.Minute, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + nas := array.NewNASCooldown(tc.expectedCooldown, 1) + for _, nasName := range tc.markForFailure { + nas.MarkFailure(nasName) + } + assert.Equal(t, tc.expectedFailures, nas.GetStatusMap()["nas1"].Failures) + assert.WithinDuration(t, time.Now(), nas.GetStatusMap()["nas1"].CooldownUntil, tc.expectedCooldown) + + nas.ResetFailure("nas1") + assert.Empty(t, nas.GetStatusMap()["nas1"]) + }) + } +} + +func TestFallbackRetry(t *testing.T) { + nas := array.NewNASCooldown(1*time.Minute, 1) + nas.MarkFailure("nas1") + nas.MarkFailure("nas1") + nas.MarkFailure("nas3") + + tests := []struct { + name string + nasList []string + want string + }{ + { + name: "Test FallbackRetry with nas1, nas2, nas3", + nasList: []string{"nas1", "nas2", "nas3"}, + want: "nas2", + }, + { + name: "Test FallbackRetry with nas1, nas2", + nasList: []string{"nas1", "nas2"}, + want: "nas2", + }, + { + name: "Test FallbackRetry with nas1", + nasList: []string{"nas1"}, + want: "nas1", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := nas.FallbackRetry(tt.nasList) + if got != tt.want { + t.Errorf("got %v, want %v", got, tt.want) + } + }) + } +} + +func Test_getVolumeIDPrefix(t *testing.T) { + legacyVolumeID := "1cd254s" + + type args struct { + ID string + } + tests := []struct { + name string + args args + wantPrefix string + }{ + { + name: "legacy volume ID", + args: args{ + ID: legacyVolumeID, + }, + wantPrefix: "", + }, + { + name: "volume UUID with no prefix", + args: args{ + ID: validBlockVolumeUUID, + }, + wantPrefix: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotPrefix := array.GetVolumeUUIDPrefix(tt.args.ID) + if gotPrefix != tt.wantPrefix { + t.Errorf("getVolumeIDPrefix() gotPrefix = %v, want %v", gotPrefix, tt.wantPrefix) + } + }) + } +} + +func TestLocker_Get(t *testing.T) { + tests := []struct { + name string // description of this test case + // Named input parameters for target function. + globalID string + locker func(tt *testing.T) *array.Locker + want storage.HealthChecker + wantErr bool + }{ + { + name: "fail", + globalID: "bad-id", + locker: func(_ *testing.T) *array.Locker { + return &array.Locker{} + }, + want: nil, + wantErr: true, + }, + { + name: "success", + globalID: validGlobalID, + locker: func(_ *testing.T) *array.Locker { + locker := array.Locker{} + locker.SetArrays(map[string]*array.PowerStoreArray{ + validGlobalID: { + GlobalID: validGlobalID, + }, + }) + return &locker + }, + want: &array.PowerStoreArray{}, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := tt.locker(t) + got, gotErr := s.Get(tt.globalID) + if gotErr != nil { + if !tt.wantErr { + t.Errorf("Get() failed: %v", gotErr) + } + return + } + if tt.wantErr { + t.Fatal("Get() succeeded unexpectedly") + } + + if !tt.wantErr && got == nil { + t.Errorf("Get() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestPowerStoreArray_IsOnline(t *testing.T) { + tests := []struct { + name string // description of this test case + array func(tt *testing.T) *array.PowerStoreArray + want bool + }{ + { + name: "is online", + array: func(tt *testing.T) *array.PowerStoreArray { + mArray := gopowerstoremock.NewClient(tt) + mArray.On("GetCluster", mock.Anything).Return(gopowerstore.Cluster{}, nil) + + return &array.PowerStoreArray{ + Client: mArray, + } + }, + want: true, + }, + { + name: "is online", + array: func(tt *testing.T) *array.PowerStoreArray { + mArray := gopowerstoremock.NewClient(tt) + mArray.On("GetCluster", mock.Anything).Return(gopowerstore.Cluster{}, errors.New("failed to get cluster")) + + return &array.PowerStoreArray{ + Client: mArray, + } + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + psa := tt.array(t) + + got := psa.IsOnline(context.Background()) + + if got != tt.want { + t.Errorf("IsOnline() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestVolumeHandle_IsMetro(t *testing.T) { + tests := []struct { + name string + volumeHandle array.VolumeHandle + want bool + }{ + { + name: "is metro", + volumeHandle: array.VolumeHandle{ + LocalUUID: validBlockVolumeUUID, + LocalArrayGlobalID: validGlobalID, + Protocol: scsi, + RemoteUUID: validRemoteBlockVolumeUUID, + RemoteArrayGlobalID: validRemoteGlobalID, + }, + want: true, + }, + { + name: "is not metro", + volumeHandle: array.VolumeHandle{ + LocalUUID: validBlockVolumeUUID, + LocalArrayGlobalID: validGlobalID, + Protocol: scsi, + RemoteUUID: "", + RemoteArrayGlobalID: "", + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + v := tt.volumeHandle + + got := v.IsMetro() + + if got != tt.want { + t.Errorf("IsMetro() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestDoesNodeMatchMetroSelectors(t *testing.T) { + t.Run("Node matches local node selector", func(t *testing.T) { + node := &k8score.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nodeid", + Labels: map[string]string{ + "zone1": "local", // does not match the node selector + }, + Annotations: map[string]string{ + "csi.volume.kubernetes.io/nodeid": `{"csi-powerstore.dellemc.com":"nodeid"}`, + }, + }, + } + metroArr := &array.PowerStoreArray{ + HostConnectivity: &array.HostConnectivity{ + Metro: array.MetroConnectivityOptions{ + ColocatedLocal: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "zone1", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"local"}, + }, + }, + }, + }, + }, + }, + }, + GlobalID: "APM0012345", + Endpoint: "https://192.168.0.3/api/rest", + Username: "admin", + Password: "pass", + Insecure: true, + IP: "192.168.0.3", + } + result := metroArr.DoesNodeMatchMetroSelectors(node) + assert.True(t, result) + }) + + t.Run("Node matches colocated node selector", func(t *testing.T) { + node := &k8score.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nodeid", + Labels: map[string]string{ + "zone3": "both", // does not match the node selector + }, + Annotations: map[string]string{ + "csi.volume.kubernetes.io/nodeid": `{"csi-powerstore.dellemc.com":"nodeid"}`, + }, + }, + } + metroArr := &array.PowerStoreArray{ + HostConnectivity: &array.HostConnectivity{ + Metro: array.MetroConnectivityOptions{ + ColocatedBoth: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "zone3", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"both"}, + }, + }, + }, + }, + }, + }, + }, + GlobalID: "APM0012345", + Endpoint: "https://192.168.0.3/api/rest", + Username: "admin", + Password: "pass", + Insecure: true, + IP: "192.168.0.3", + } + result := metroArr.DoesNodeMatchMetroSelectors(node) + assert.True(t, result) + }) + + t.Run("Node matches remote node selector", func(t *testing.T) { + node := &k8score.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nodeid", + Labels: map[string]string{ + "zone2": "remote", // does not match the node selector + }, + Annotations: map[string]string{ + "csi.volume.kubernetes.io/nodeid": `{"csi-powerstore.dellemc.com":"nodeid"}`, + }, + }, + } + metroArr := &array.PowerStoreArray{ + HostConnectivity: &array.HostConnectivity{ + Metro: array.MetroConnectivityOptions{ + ColocatedRemote: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "zone2", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"remote"}, + }, + }, + }, + }, + }, + }, + }, + GlobalID: "APM0012345", + Endpoint: "https://192.168.0.3/api/rest", + Username: "admin", + Password: "pass", + Insecure: true, + IP: "192.168.0.3", + } + result := metroArr.DoesNodeMatchMetroSelectors(node) + assert.True(t, result) + }) + + t.Run("Node does not match any node selector", func(t *testing.T) { + node := &k8score.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nodeid", + Labels: map[string]string{ + "zone2": "local", // does not match the node selector + }, + Annotations: map[string]string{ + "csi.volume.kubernetes.io/nodeid": `{"csi-powerstore.dellemc.com":"nodeid"}`, + }, + }, + } + metroArr := &array.PowerStoreArray{ + HostConnectivity: &array.HostConnectivity{ + Metro: array.MetroConnectivityOptions{ + ColocatedRemote: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "zone2", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"remote"}, + }, + }, + }, + }, + }, + }, + }, + GlobalID: "APM0012345", + Endpoint: "https://192.168.0.3/api/rest", + Username: "admin", + Password: "pass", + Insecure: true, + IP: "192.168.0.3", + } + result := metroArr.DoesNodeMatchMetroSelectors(node) + assert.False(t, result) + }) + + t.Run("Error creating local node selector", func(t *testing.T) { + node := &k8score.Node{} + arr := &array.PowerStoreArray{ + HostConnectivity: &array.HostConnectivity{ + Metro: array.MetroConnectivityOptions{ + ColocatedLocal: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"value"}, + }, + }, + }, + }, + }, + }, + }, + } + result := arr.DoesNodeMatchMetroSelectors(node) + assert.False(t, result) + }) + t.Run("Error creating remote node selector", func(t *testing.T) { + node := &k8score.Node{} + arr := &array.PowerStoreArray{ + HostConnectivity: &array.HostConnectivity{ + Metro: array.MetroConnectivityOptions{ + ColocatedRemote: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"value"}, + }, + }, + }, + }, + }, + }, + }, + } + result := arr.DoesNodeMatchMetroSelectors(node) + assert.False(t, result) + }) + t.Run("Error creating Both node selector", func(t *testing.T) { + node := &k8score.Node{} + arr := &array.PowerStoreArray{ + HostConnectivity: &array.HostConnectivity{ + Metro: array.MetroConnectivityOptions{ + ColocatedBoth: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"value"}, + }, + }, + }, + }, + }, + }, + }, + } + result := arr.DoesNodeMatchMetroSelectors(node) + assert.False(t, result) + }) + t.Run("Node does not find any metronode selector", func(t *testing.T) { + node := &k8score.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nodeid", + Labels: map[string]string{ + "zone2": "local", // does not match the node selector + }, + Annotations: map[string]string{ + "csi.volume.kubernetes.io/nodeid": `{"csi-powerstore.dellemc.com":"nodeid"}`, + }, + }, + } + metroArr := &array.PowerStoreArray{ + HostConnectivity: &array.HostConnectivity{ + Local: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "zone4", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"local"}, + }, + }, + }, + }, + }, + }, + GlobalID: "APM0012345", + Endpoint: "https://192.168.0.3/api/rest", + Username: "admin", + Password: "pass", + Insecure: true, + IP: "192.168.0.3", + } + result := metroArr.DoesNodeMatchMetroSelectors(node) + assert.False(t, result) + }) +} diff --git a/pkg/array/k8svisibility.go b/pkg/array/k8svisibility.go deleted file mode 100644 index bdd24a40..00000000 --- a/pkg/array/k8svisibility.go +++ /dev/null @@ -1,160 +0,0 @@ -/* - * - * Copyright © 2022-2023 Dell Inc. or its subsidiaries. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package array - -import ( - "context" - "path/filepath" - "strconv" - "strings" - - "github.com/dell/csi-powerstore/v2/pkg/common" - "github.com/dell/csi-powerstore/v2/pkg/common/fs" - csictx "github.com/dell/gocsi/context" - "github.com/dell/gopowerstore" - log "github.com/sirupsen/logrus" - "gopkg.in/yaml.v3" -) - -// ClusterInfoStruct contains k8s server address -type ClusterInfoStruct struct { - Server string `yaml:"server"` -} - -// Cluster contains k8s cluster information -type Cluster struct { - ClusterInfo ClusterInfoStruct `yaml:"cluster"` - Name string `yaml:"name"` -} - -// KubeConfig contains information read from kubeconfig file -type KubeConfig struct { - APIVersion string `yaml:"apiVersion"` - Clusters []Cluster `yaml:"clusters,flow"` -} - -// K8sClusterInfo contains information of k8s cluster -type K8sClusterInfo struct { - Name string - IPAddress string - Port int - Token string -} - -func getRootDirectory() string { - if rootPath, ok := csictx.LookupEnv(context.Background(), common.EnvCtrlRootPath); ok { - return rootPath - } - return "/" -} - -func getKubeConfigInfo(fs fs.Interface) (KubeConfig, error) { - var kubeconfig KubeConfig - kubeconfigInfo := []byte{} - kubeconfigDir := filepath.Join(getRootDirectory(), "etc/kubernetes") - - kubeconfigPath := filepath.Join(kubeconfigDir, "kubelet.conf") - - log.Debug("K8s visibility: Reading file: \n", kubeconfigPath) - kubeconfigInfo, err := fs.ReadFile(kubeconfigPath) - if err != nil { - log.Warnf("K8s visibility: Error reading file: %s err: %s\n", kubeconfigPath, err.Error()) - - kubeconfigPath = filepath.Join(kubeconfigDir, "admin.conf") - log.Info("K8s visibility: Reading file: \n", kubeconfigPath) - kubeconfigInfo, err = fs.ReadFile(kubeconfigPath) - if err != nil { - return kubeconfig, err - } - } - - err = yaml.Unmarshal([]byte(kubeconfigInfo), &kubeconfig) - if err != nil { - return kubeconfig, err - } - - return kubeconfig, nil -} - -func getK8sVisibilityServiceToken(fs fs.Interface) (string, error) { - tokenPath := filepath.Join("/powerstore-visibility", "token") - token, err := fs.ReadFile(tokenPath) - if err != nil { - return "", err - } - - return string(token), nil -} - -func getK8sClusterInfo(fs fs.Interface) ([]K8sClusterInfo, error) { - k8sClusters := []K8sClusterInfo{} - - kubeconfig, err := getKubeConfigInfo(fs) - if err != nil { - return k8sClusters, err - } - log.Debug("K8s visibility: Number of clusters in kube config file: \n", len(kubeconfig.Clusters)) - - token, err := getK8sVisibilityServiceToken(fs) - if err != nil { - return k8sClusters, err - } - - for _, cluster := range kubeconfig.Clusters { - log.Debugf("K8s visibility: cluster name: %s server IP: %s", cluster.Name, cluster.ClusterInfo.Server) - ipAddressPort := strings.Replace(cluster.ClusterInfo.Server, "https://", "", 1) - ipAddressPortArray := strings.Split(ipAddressPort, ":") - - if len(ipAddressPortArray) != 2 { - log.Errorf("cannot determine k8s cluster IP address and port from kube config\n") - continue - } - - port, err := strconv.Atoi(ipAddressPortArray[1]) - if err != nil { - log.Errorf("cannot determine k8s cluster port from kube config\n") - continue - } - - log.Debugf("K8s visibility: K8s Cluster IP Address: %s Port: %s", ipAddressPortArray[0], ipAddressPortArray[1]) - k8sClusters = append(k8sClusters, K8sClusterInfo{ - Name: cluster.Name, - IPAddress: ipAddressPortArray[0], - Port: port, - Token: token, - }) - } - - return k8sClusters, nil -} - -func isK8sClusterAPISupported(client gopowerstore.Client) bool { - k8sClusterAPISupported := false - majorMinorVersion, err := client.GetSoftwareMajorMinorVersion(context.Background()) - if err != nil { - log.Errorf("couldn't get the software version installed on the PowerStore array: %v", err) - return k8sClusterAPISupported - } - - if majorMinorVersion >= 4.0 { - k8sClusterAPISupported = true - } else { - log.Debugf("Software version installed on the PowerStore array: %v\n", majorMinorVersion) - } - - return k8sClusterAPISupported -} diff --git a/pkg/array/metro_utils.go b/pkg/array/metro_utils.go new file mode 100644 index 00000000..4d09176b --- /dev/null +++ b/pkg/array/metro_utils.go @@ -0,0 +1,200 @@ +/* + * + * Copyright © 2025 Dell Inc. or its subsidiaries. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package array + +import ( + "context" + "time" + + drv1 "github.com/dell/csm-dr/api/v1" + drv1Client "github.com/dell/csm-dr/pkg/client" + "github.com/dell/gopowerstore" + k8sErrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrlClient "sigs.k8s.io/controller-runtime/pkg/client" +) + +var GetDRClientFunc = drv1Client.Get + +type MetroFracturedResponse struct { + IsFractured bool + VolumeName string + State string +} + +const ( + MediumTimeout = 30 * time.Second + MetroPrefixRegex = `^Metro_(Demote|Promote|Reprotect).*` +) + +func IsMetroFractured(ctx context.Context, client gopowerstore.Client, id string) (*MetroFracturedResponse, error) { + log := log.WithContext(ctx) + arrayVolume, err := client.GetVolume(ctx, id) + if err != nil { + return nil, err + } + + if arrayVolume.MetroReplicationSessionID != "" { + log.Infof("[METRO] MetroReplicationSessionID %s", arrayVolume.MetroReplicationSessionID) + + ctxLocal, cancelLocal := context.WithTimeout(ctx, 5*time.Second) + defer cancelLocal() + replicationSession, err := client.GetReplicationSessionByID(ctxLocal, arrayVolume.MetroReplicationSessionID) + if err != nil { + log.Errorf("[METRO] Unable to get replication session information by ID: %s, errror: %s", arrayVolume.MetroReplicationSessionID, err.Error()) + return nil, err + } + + if replicationSession.State == "Fractured" { + // We should only go here if the replicationSession is Fractured. + log.Infof("[METRO] ReplicationSession Status %s, LocalResourceState %s", replicationSession.State, replicationSession.LocalResourceState) + + return &MetroFracturedResponse{true, arrayVolume.Name, replicationSession.LocalResourceState}, nil + } + } + + return &MetroFracturedResponse{false, arrayVolume.Name, ""}, nil +} + +// checkMetroState checks metro state of a volume. +// Tries to get metroState from the localArray first. if there was error fetching this, tries to get the metro state from the remote array. +// Parameters: volumeHandle of the metro volume and the clients for the local and remote arrays. +// Returns: MetroFracturedResponse, bool indicating if localVolume of the metro was demoted or not and error +// - empty MetroFracturedResponse , false and error in case of error checking metro state. +// - MetroFracturedResponse(including isFractured and volumeName), true, nil error in case metro is Fractured and localVolume is demoted. +// - MetroFracturedResponse(including isFractured and volumeName), false, nil error in case metro is Fractured and localVolume is promoted. +// +// MetroFracturedResponse ( includes isFractured and volumeName which are used from the response) , a boolean that indicates whether the localVolume of the metro was demoted or not and error. +func CheckMetroState(ctx context.Context, volumeHandle VolumeHandle, localClient gopowerstore.Client, remoteClient gopowerstore.Client) (*MetroFracturedResponse, bool, error) { + log := log.WithContext(ctx) + localDemoted := false + ctxLocal, cancelLocal := context.WithTimeout(context.Background(), MediumTimeout) + defer cancelLocal() + metroResp, err := IsMetroFractured(ctxLocal, localClient, volumeHandle.LocalUUID) + if err != nil { + log.Errorf("error checking on local array if metro is fractured: %s", err.Error()) + ctxRemote, cancelRemote := context.WithTimeout(context.Background(), MediumTimeout) + defer cancelRemote() + metroResp, err = IsMetroFractured(ctxRemote, remoteClient, volumeHandle.RemoteUUID) + if err != nil { + log.Errorf("error checking on remote array if metro is fractured: %s", err.Error()) + return metroResp, false, err + } + + if metroResp.IsFractured && (metroResp.State == "Demoted" || metroResp.State == "System_Demoted") { + // Remote is Demoted. So local must be Promoted + localDemoted = false + } else { + localDemoted = true + } + } else { + if metroResp.IsFractured && (metroResp.State == "Demoted" || metroResp.State == "System_Demoted") { + localDemoted = true + } + } + return metroResp, localDemoted, nil +} + +func CreateOrUpdateJournalEntry(ctx context.Context, name string, + volumeHandle VolumeHandle, deferredArrayID, nodeName, operation string, + request []byte, +) error { + log := log.WithContext(ctx) + + id := volumeHandle.LocalUUID + arrayID := volumeHandle.LocalArrayGlobalID + remoteArrayID := volumeHandle.RemoteArrayGlobalID + + drClient, err := GetDRClientFunc(ctx) + if err != nil { + log.Errorf("[METRO] Unable to get dr client, error: %s", err.Error()) + return err + } + + var journal drv1.VolumeJournal + key := ctrlClient.ObjectKey{ + Name: "journal-" + name, + } + + deferEntry := drv1.JournalEntry{ + Operation: operation, + Status: "pending-reconciliation", + Time: time.Now().Format(time.RFC3339), + Host: nodeName, + Array: deferredArrayID, + Request: request, + } + + err = drClient.Get(ctx, key, &journal) + if err != nil { + if !k8sErrors.IsNotFound(err) { + log.Errorf("Unable to retrieve volume journal: %s", err.Error()) + return err + } + + // We didn't find the entry so we would need to create it. + journal = drv1.VolumeJournal{ + ObjectMeta: metav1.ObjectMeta{ + Name: "journal-" + name, + }, + Spec: drv1.VolumeJournalSpec{ + VolumeUUID: id, + OriginalArray: arrayID, + FailoverArray: remoteArrayID, + JournalEntries: []drv1.JournalEntry{ + deferEntry, + }, + }, + } + + err = drClient.Create(context.Background(), &journal) + if err != nil { + log.Errorf("[METRO] Error creating volume journals: %s", err.Error()) + return err + } + + log.Infof("[METRO] Successfully created volume journal: %s", journal.Name) + return nil + } + + found := false + for i, entry := range journal.Spec.JournalEntries { + if entry.Operation == operation { + if entry.Status == "pending-reconciliation" && entry.Host == nodeName { + journal.Spec.JournalEntries[i] = deferEntry + } + + found = true + + break + } + } + + if !found { + journal.Spec.JournalEntries = append(journal.Spec.JournalEntries, deferEntry) + } + + err = drClient.Update(ctx, &journal) + if err != nil { + log.Errorf("Unable to update volume journal: %s", err) + return err + } + + return nil +} diff --git a/pkg/array/metro_utils_test.go b/pkg/array/metro_utils_test.go new file mode 100644 index 00000000..143840c7 --- /dev/null +++ b/pkg/array/metro_utils_test.go @@ -0,0 +1,602 @@ +/* + * + * Copyright © 2025 Dell Inc. or its subsidiaries. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package array + +import ( + "context" + "errors" + "fmt" + "testing" + + drv1 "github.com/dell/csm-dr/api/v1" + "github.com/dell/gopowerstore" + gopowerstoremock "github.com/dell/gopowerstore/mocks" + "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/gogo/protobuf/proto" + "github.com/google/uuid" + "github.com/stretchr/testify/mock" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestIsMetroFractured(t *testing.T) { + volumeHandle := VolumeHandle{ + LocalUUID: uuid.New().String(), + } + replicationSessionID := uuid.New().String() + + tests := []struct { + name string + client gopowerstore.Client + before func(*gopowerstoremock.Client) + wantResponse *MetroFracturedResponse + wantErr error + }{ + { + name: "IsMetroFractured - Not Replication Session ID", + client: new(gopowerstoremock.Client), + before: func(client *gopowerstoremock.Client) { + client.On("GetVolume", mock.Anything, mock.Anything).Return( + gopowerstore.Volume{ + ID: uuid.New().String(), + Name: "myVolume", + MetroReplicationSessionID: "", + }, nil) + }, + wantResponse: &MetroFracturedResponse{IsFractured: false, VolumeName: "myVolume", State: ""}, + wantErr: nil, + }, + { + name: "IsMetroFractured - Replication Session is OK", + client: new(gopowerstoremock.Client), + before: func(client *gopowerstoremock.Client) { + client.On("GetVolume", mock.Anything, mock.Anything).Return( + gopowerstore.Volume{ + ID: uuid.New().String(), + Name: "myVolume", + MetroReplicationSessionID: replicationSessionID, + }, nil) + + client.On("GetReplicationSessionByID", mock.Anything, mock.Anything).Return( + gopowerstore.ReplicationSession{ + ID: replicationSessionID, + State: "OK", + }, nil) + }, + wantResponse: &MetroFracturedResponse{IsFractured: false, VolumeName: "myVolume", State: ""}, + wantErr: nil, + }, + { + name: "IsMetroFractured - Replication Session is Fractured", + client: new(gopowerstoremock.Client), + before: func(client *gopowerstoremock.Client) { + client.On("GetVolume", mock.Anything, mock.Anything).Return( + gopowerstore.Volume{ + ID: uuid.New().String(), + Name: "myVolume", + MetroReplicationSessionID: replicationSessionID, + }, nil) + + client.On("GetReplicationSessionByID", mock.Anything, mock.Anything).Return( + gopowerstore.ReplicationSession{ + ID: replicationSessionID, + State: "Fractured", + LocalResourceState: "Promoted", + }, nil) + }, + wantResponse: &MetroFracturedResponse{IsFractured: true, VolumeName: "myVolume", State: "Promoted"}, + wantErr: nil, + }, + { + name: "IsMetroFractured - error: unable to get volume", + client: new(gopowerstoremock.Client), + before: func(client *gopowerstoremock.Client) { + client.On("GetVolume", mock.Anything, mock.Anything).Return( + gopowerstore.Volume{}, errors.New("unable to get volume")) + }, + wantResponse: nil, + wantErr: errors.New("unable to get volume"), + }, + { + name: "IsMetroFractured - error: unable to get replication session by ID", + client: new(gopowerstoremock.Client), + before: func(client *gopowerstoremock.Client) { + client.On("GetVolume", mock.Anything, mock.Anything).Return( + gopowerstore.Volume{ + ID: uuid.New().String(), + Name: "myVolume", + MetroReplicationSessionID: replicationSessionID, + }, nil) + + client.On("GetReplicationSessionByID", mock.Anything, mock.Anything).Return( + gopowerstore.ReplicationSession{}, errors.New("unable to get replication session by ID")) + }, + wantResponse: nil, + wantErr: errors.New("unable to get replication session by ID"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.before != nil { + tt.before(tt.client.(*gopowerstoremock.Client)) + } + + response, err := IsMetroFractured(t.Context(), tt.client, volumeHandle.LocalUUID) + if err != nil { + if errors.Is(err, tt.wantErr) { + t.Errorf("IsMetroFractured() received unexpected error: got %v, want %v", err, tt.wantErr) + } + } + + if tt.wantResponse != nil { + if tt.wantResponse.IsFractured != response.IsFractured || tt.wantResponse.VolumeName != response.VolumeName || tt.wantResponse.State != response.State { + t.Errorf("IsMetroFractured() received unexpected response: got %v, want %v", response, tt.wantResponse) + } + } + }) + } +} + +func TestCheckMetroState(t *testing.T) { + tests := []struct { + name string + volumeHandle VolumeHandle + localClient gopowerstore.Client + remoteClient gopowerstore.Client + wantResponse *MetroFracturedResponse + wantLocalDemoted bool + wantErr error + beforeLocal func(*gopowerstoremock.Client) + beforeRemote func(*gopowerstoremock.Client) + }{ + { + name: "CheckMetroState - Local volume is demoted", + volumeHandle: VolumeHandle{ + LocalUUID: "local-uuid", + RemoteUUID: "remote-uuid", + }, + localClient: new(gopowerstoremock.Client), + remoteClient: new(gopowerstoremock.Client), + wantResponse: &MetroFracturedResponse{ + IsFractured: true, + VolumeName: "volume-name", + State: "Demoted", + }, + wantLocalDemoted: true, + wantErr: nil, + beforeLocal: func(client *gopowerstoremock.Client) { + client.On("GetVolume", mock.Anything, "local-uuid").Return(gopowerstore.Volume{ + ID: "local-uuid", + Name: "volume-name", + MetroReplicationSessionID: "replication-session-id", + }, nil) + client.On("GetReplicationSessionByID", mock.Anything, "replication-session-id").Return(gopowerstore.ReplicationSession{ + ID: "replication-session-id", + State: "Fractured", + LocalResourceState: "Demoted", + }, nil) + }, + beforeRemote: func(_ *gopowerstoremock.Client) { + // Not expected to be called + }, + }, + { + name: "CheckMetroState - Local volume is promoted", + volumeHandle: VolumeHandle{ + LocalUUID: "local-uuid", + RemoteUUID: "remote-uuid", + }, + localClient: new(gopowerstoremock.Client), + remoteClient: new(gopowerstoremock.Client), + wantResponse: &MetroFracturedResponse{ + IsFractured: true, + VolumeName: "volume-name", + State: "Promoted", + }, + wantLocalDemoted: false, + wantErr: nil, + beforeLocal: func(client *gopowerstoremock.Client) { + client.On("GetVolume", mock.Anything, "local-uuid").Return(gopowerstore.Volume{ + ID: "local-uuid", + Name: "volume-name", + MetroReplicationSessionID: "replication-session-id", + }, nil) + client.On("GetReplicationSessionByID", mock.Anything, "replication-session-id").Return(gopowerstore.ReplicationSession{ + ID: "replication-session-id", + State: "Fractured", + LocalResourceState: "Promoted", + }, nil) + }, + beforeRemote: func(_ *gopowerstoremock.Client) { + // Not expected to be called + }, + }, + { + name: "CheckMetroState - Error getting local volume, remote volume promoted", + volumeHandle: VolumeHandle{ + LocalUUID: "local-uuid", + RemoteUUID: "remote-uuid", + }, + localClient: new(gopowerstoremock.Client), + remoteClient: new(gopowerstoremock.Client), + wantResponse: &MetroFracturedResponse{ + IsFractured: true, + VolumeName: "volume-name", + State: "Promoted", + }, + wantLocalDemoted: true, + wantErr: nil, + beforeLocal: func(client *gopowerstoremock.Client) { + client.On("GetVolume", mock.Anything, "local-uuid").Return(gopowerstore.Volume{}, fmt.Errorf("error getting local volume")) + }, + beforeRemote: func(client *gopowerstoremock.Client) { + client.On("GetVolume", mock.Anything, "remote-uuid").Return(gopowerstore.Volume{ + ID: "remote-uuid", + Name: "volume-name", + MetroReplicationSessionID: "replication-session-id", + }, nil) + client.On("GetReplicationSessionByID", mock.Anything, "replication-session-id").Return(gopowerstore.ReplicationSession{ + ID: "replication-session-id", + State: "Fractured", + LocalResourceState: "Promoted", + }, nil) + }, + }, + { + name: "CheckMetroState - Error getting local volume, remote volume Demoted", + volumeHandle: VolumeHandle{ + LocalUUID: "local-uuid", + RemoteUUID: "remote-uuid", + }, + localClient: new(gopowerstoremock.Client), + remoteClient: new(gopowerstoremock.Client), + wantResponse: &MetroFracturedResponse{ + IsFractured: true, + VolumeName: "volume-name", + State: "Demoted", + }, + wantLocalDemoted: false, + wantErr: nil, + beforeLocal: func(client *gopowerstoremock.Client) { + client.On("GetVolume", mock.Anything, "local-uuid").Return(gopowerstore.Volume{}, fmt.Errorf("error getting local volume")) + }, + beforeRemote: func(client *gopowerstoremock.Client) { + client.On("GetVolume", mock.Anything, "remote-uuid").Return(gopowerstore.Volume{ + ID: "remote-uuid", + Name: "volume-name", + MetroReplicationSessionID: "replication-session-id", + }, nil) + client.On("GetReplicationSessionByID", mock.Anything, "replication-session-id").Return(gopowerstore.ReplicationSession{ + ID: "replication-session-id", + State: "Fractured", + LocalResourceState: "Demoted", + }, nil) + }, + }, + { + name: "CheckMetroState - Error getting local volume and remote volume", + volumeHandle: VolumeHandle{ + LocalUUID: "local-uuid", + RemoteUUID: "remote-uuid", + }, + localClient: new(gopowerstoremock.Client), + remoteClient: new(gopowerstoremock.Client), + wantResponse: nil, + wantLocalDemoted: false, + wantErr: fmt.Errorf("error getting remote volume"), + beforeLocal: func(client *gopowerstoremock.Client) { + client.On("GetVolume", mock.Anything, "local-uuid").Return(gopowerstore.Volume{}, fmt.Errorf("error getting local volume")) + }, + beforeRemote: func(client *gopowerstoremock.Client) { + client.On("GetVolume", mock.Anything, "remote-uuid").Return(gopowerstore.Volume{}, fmt.Errorf("error getting remote volume")) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.beforeLocal(tt.localClient.(*gopowerstoremock.Client)) + tt.beforeRemote(tt.remoteClient.(*gopowerstoremock.Client)) + + response, localDemoted, err := CheckMetroState(context.Background(), tt.volumeHandle, tt.localClient, tt.remoteClient) + + if tt.wantResponse != nil { + if tt.wantResponse.IsFractured != response.IsFractured || tt.wantResponse.VolumeName != response.VolumeName || tt.wantResponse.State != response.State { + t.Errorf("CheckMetroState() response = %v, want %v", response, tt.wantResponse) + } + } + + if localDemoted != tt.wantLocalDemoted { + t.Errorf("CheckMetroState() localDemoted = %v, want %v", localDemoted, tt.wantLocalDemoted) + } + + if (tt.wantErr != nil && err == nil) || (tt.wantErr == nil && err != nil) { + t.Errorf("CheckMetroState() error = %v, want %v", err, tt.wantErr) + } + }) + } +} + +func TestCreateOrUpdateJournalEntry(t *testing.T) { + defaultGetClientFunc := GetDRClientFunc + + volumeName := "my-volume" + volumeHandle := VolumeHandle{ + LocalUUID: uuid.New().String(), + } + nodeName := "myNode" + tests := []struct { + name string + operation string + wantErr error + init func(myClient client.Client) + before func(operation string) ([]byte, client.Client) + }{ + { + name: "CreateOrUpdateJournalEntry - Success: Creation of Journal", + operation: "NodeStageVolume", + init: func(myClient client.Client) { + GetDRClientFunc = func(_ context.Context) (client.Client, error) { + return myClient, nil + } + }, + before: func(_ string) ([]byte, client.Client) { + req := &csi.NodeStageVolumeRequest{ + VolumeId: uuid.NewString(), + } + + deferredRequest, err := proto.Marshal(req) + if err != nil { + return nil, nil + } + + volumeJournal := drv1.VolumeJournal{} + scheme := setupDrScheme() + client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&volumeJournal).Build() + + return deferredRequest, client + }, + wantErr: nil, + }, + { + name: "CreateOrUpdateJournalEntry - Success: Update of Journal (found)", + operation: "NodeStageVolume", + init: func(myClient client.Client) { + GetDRClientFunc = func(_ context.Context) (client.Client, error) { + return myClient, nil + } + }, + before: func(operation string) ([]byte, client.Client) { + req := &csi.NodeStageVolumeRequest{ + VolumeId: uuid.NewString(), + } + + deferredRequest, err := proto.Marshal(req) + if err != nil { + return nil, nil + } + + volumeJournal := drv1.VolumeJournal{ + ObjectMeta: metav1.ObjectMeta{ + Name: "journal-" + volumeName, + }, + Spec: drv1.VolumeJournalSpec{ + JournalEntries: []drv1.JournalEntry{ + { + Operation: operation, + Status: "pending-reconciliation", + Host: nodeName, + }, + }, + }, + } + scheme := setupDrScheme() + client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&volumeJournal).Build() + + return deferredRequest, client + }, + wantErr: nil, + }, + { + name: "CreateOrUpdateJournalEntry - Success: Update of Journal (not found)", + operation: "NodeStageVolume", + init: func(myClient client.Client) { + GetDRClientFunc = func(_ context.Context) (client.Client, error) { + return myClient, nil + } + }, + before: func(_ string) ([]byte, client.Client) { + req := &csi.NodeStageVolumeRequest{ + VolumeId: uuid.NewString(), + } + + deferredRequest, err := proto.Marshal(req) + if err != nil { + return nil, nil + } + + volumeJournal := drv1.VolumeJournal{ + ObjectMeta: metav1.ObjectMeta{ + Name: "journal-" + volumeName, + }, + Spec: drv1.VolumeJournalSpec{ + JournalEntries: []drv1.JournalEntry{ + { + Operation: "ControllerPublishVolume", + Status: "pending-reconciliation", + }, + }, + }, + } + scheme := setupDrScheme() + client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&volumeJournal).Build() + + return deferredRequest, client + }, + wantErr: nil, + }, + { + name: "CreateOrUpdateJournalEntry - Error: Unable to get client", + operation: "NodeStageVolume", + init: func(_ client.Client) { + GetDRClientFunc = func(_ context.Context) (client.Client, error) { + return nil, errors.New("unable to get dr client") + } + }, + before: func(_ string) ([]byte, client.Client) { + volumeJournal := drv1.VolumeJournal{} + scheme := setupDrScheme() + client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&volumeJournal).Build() + + return nil, client + }, + wantErr: errors.New("unable to get dr client"), + }, + { + name: "CreateOrUpdateJournalEntry - Error: CSMDR not registered", + operation: "NodeStageVolume", + init: func(myClient client.Client) { + GetDRClientFunc = func(_ context.Context) (client.Client, error) { + return myClient, nil + } + }, + before: func(_ string) ([]byte, client.Client) { + client := fake.NewClientBuilder().Build() + + return nil, client + }, + wantErr: errors.New("not registered"), + }, + { + name: "CreateOrUpdateJournalEntry - Error: Creation of Journal", + operation: "NodeStageVolume", + init: func(myClient client.Client) { + GetDRClientFunc = func(_ context.Context) (client.Client, error) { + return myClient, nil + } + }, + before: func(_ string) ([]byte, client.Client) { + req := &csi.NodeStageVolumeRequest{ + VolumeId: uuid.NewString(), + } + + deferredRequest, err := proto.Marshal(req) + if err != nil { + return nil, nil + } + + volumeJournal := drv1.VolumeJournal{} + scheme := setupDrScheme() + client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&volumeJournal).Build() + + testClient := &failingClient{Client: client, failOperator: map[string]bool{"create": true}} + + return deferredRequest, testClient + }, + }, + { + name: "CreateOrUpdateJournalEntry - Error: Update of Journal", + operation: "NodeStageVolume", + init: func(myClient client.Client) { + GetDRClientFunc = func(_ context.Context) (client.Client, error) { + return myClient, nil + } + }, + before: func(operation string) ([]byte, client.Client) { + req := &csi.NodeStageVolumeRequest{ + VolumeId: uuid.NewString(), + } + + deferredRequest, err := proto.Marshal(req) + if err != nil { + return nil, nil + } + + volumeJournal := drv1.VolumeJournal{ + ObjectMeta: metav1.ObjectMeta{ + Name: "journal-" + volumeName, + }, + Spec: drv1.VolumeJournalSpec{ + JournalEntries: []drv1.JournalEntry{ + { + Operation: operation, + Status: "pending-reconciliation", + }, + }, + }, + } + scheme := setupDrScheme() + client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&volumeJournal).Build() + + testClient := &failingClient{Client: client, failOperator: map[string]bool{"update": true}} + + return deferredRequest, testClient + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + request, client := tt.before(tt.operation) + t.Cleanup(func() { + GetDRClientFunc = defaultGetClientFunc + }) + + tt.init(client) + + err := CreateOrUpdateJournalEntry(t.Context(), volumeName, volumeHandle, volumeHandle.LocalArrayGlobalID, nodeName, tt.operation, request) + if err != nil { + if errors.Is(err, tt.wantErr) { + t.Errorf("IsMetroFractured() received unexpected error: got %v, want %v", err, tt.wantErr) + } + } + }) + } +} + +func setupDrScheme() *runtime.Scheme { + scheme := runtime.NewScheme() + _ = drv1.AddToScheme(scheme) + return scheme +} + +// failingClient is a client that fails on create and update +type failingClient struct { + client.Client + failOperator map[string]bool +} + +func (f *failingClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + if !f.failOperator["create"] { + return f.Client.Create(ctx, obj, opts...) + } + + return fmt.Errorf("simulated create failure") +} + +func (f *failingClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + if !f.failOperator["update"] { + return f.Client.Update(ctx, obj, opts...) + } + + return fmt.Errorf("simulated create failure") +} diff --git a/pkg/common/common_test.go b/pkg/common/common_test.go deleted file mode 100644 index ab495426..00000000 --- a/pkg/common/common_test.go +++ /dev/null @@ -1,431 +0,0 @@ -/* - * - * Copyright © 2021-2023 Dell Inc. or its subsidiaries. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package common_test - -import ( - "context" - "errors" - "fmt" - "os" - "reflect" - "testing" - - "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/dell/csi-powerstore/v2/mocks" - "github.com/dell/csi-powerstore/v2/pkg/common" - csictx "github.com/dell/gocsi/context" - "github.com/dell/gocsi/utils" - "github.com/dell/gopowerstore" - gopowerstoremock "github.com/dell/gopowerstore/mocks" - log "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" -) - -func TestCustomLogger(t *testing.T) { - log.SetLevel(log.DebugLevel) - lg := &common.CustomLogger{} - ctx := context.Background() - lg.Info(ctx, "foo") - lg.Debug(ctx, "bar") - lg.Error(ctx, "spam") -} - -func TestRmSockFile(t *testing.T) { - sockPath := "unix:///var/run/csi/csi.sock" - trimmedSockPath := "/var/run/csi/csi.sock" - _ = os.Setenv(utils.CSIEndpoint, sockPath) - - t.Run("removed socket", func(t *testing.T) { - fsMock := new(mocks.FsInterface) - fsMock.On("Stat", trimmedSockPath).Return(&mocks.FileInfo{}, nil) - fsMock.On("RemoveAll", trimmedSockPath).Return(nil) - - common.RmSockFile(fsMock) - }) - - t.Run("failed to remove socket", func(t *testing.T) { - fsMock := new(mocks.FsInterface) - fsMock.On("Stat", trimmedSockPath).Return(&mocks.FileInfo{}, nil) - fsMock.On("RemoveAll", trimmedSockPath).Return(fmt.Errorf("some error")) - - common.RmSockFile(fsMock) - }) - - t.Run("not found", func(t *testing.T) { - fsMock := new(mocks.FsInterface) - fsMock.On("Stat", trimmedSockPath).Return(&mocks.FileInfo{}, os.ErrNotExist) - - common.RmSockFile(fsMock) - }) - - t.Run("may or may not exist", func(t *testing.T) { - fsMock := new(mocks.FsInterface) - fsMock.On("Stat", trimmedSockPath).Return(&mocks.FileInfo{}, fmt.Errorf("some other error")) - - common.RmSockFile(fsMock) - }) - - t.Run("no endpoint set", func(t *testing.T) { - fsMock := new(mocks.FsInterface) - _ = os.Setenv(utils.CSIEndpoint, "") - - common.RmSockFile(fsMock) - }) - -} - -func TestSetLogFields(t *testing.T) { - t.Run("empty context", func(t *testing.T) { - common.SetLogFields(nil, log.Fields{}) - }) -} - -func TestGetLogFields(t *testing.T) { - t.Run("empty context", func(t *testing.T) { - fields := common.GetLogFields(nil) - assert.Equal(t, log.Fields{}, fields) - }) - - t.Run("req id", func(t *testing.T) { - ctx := context.WithValue(context.Background(), csictx.RequestIDKey, "1") - fields := common.GetLogFields(ctx) - assert.Equal(t, log.Fields{"RequestID": "1"}, fields) - }) -} - -func TestGetISCSITargetsInfoFromStorage(t *testing.T) { - t.Run("api error", func(t *testing.T) { - e := errors.New("some error") - clientMock := new(gopowerstoremock.Client) - clientMock.On("GetStorageISCSITargetAddresses", context.Background()).Return([]gopowerstore.IPPoolAddress{}, e) - _, err := common.GetISCSITargetsInfoFromStorage(clientMock, "A1") - assert.EqualError(t, err, e.Error()) - }) -} - -func TestGetFCTargetsInfoFromStorage(t *testing.T) { - t.Run("api error", func(t *testing.T) { - e := errors.New("some error") - clientMock := new(gopowerstoremock.Client) - clientMock.On("GetFCPorts", context.Background()).Return([]gopowerstore.FcPort{}, e) - _, err := common.GetFCTargetsInfoFromStorage(clientMock, "A1") - assert.EqualError(t, err, e.Error()) - }) -} - -func TestIsK8sMetadataSupported(t *testing.T) { - t.Run("api error", func(t *testing.T) { - e := errors.New("some error") - clientMock := new(gopowerstoremock.Client) - clientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(0.0), e) - version := common.IsK8sMetadataSupported(clientMock) - assert.Equal(t, version, false) - }) -} - -func TestGetNVMEFCTargetInfoFromStorage(t *testing.T) { - t.Run("api error", func(t *testing.T) { - e := errors.New("some error") - clientMock := new(gopowerstoremock.Client) - clientMock.On("GetCluster", context.Background()).Return(gopowerstore.Cluster{}, e) - clientMock.On("GetFCPorts", context.Background()).Return([]gopowerstore.FcPort{}, e) - _, err := common.GetNVMEFCTargetInfoFromStorage(clientMock, "A1") - assert.EqualError(t, err, e.Error()) - }) -} - -func TestHasRequiredTopology(t *testing.T) { - nfsTopology := &csi.Topology{Segments: map[string]string{"csi-powerstore.dellemc.com/10.0.0.0-nfs": "true"}} - iscsiTopology := &csi.Topology{Segments: map[string]string{"csi-powerstore.dellemc.com/10.0.0.0-iscsi": "true"}} - - type args struct { - topologies []*csi.Topology - arrIP string - requiredTopology string - } - tests := []struct { - name string - args args - want bool - }{ - { - name: "only nfs is present in topologies", - args: args{topologies: []*csi.Topology{nfsTopology}, arrIP: "10.0.0.0", requiredTopology: "nfs"}, - want: true, - }, - { - name: "nfs & iscsi is present in topologies", - args: args{topologies: []*csi.Topology{iscsiTopology, nfsTopology}, arrIP: "10.0.0.0", requiredTopology: "nfs"}, - want: true, - }, - { - name: "nfs is not present in topologies", - args: args{topologies: []*csi.Topology{iscsiTopology}, arrIP: "10.0.0.0", requiredTopology: "nfs"}, - want: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equalf(t, tt.want, common.HasRequiredTopology(tt.args.topologies, tt.args.arrIP, tt.args.requiredTopology), "HasRequiredTopology(%v, %v, %v)", tt.args.topologies, tt.args.arrIP, tt.args.requiredTopology) - }) - } -} - -func TestGetNfsTopology(t *testing.T) { - t.Run("nfs topology is true", func(t *testing.T) { - topology := common.GetNfsTopology("10.0.0.0") - assert.Equal(t, topology, []*csi.Topology{{Segments: map[string]string{"csi-powerstore.dellemc.com/10.0.0.0-nfs": "true"}}}) - }) - - t.Run("nfs topology should not be false", func(t *testing.T) { - topology := common.GetNfsTopology("10.0.0.0") - assert.NotEqual(t, topology, []*csi.Topology{{Segments: map[string]string{"csi-powerstore.dellemc.com/10.0.0.0-nfs": "false"}}}) - }) -} - -func Test_contains(t *testing.T) { - type args struct { - slice []string - element string - } - tests := []struct { - name string - args args - want bool - }{ - {"elementPresent", args{slice: []string{"firstElement", "secondElement"}, element: "secondElement"}, true}, - {"elementNotPresent", args{slice: []string{"firstElement", "secondElement"}, element: "thirdElement"}, false}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := common.Contains(tt.args.slice, tt.args.element); got != tt.want { - t.Errorf("contains() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestExternalAccessAlreadyAdded(t *testing.T) { - type args struct { - export gopowerstore.NFSExport - externalAccess string - } - tests := []struct { - name string - args args - want bool - }{ - {"externalAccessPresentInRWHosts", args{export: gopowerstore.NFSExport{RWHosts: []string{"10.0.0.0/255.255.255.255"}}, externalAccess: "10.0.0.0"}, true}, - {"externalAccessNotPresentInRWHosts", args{export: gopowerstore.NFSExport{RWHosts: []string{"10.232.0.0/255.255.255.255"}}, externalAccess: "10.10.0.0"}, false}, - {"externalAccessPresentInROHosts", args{export: gopowerstore.NFSExport{ROHosts: []string{"10.0.0.0/255.255.255.255"}}, externalAccess: "10.0.0.0"}, true}, - {"externalAccessNotPresentInROHosts", args{export: gopowerstore.NFSExport{ROHosts: []string{"10.232.0.0/255.255.255.255"}}, externalAccess: "10.10.0.0"}, false}, - {"externalAccessPresentInRWRootHosts", args{export: gopowerstore.NFSExport{RWRootHosts: []string{"10.0.0.0/255.255.255.255"}}, externalAccess: "10.0.0.0"}, true}, - {"externalAccessNotPresentInRWRootHosts", args{export: gopowerstore.NFSExport{RWRootHosts: []string{"10.232.0.0/255.255.255.255"}}, externalAccess: "10.10.0.0"}, false}, - {"externalAccessPresentInRORootHosts", args{export: gopowerstore.NFSExport{RORootHosts: []string{"10.0.0.0/255.255.255.255"}}, externalAccess: "10.0.0.0"}, true}, - {"externalAccessNotPresentInRORootHosts", args{export: gopowerstore.NFSExport{RORootHosts: []string{"10.232.0.0/255.255.255.255"}}, externalAccess: "10.10.0.0"}, false}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := common.ExternalAccessAlreadyAdded(tt.args.export, tt.args.externalAccess); got != tt.want { - t.Errorf("ExternalAccessAlreadyAdded() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestParseCIDR(t *testing.T) { - type args struct { - externalAccessCIDR string - } - tests := []struct { - name string - args args - want string - wantErr bool - }{ - {"Valid IP with net mask", args{externalAccessCIDR: "10.232.58.2/16"}, "10.232.0.0/255.255.0.0", false}, - {"Valid IP without net mask", args{externalAccessCIDR: "10.232.58.2"}, "10.232.58.2/255.255.255.255", false}, - {"InValid IP without net mask", args{externalAccessCIDR: "10.232.58"}, "", true}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := common.ParseCIDR(tt.args.externalAccessCIDR) - if (err != nil) != tt.wantErr { - t.Errorf("ParseCIDR() error = %v, wantErr %v", err, tt.wantErr) - return - } - if got != tt.want { - t.Errorf("ParseCIDR() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestSetPollingFrequency(t *testing.T) { - type args struct { - ctx context.Context - } - tests := []struct { - name string - args args - want int64 - }{ - {"Setting environament variable", args{ctx: context.TODO()}, 100}, - {"Expecting default value to be set", args{ctx: context.TODO()}, 60}, - } - for i, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if i == 0 { - os.Setenv("X_CSI_PODMON_ARRAY_CONNECTIVITY_POLL_RATE", "100") - } - // need to import this function because the package name in this file is not common - // @TO-DO rename package name to common - if got := common.SetPollingFrequency(tt.args.ctx); got != tt.want { - t.Errorf("SetPollingFrequency() = %v, want %v", got, tt.want) - } - os.Unsetenv("X_CSI_PODMON_ARRAY_CONNECTIVITY_POLL_RATE") - }) - } -} - -func Test_setAPIPort(t *testing.T) { - type args struct { - ctx context.Context - } - tests := []struct { - name string - args args - }{ - {"Fetching port number from Environment variable", args{ctx: context.TODO()}}, - {"Fetching & setting default port number", args{ctx: context.TODO()}}, - } - - for i, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if i == 0 { - os.Setenv("X_CSI_PODMON_API_PORT", "8090") - common.SetAPIPort(tt.args.ctx) - if common.APIPort != ":8090" { - t.Errorf("setAPIPort() error, want 8090 port found %v", common.APIPort) - } - os.Unsetenv("X_CSI_PODMON_API_PORT") - } - common.SetAPIPort(tt.args.ctx) - if common.APIPort != ":8083" { - t.Errorf("setAPIPort() error, want 8083 port found %v", common.APIPort) - } - }) - } -} - -func TestRandomString(t *testing.T) { - type args struct { - len int - } - tests := []struct { - name string - args args - }{ - {"Generating some random string", args{len: 5}}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Since each byte in the slice is represented by two hex characters in the resulting string, the length of the string returned by the function will be len * 2. - if got := common.RandomString(tt.args.len); len(got) != 5*2 { - t.Errorf("RandomString() = %v, have len %d and want 5*2", got, len(got)) - } - }) - } -} - -func TestGetIPListWithMaskFromString(t *testing.T) { - type args struct { - input string - } - tests := []struct { - name string - args args - want string - wantErr bool - }{ - {"Valid IP without subnet mask, Test 1", args{input: "10.1.1.2"}, "10.1.1.2", false}, - {"Invalid IP without subnet maskTest 2", args{input: "10.256.1.2"}, "", true}, - {"Invalid IP with subnet mask, Test 3", args{input: "10.256.1.2/24"}, "", true}, - {"Valid IP with subnet mask, Test 4", args{input: "10.1.1.2/24"}, "10.1.1.2/255.255.255.0", false}, - {"Invalid IP with subnet maskTest 5", args{input: "10.256.1.2/24/25"}, "", true}, - {"Invalid IP with Invalid subnet mask, Test 6", args{input: "10.255.1.2/24/25"}, "", true}, - {"Invalid IP with Invalid subnet mask, Test 7", args{input: "10.255.1.2/38"}, "", true}, - {"Invalid IP with Invalid subnet mask, Test 8", args{input: "10.255.1.2/x"}, "", true}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := common.GetIPListWithMaskFromString(tt.args.input) - if (err != nil) != tt.wantErr { - t.Errorf("GetIPListWithMaskFromString() error = %v, wantErr %v", err, tt.wantErr) - return - } - if got != tt.want { - t.Errorf("GetIPListWithMaskFromString() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestGetIPListFromString(t *testing.T) { - type args struct { - input string - } - x := []string{} - x = nil - tests := []struct { - name string - args args - want []string - }{ - {"Valid IP, Test 1", args{input: "10.255.1.2"}, []string{"10.255.1.2"}}, - {"InValid IP, Test 2", args{input: "10.256.1.2"}, x}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := common.GetIPListFromString(tt.args.input); !reflect.DeepEqual(got, tt.want) { - t.Errorf("GetIPListFromString() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestReachableEndPoint(t *testing.T) { - type args struct { - endpoint string - } - tests := []struct { - name string - args args - want bool - }{ - {"Unreachable IP, ", args{endpoint: "10.255.1.2:100"}, false}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := common.ReachableEndPoint(tt.args.endpoint); got != tt.want { - t.Errorf("ReachableEndPoint() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/pkg/common/k8sutils/k8sutils.go b/pkg/common/k8sutils/k8sutils.go deleted file mode 100644 index 8f52f4a8..00000000 --- a/pkg/common/k8sutils/k8sutils.go +++ /dev/null @@ -1,110 +0,0 @@ -/* - Copyright (c) 2023 Dell Inc, or its subsidiaries. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package k8sutils - -import ( - "context" - - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" -) - -// NodeLabelsRetrieverInterface defines the methods for retrieving Kubernetes Node Labels -type NodeLabelsRetrieverInterface interface { - BuildConfigFromFlags(masterURL, kubeconfig string) (*rest.Config, error) - InClusterConfig() (*rest.Config, error) - NewForConfig(config *rest.Config) (*kubernetes.Clientset, error) - GetNodeLabels(ctx context.Context, k8sclientset *kubernetes.Clientset, kubeNodeName string) (map[string]string, error) -} - -// NodeLabelsRetrieverImpl provided the implementation for NodeLabelsRetrieverInterface -type NodeLabelsRetrieverImpl struct{} - -// NodeLabelsRetriever is the actual instance of NodeLabelsRetrieverInterface which is used to retrieve the node labels -var NodeLabelsRetriever NodeLabelsRetrieverInterface - -func init() { - NodeLabelsRetriever = new(NodeLabelsRetrieverImpl) -} - -// BuildConfigFromFlags is a method for building kubernetes client config -func (svc *NodeLabelsRetrieverImpl) BuildConfigFromFlags(masterURL, kubeconfig string) (*rest.Config, error) { - return clientcmd.BuildConfigFromFlags(masterURL, kubeconfig) -} - -// InClusterConfig returns a config object which uses the service account kubernetes gives to pods -func (svc *NodeLabelsRetrieverImpl) InClusterConfig() (*rest.Config, error) { - return rest.InClusterConfig() -} - -// NewForConfig creates a new Clientset for the given config -func (svc *NodeLabelsRetrieverImpl) NewForConfig(config *rest.Config) (*kubernetes.Clientset, error) { - return kubernetes.NewForConfig(config) -} - -// GetNodeLabels retrieves the kubernetes node object and returns its labels -func (svc *NodeLabelsRetrieverImpl) GetNodeLabels(ctx context.Context, k8sclientset *kubernetes.Clientset, kubeNodeName string) (map[string]string, error) { - if k8sclientset != nil { - node, err := k8sclientset.CoreV1().Nodes().Get(ctx, kubeNodeName, v1.GetOptions{}) - if err != nil { - return nil, err - } - - return node.Labels, nil - } - - return nil, nil -} - -// CreateKubeClientSet creates and returns kubeclient set -func CreateKubeClientSet(kubeconfig string) (*kubernetes.Clientset, error) { - var clientset *kubernetes.Clientset - if kubeconfig != "" { - config, err := NodeLabelsRetriever.BuildConfigFromFlags("", kubeconfig) - if err != nil { - return nil, err - } - // create the clientset - clientset, err = NodeLabelsRetriever.NewForConfig(config) - if err != nil { - return nil, err - } - } else { - config, err := NodeLabelsRetriever.InClusterConfig() - if err != nil { - return nil, err - } - // creates the clientset - clientset, err = NodeLabelsRetriever.NewForConfig(config) - if err != nil { - return nil, err - } - } - return clientset, nil -} - -// GetNodeLabels returns labels present in the k8s node -func GetNodeLabels(ctx context.Context, kubeConfigPath string, kubeNodeName string) (map[string]string, error) { - k8sclientset, err := CreateKubeClientSet(kubeConfigPath) - if err != nil { - return nil, err - } - - return NodeLabelsRetriever.GetNodeLabels(ctx, k8sclientset, kubeNodeName) -} diff --git a/pkg/controller/base.go b/pkg/controller/base.go index e9687cf7..63169fbe 100644 --- a/pkg/controller/base.go +++ b/pkg/controller/base.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2021-2023 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2021-2024 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,12 +23,12 @@ import ( "strings" "unicode/utf8" - "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/dell/csi-powerstore/v2/pkg/common" + "github.com/dell/csi-powerstore/v2/pkg/identifiers" "github.com/dell/gopowerstore" - "github.com/golang/protobuf/ptypes" + "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/timestamppb" ) const ( @@ -42,6 +42,8 @@ const ( VolumeSizeMultiple = 8192 // MaxVolumeNameLength max length for the volume name MaxVolumeNameLength = 128 + // ReplicationPrefix represents replication prefix + ReplicationPrefix = "replication.storage.dell.com" // ErrUnknownAccessType represents error message for unknown access type ErrUnknownAccessType = "unknown access type is not Block or Mount" // ErrUnknownAccessMode represents error message for unknown access mode @@ -54,6 +56,8 @@ const ( KeyFsTypeOld = "FsType" // KeyReplicationEnabled represents key for replication enabled KeyReplicationEnabled = "isReplicationEnabled" + // KeyReplicationMode represents key for replication mode + KeyReplicationMode = "mode" // KeyReplicationRPO represents key for replication RPO KeyReplicationRPO = "rpo" // KeyReplicationRemoteSystem represents key for replication remote system @@ -117,7 +121,7 @@ func getCSISnapshot(snapshotID string, sourceVolumeID string, sizeInBytes int64) SizeBytes: sizeInBytes, SnapshotId: snapshotID, SourceVolumeId: sourceVolumeID, - CreationTime: ptypes.TimestampNow(), + CreationTime: timestamppb.Now(), ReadyToUse: true, } return snap @@ -175,7 +179,7 @@ func checkValidAccessTypes(vcs []*csi.VolumeCapability) bool { } func getDescription(params map[string]string) string { - if description, ok := params[common.KeyVolumeDescription]; ok { + if description, ok := params[identifiers.KeyVolumeDescription]; ok { return description } return params[KeyCSIPVCName] + "-" + params[KeyCSIPVCNamespace] diff --git a/pkg/controller/base_test.go b/pkg/controller/base_test.go index 9c7ad044..2ea4b7b4 100644 --- a/pkg/controller/base_test.go +++ b/pkg/controller/base_test.go @@ -21,6 +21,10 @@ package controller import ( "context" "errors" + "net/http" + "strings" + "testing" + "github.com/dell/gopowerstore" "github.com/dell/gopowerstore/api" "github.com/dell/gopowerstore/mocks" @@ -28,9 +32,6 @@ import ( "github.com/stretchr/testify/mock" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "net/http" - "strings" - "testing" ) func TestVolumeName(t *testing.T) { @@ -40,8 +41,10 @@ func TestVolumeName(t *testing.T) { }{ {"IsOkName", nil}, {"", status.Errorf(codes.InvalidArgument, "name cannot be empty")}, - {strings.Repeat("N", MaxVolumeNameLength+1), - status.Errorf(codes.InvalidArgument, "name must contain %d or fewer printable Unicode characters", MaxVolumeNameLength)}, + { + strings.Repeat("N", MaxVolumeNameLength+1), + status.Errorf(codes.InvalidArgument, "name must contain %d or fewer printable Unicode characters", MaxVolumeNameLength), + }, } for _, test := range tests { @@ -154,5 +157,4 @@ func TestDetachVolumeFromHost(t *testing.T) { assert.Error(t, err) assert.Contains(t, err.Error(), "unexpected api error when detaching volume from host") }) - } diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 0a974e23..dfe29e81 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2021-2023 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2021-2026 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,25 +23,26 @@ import ( "context" "errors" "fmt" - "net/http" + "regexp" "sort" "strconv" "strings" "sync" - "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/dell/csi-powerstore/v2/core" "github.com/dell/csi-powerstore/v2/pkg/array" - "github.com/dell/csi-powerstore/v2/pkg/common" - "github.com/dell/csi-powerstore/v2/pkg/common/fs" + "github.com/dell/csi-powerstore/v2/pkg/identifiers" + "github.com/dell/csi-powerstore/v2/pkg/identifiers/fs" + "github.com/dell/csi-powerstore/v2/pkg/identifiers/k8sutils" + "github.com/dell/csmlog" commonext "github.com/dell/dell-csi-extensions/common" podmon "github.com/dell/dell-csi-extensions/podmon" csiext "github.com/dell/dell-csi-extensions/replication" vgsext "github.com/dell/dell-csi-extensions/volumeGroupSnapshot" csictx "github.com/dell/gocsi/context" "github.com/dell/gopowerstore" - "github.com/golang/protobuf/ptypes/wrappers" - log "github.com/sirupsen/logrus" + "github.com/dell/gopowerstore/api" + "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/golang/protobuf/proto" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -62,8 +63,9 @@ type Interface interface { type Service struct { Fs fs.Interface - externalAccess string - nfsAcls string + externalAccess string + exclusiveAccess bool + nfsAcls string array.Locker @@ -71,59 +73,79 @@ type Service struct { replicationPrefix string isHealthMonitorEnabled bool isAutoRoundOffFsSizeEnabled bool - - K8sVisibilityAutoRegistration bool + IsCSMDREnabled bool } // maxVolumesSizeForArray - store the maxVolumesSizeForArray var maxVolumesSizeForArray = make(map[string]int64) +// function variables for mocking during testing of metro replication and deferrals related to metro replication +var ( + isNodeConnectedToArrayFunc = isNodeConnectedToArray + unpublishVolumeFunc = unpublishVolume + checkMetroStateFunc = array.CheckMetroState + createOrUpdateJournalEntryFunc = array.CreateOrUpdateJournalEntry +) + var mutex = &sync.Mutex{} +// Instantiate csmlog at package level +var log = csmlog.GetLogger() + // Init is a method that initializes internal variables of controller service func (s *Service) Init() error { ctx := context.Background() - if nat, ok := csictx.LookupEnv(ctx, common.EnvExternalAccess); ok { + kubeConfigPath, _ := csictx.LookupEnv(ctx, identifiers.EnvKubeConfigPath) + _, err := k8sutils.CreateKubeClientSet(kubeConfigPath) + if err != nil { + return fmt.Errorf("failed to create Kubernetes client: %s", err.Error()) + } + + if nat, ok := csictx.LookupEnv(ctx, identifiers.EnvExternalAccess); ok { s.externalAccess = nat } - if replicationContextPrefix, ok := csictx.LookupEnv(ctx, common.EnvReplicationContextPrefix); ok { + if exclusive, ok := csictx.LookupEnv(ctx, identifiers.EnvExclusiveAccess); ok { + // Only enable exclusive access if external access is configured + if s.externalAccess != "" { + s.exclusiveAccess = strings.EqualFold(exclusive, "true") + } + } + + if replicationContextPrefix, ok := csictx.LookupEnv(ctx, identifiers.EnvReplicationContextPrefix); ok { s.replicationContextPrefix = replicationContextPrefix + "/" } - if replicationPrefix, ok := csictx.LookupEnv(ctx, common.EnvReplicationPrefix); ok { + if replicationPrefix, ok := csictx.LookupEnv(ctx, identifiers.EnvReplicationPrefix); ok { s.replicationPrefix = replicationPrefix } - if isHealthMonitorEnabled, ok := csictx.LookupEnv(ctx, common.EnvIsHealthMonitorEnabled); ok { + if isHealthMonitorEnabled, ok := csictx.LookupEnv(ctx, identifiers.EnvIsHealthMonitorEnabled); ok { s.isHealthMonitorEnabled, _ = strconv.ParseBool(isHealthMonitorEnabled) } s.nfsAcls = "" - if nfsAcls, ok := csictx.LookupEnv(ctx, common.EnvNfsAcls); ok { + if nfsAcls, ok := csictx.LookupEnv(ctx, identifiers.EnvNfsAcls); ok { if nfsAcls != "" { s.nfsAcls = nfsAcls } } - if isAutoRoundOffFsSizeEnabled, ok := csictx.LookupEnv(ctx, common.EnvAllowAutoRoundOffFilesystemSize); ok { + if isAutoRoundOffFsSizeEnabled, ok := csictx.LookupEnv(ctx, identifiers.EnvAllowAutoRoundOffFilesystemSize); ok { log.Warn("Auto round off Filesystem size has been enabled! This will round off NFS PVC size to 3Gi when the requested size is less than 3Gi.") s.isAutoRoundOffFsSizeEnabled, _ = strconv.ParseBool(isAutoRoundOffFsSizeEnabled) } - if isk8sVisibilityAutoRegistrationEnabled, ok := csictx.LookupEnv(ctx, common.EnvK8sVisibilityAutoRegistration); ok { - s.K8sVisibilityAutoRegistration, _ = strconv.ParseBool(isk8sVisibilityAutoRegistrationEnabled) - } - return nil } // CreateVolume creates either FileSystem or Volume on storage array. func (s *Service) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { + log := log.WithContext(ctx) params := req.GetParameters() // Get array from map - arrayID, ok := params[common.KeyArrayID] + arrayID, ok := params[identifiers.KeyArrayID] var arr *array.PowerStoreArray // If no ArrayID was provided in storage class we just use default array @@ -145,6 +167,13 @@ func (s *Service) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest fsType := req.VolumeCapabilities[0].GetMount().GetFsType() useNFS = fsType == "nfs" + // If capability does not have NFS, check if params request NFS + // This can happen when running csi-sanity tests + if !useNFS && params[KeyFsType] == "nfs" { + log.Infof("Request's volume capability does not specify NFS, but params do, using NFS") + useNFS = true + } + if req.VolumeCapabilities[0].GetBlock() != nil { // We need to check if user requests raw block access from nfs and prevent that fsType, ok := params[KeyFsType] @@ -161,30 +190,53 @@ func (s *Service) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest // Prevent user from creating an NFS volume with incorrect topology(e.g. iscsi, nvme). At least one entry for nfs should be present in the topology, otherwise return an error if useNFS && req.AccessibilityRequirements != nil { - if ok := common.HasRequiredTopology(req.AccessibilityRequirements.Preferred, arr.GetIP(), "nfs"); !ok { - return nil, status.Errorf(codes.InvalidArgument, "invalid topology requested for NFS Volume. Please validate your storage class has nfs topology.") + if ok := identifiers.HasRequiredTopology(req.AccessibilityRequirements.Preferred, arr.GetIP(), "nfs"); !ok { + // if not in preferred, try requisite next + if ok := identifiers.HasRequiredTopology(req.AccessibilityRequirements.Requisite, arr.GetIP(), "nfs"); !ok { + return nil, status.Errorf(codes.InvalidArgument, "invalid topology requested for NFS Volume. Please validate your storage class has nfs topology.") + } } } var creator VolumeCreator var protocol string + var selectedNasName string nfsAcls := s.nfsAcls if useNFS { protocol = "nfs" nasParamsName, ok := params[KeyNasName] if ok { - creator = &NfsCreator{ - nasName: nasParamsName, + if strings.Contains(nasParamsName, ",") { + // Comma-separated NAS names + rawNasList := strings.Split(nasParamsName, ",") + nasList := make([]string, 0, len(rawNasList)) + for _, nas := range rawNasList { + trimmed := strings.TrimSpace(nas) + if trimmed != "" { + nasList = append(nasList, trimmed) + } + } + leastUsedNas, err := array.GetLeastUsedActiveNAS(ctx, arr, nasList) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to get least used NAS: %s", err) + } + selectedNasName = leastUsedNas + } else { + // Single NAS name + selectedNasName = nasParamsName } } else { - creator = &NfsCreator{ - nasName: arr.GetNasName(), - } + // No NAS name provided in params + selectedNasName = arr.GetNasName() } - if params[common.KeyNfsACL] != "" { - nfsAcls = params[common.KeyNfsACL] // Storage class takes precedence + creator = &NfsCreator{ + nasName: selectedNasName, + } + + if params[identifiers.KeyNfsACL] != "" { + nfsAcls = params[identifiers.KeyNfsACL] // Storage class takes precedence } else if arr.NfsAcls != "" { nfsAcls = arr.NfsAcls // Secrets next } @@ -207,26 +259,57 @@ func (s *Service) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest return nil, err } + replicationEnabled := params[s.WithRP(KeyReplicationEnabled)] + repMode := params[s.WithRP(KeyReplicationMode)] + // Default to ASYNC for backward compatibility + if repMode == "" { + repMode = identifiers.AsyncMode + } + repMode = strings.ToUpper(repMode) + contentSource := req.GetVolumeContentSource() if contentSource != nil { var volResp *csi.Volume var err error + // Configuring Metro is not allowed on clones or volumes created from Metro snapshot. + // So, fail the request if the requested volume is to be placed in Metro storage class. + // However, one can place the volume in a non-Metro storage class. + if replicationEnabled == "true" && repMode == identifiers.MetroMode { + return nil, status.Errorf(codes.InvalidArgument, + "Configuring Metro is not supported on clones or volumes created from Metro snapshot. Choose a non-Metro storage class.") + } + volumeSource := contentSource.GetVolume() if volumeSource != nil { - log.Printf("volume %s specified as volume content source", volumeSource.VolumeId) - parsedID, _, _, _ := array.ParseVolumeID(ctx, volumeSource.VolumeId, s.DefaultArray(), nil) - volumeSource.VolumeId = parsedID + log.Infof("volume %s specified as volume content source", volumeSource.VolumeId) + volumeHandle, parseVolErr := array.ParseVolumeID(ctx, volumeSource.VolumeId, s.DefaultArray(), nil) + if parseVolErr != nil { + if apiError, ok := parseVolErr.(gopowerstore.APIError); ok && apiError.NotFound() { + // Return error code csi-sanity test expects + log.Errorf("Volume source: %s not found", volumeSource.VolumeId) + return nil, status.Error(codes.NotFound, parseVolErr.Error()) + } + } + volumeSource.VolumeId = volumeHandle.LocalUUID volResp, err = creator.Clone(ctx, volumeSource, req.GetName(), sizeInBytes, req.Parameters, arr.GetClient()) } snapshotSource := contentSource.GetSnapshot() if snapshotSource != nil { - log.Printf("snapshot %s specified as volume content source", snapshotSource.SnapshotId) - parsedID, _, _, _ := array.ParseVolumeID(ctx, snapshotSource.SnapshotId, s.DefaultArray(), nil) - snapshotSource.SnapshotId = parsedID + log.Infof("snapshot %s specified as volume content source", snapshotSource.SnapshotId) + volumeHandle, parseVolErr := array.ParseVolumeID(ctx, snapshotSource.SnapshotId, s.DefaultArray(), nil) + if parseVolErr != nil { + if apiError, ok := parseVolErr.(gopowerstore.APIError); ok && apiError.NotFound() { + // Return error code csi-sanity test expects + log.Errorf("Snapshot source: %s not found", snapshotSource.SnapshotId) + return nil, status.Error(codes.NotFound, parseVolErr.Error()) + } + } + snapshotSource.SnapshotId = volumeHandle.LocalUUID volResp, err = creator.CreateVolumeFromSnapshot(ctx, snapshotSource, req.GetName(), sizeInBytes, req.Parameters, arr.GetClient()) } if err != nil { + log.Warnf("Failed to create volume: %s from snapshot: %s", req.GetName(), err.Error()) resp, err := creator.CheckIfAlreadyExists(ctx, req.GetName(), sizeInBytes, arr.GetClient()) if err != nil { return nil, err @@ -243,7 +326,7 @@ func (s *Service) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest } volResp.VolumeId = volResp.VolumeId + "/" + arr.GetGlobalID() + "/" + protocol if useNFS { - topology = common.GetNfsTopology(arr.GetIP()) + topology = identifiers.GetNfsTopology(arr.GetIP()) log.Infof("Modified topology to nfs for %s", req.GetName()) } volResp.AccessibleTopology = topology @@ -253,124 +336,252 @@ func (s *Service) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest } var vg gopowerstore.VolumeGroup - + var remoteSystem gopowerstore.RemoteSystem + var remoteSystemName string + var vgName string + isMetroVolume := false // Check if replication is enabled - replicationEnabled := params[s.WithRP(KeyReplicationEnabled)] + if replicationEnabled == "true" { - if replicationEnabled == "true" && !useNFS { log.Info("Preparing volume replication") - vgPrefix, ok := params[s.WithRP(KeyReplicationVGPrefix)] + remoteSystemName, ok = params[s.WithRP(KeyReplicationRemoteSystem)] if !ok { - return nil, status.Errorf(codes.InvalidArgument, "replication enabled but no volume group prefix specified in storage class") - } - rpo, ok := params[s.WithRP(KeyReplicationRPO)] - if !ok { - return nil, status.Errorf(codes.InvalidArgument, "replication enabled but no RPO specified in storage class") - } - rpoEnum := gopowerstore.RPOEnum(rpo) - if err := rpoEnum.IsValid(); err != nil { - return nil, status.Errorf(codes.InvalidArgument, "invalid rpo value") - } - remoteSystemName, ok := params[s.WithRP(KeyReplicationRemoteSystem)] - if !ok { - return nil, status.Errorf(codes.InvalidArgument, "replication enabled but no remote system specified in storage class") + return nil, status.Error(codes.InvalidArgument, "replication enabled but no remote system specified in storage class") } - namespace := "" - if ignoreNS, ok := params[s.WithRP(KeyReplicationIgnoreNamespaces)]; ok && ignoreNS == "false" { - pvcNS, ok := params[KeyCSIPVCNamespace] - if ok { - namespace = pvcNS + "-" + switch repMode { + case identifiers.SyncMode, identifiers.AsyncMode: + // handle Sync and Async modes where protection policy with replication rule is applied on volume group + log.Infof("%s replication mode requested", repMode) + vgPrefix, ok := params[s.WithRP(KeyReplicationVGPrefix)] + if !ok { + return nil, status.Error(codes.InvalidArgument, "replication enabled but no volume group prefix specified in storage class") } - } - vgName := vgPrefix + "-" + namespace + remoteSystemName + "-" + rpo - if len(vgName) > 128 { - vgName = vgName[:128] - } + rpo, ok := params[s.WithRP(KeyReplicationRPO)] + if !ok { + // If Replication mode is ASYNC and there is no RPO specified, returning an error + if repMode == identifiers.AsyncMode { + return nil, status.Error(codes.InvalidArgument, "replication mode is ASYNC but no RPO specified in storage class") + } + // If Replication mode is SYNC and there is no RPO, defaulting the value to Zero + rpo = identifiers.Zero + } + rpoEnum := gopowerstore.RPOEnum(rpo) + if err := rpoEnum.IsValid(); err != nil { + return nil, status.Error(codes.InvalidArgument, "invalid RPO value") + } - vg, err = arr.Client.GetVolumeGroupByName(ctx, vgName) - if err != nil { - if apiError, ok := err.(gopowerstore.APIError); ok && apiError.NotFound() { - log.Infof("Volume group with name %s not found, creating it", vgName) + // Validating RPO to be non Zero when replication mode is ASYNC + if repMode == identifiers.AsyncMode && rpo == identifiers.Zero { + log.Errorf("RPO value for %s cannot be : %s", repMode, rpo) + return nil, status.Error(codes.InvalidArgument, "replication mode ASYNC requires RPO value to be non Zero") + } - // ensure protection policy exists - pp, err := EnsureProtectionPolicyExists(ctx, arr, vgName, remoteSystemName, rpoEnum) - if err != nil { - return nil, status.Errorf(codes.Internal, "can't ensure protection policy exists %s", err.Error()) + // Validating RPO to be Zero whe replication mode is SYNC + if repMode == identifiers.SyncMode && rpo != identifiers.Zero { + return nil, status.Error(codes.InvalidArgument, "replication mode SYNC requires RPO value to be Zero") + } + namespace := "" + if ignoreNS, ok := params[s.WithRP(KeyReplicationIgnoreNamespaces)]; ok && ignoreNS == "false" { + pvcNS, ok := params[KeyCSIPVCNamespace] + if ok { + namespace = pvcNS + "-" } + } - group, err := arr.Client.CreateVolumeGroup(ctx, &gopowerstore.VolumeGroupCreate{ - Name: vgName, - ProtectionPolicyID: pp, - }) + if useNFS { + vgName = vgPrefix + "-nfs-" + namespace + remoteSystemName + "-" + rpo + } else { + vgName = vgPrefix + "-" + namespace + remoteSystemName + "-" + rpo + } + if len(vgName) > 128 { + vgName = vgName[:128] + } + if !useNFS { + vg, err = arr.Client.GetVolumeGroupByName(ctx, vgName) if err != nil { - return nil, status.Errorf(codes.Internal, "can't create volume group: %s", err.Error()) - } + if apiError, ok := err.(gopowerstore.APIError); ok && apiError.NotFound() { + log.Infof("Volume group with name %s not found, creating it", vgName) - vg, err = arr.Client.GetVolumeGroup(ctx, group.ID) - if err != nil { - return nil, status.Errorf(codes.Internal, "can't query volume group by id %s : %s", group.ID, err.Error()) + // ensure protection policy exists + pp, err := EnsureProtectionPolicyExists(ctx, arr, vgName, remoteSystemName, rpoEnum) + if err != nil { + return nil, status.Errorf(codes.Internal, "can't ensure protection policy exists %s", err.Error()) + } + + group, err := arr.Client.CreateVolumeGroup(ctx, &gopowerstore.VolumeGroupCreate{ + Name: vgName, + ProtectionPolicyID: pp, + }) + if err != nil { + return nil, status.Errorf(codes.Internal, "can't create volume group: %s", err.Error()) + } + + vg, err = arr.Client.GetVolumeGroup(ctx, group.ID) + if err != nil { + return nil, status.Errorf(codes.Internal, "can't query volume group by id %s : %s", group.ID, err.Error()) + } + + } else { + return nil, status.Errorf(codes.Internal, "can't query volume group by name %s : %s", vgName, err.Error()) + } + } else { + // if Replication mode is SYNC, check if the VolumeGroup is write-order consistent + if repMode == identifiers.SyncMode { + if !vg.IsWriteOrderConsistent { + return nil, status.Errorf(codes.Internal, "can't apply protection policy with sync rule if volume group is not write-order consistent") + } + } + // group exists, check that protection policy applied + if vg.ProtectionPolicyID == "" { + pp, err := EnsureProtectionPolicyExists(ctx, arr, vgName, remoteSystemName, rpoEnum) + if err != nil { + return nil, status.Errorf(codes.Internal, "can't ensure protection policy exists %s", err.Error()) + } + policyUpdate := gopowerstore.VolumeGroupChangePolicy{ProtectionPolicyID: pp} + _, err = arr.Client.UpdateVolumeGroupProtectionPolicy(ctx, vg.ID, &policyUpdate) + if err != nil { + return nil, status.Errorf(codes.Internal, "can't update volume group policy %s", err.Error()) + } + } } + // Pass the VolumeGroup to the creator so it can create the new volume inside the vg + if c, ok := creator.(*SCSICreator); ok { + c.vg = &vg + } } else { - return nil, status.Errorf(codes.Internal, "can't query volume group by name %s : %s", vgName, err.Error()) - } - } else { - // group exists, check that protection policy applied - if vg.ProtectionPolicyID == "" { pp, err := EnsureProtectionPolicyExists(ctx, arr, vgName, remoteSystemName, rpoEnum) if err != nil { return nil, status.Errorf(codes.Internal, "can't ensure protection policy exists %s", err.Error()) } - policyUpdate := gopowerstore.VolumeGroupChangePolicy{ProtectionPolicyID: pp} - _, err = arr.Client.UpdateVolumeGroupProtectionPolicy(ctx, vg.ID, &policyUpdate) + log.Infof("Protection policy %s verified or created successfully.", pp) + err = arr.Client.ModifyNASByName(ctx, &gopowerstore.NASModify{ProtectionPolicyID: pp}, selectedNasName) if err != nil { - return nil, status.Errorf(codes.Internal, "can't update volume group policy %s", err.Error()) + return nil, status.Errorf(codes.Internal, "can't update NAS protection policy %s", err.Error()) } + + log.Infof("Protection policy %s applied to NAS server %s", pp, remoteSystemName) + + } + case identifiers.MetroMode: + // handle Metro mode where metro is configured directly on the volume + // Note: Metro on volume group support is not added + log.Info("Metro replication mode requested") + + // Get specified remote system object for its ID + remoteSystem, err = arr.Client.GetRemoteSystemByName(ctx, remoteSystemName) + if err != nil { + return nil, status.Errorf(codes.Internal, "can't query remote system by name: %s", err.Error()) } - } - if c, ok := creator.(*SCSICreator); ok { - c.vg = &vg + isMetroVolume = true // set to true + default: + return nil, status.Errorf(codes.InvalidArgument, "replication enabled but invalid replication mode specified in storage class") } + } - params[common.KeyVolumeDescription] = getDescription(req.GetParameters()) + params[identifiers.KeyVolumeDescription] = getDescription(req.GetParameters()) var volumeResponse *csi.Volume - resp, err := creator.Create(ctx, req, sizeInBytes, arr.GetClient()) + + // check if job is already in progress on array, if so, return error and let CO check again + if useNFS { + jobs, err := arr.Client.GetInProgressJobsByFsName(ctx, req.GetName()) + if err != nil { + log.Errorf("Error getting jobs that are in progress for FileSystem: %s error: %s", req.Name, err.Error()) + return nil, status.Errorf(codes.Internal, "Error getting jobs that are in progress for FileSystem: %s error: %s", req.Name, err.Error()) + } + if len(jobs) > 0 { + log.Infof("Job already in progress to create FileSystem %s", req.GetName()) + return nil, status.Errorf(codes.AlreadyExists, "Job already in progress to create FileSystem %s", req.GetName()) + } + } + + // check if vol exists before creating it in the array + volumeResponse, err = creator.CheckIfAlreadyExists(ctx, req.GetName(), sizeInBytes, arr.GetClient()) if err != nil { - if apiError, ok := err.(gopowerstore.APIError); ok && (apiError.VolumeNameIsAlreadyUse() || apiError.FSNameIsAlreadyUse()) { - volumeResponse, err = creator.CheckIfAlreadyExists(ctx, req.GetName(), sizeInBytes, arr.GetClient()) - if err != nil { - return nil, err + // internal means something went wrong trying to check the volume and request needs to be retried + if status.Code(err) == codes.Internal || status.Code(err) == codes.AlreadyExists { + log.Warnf("CheckIfAlreadyExists returned error: %s for vol: %s", err.Error(), req.GetName()) + return nil, err + } + } + + if isMetroVolume && !s.IsCSMDREnabled { + log.Info("Failed to create metro volume due to CSM DR not available.") + return nil, status.Error(codes.InvalidArgument, "Metro replication mode requires CSM-DR to be enabled") + } + + if volumeResponse == nil { + resp, createError := creator.Create(ctx, req, sizeInBytes, arr.GetClient()) + if createError != nil { + log.Warnf("create volume for %s failed: '%s'", req.GetName(), createError.Error()) + if useNFS { + arr.NASCooldownTracker.MarkFailure(selectedNasName) + return nil, status.Error(codes.ResourceExhausted, createError.Error()) } - } else { - return nil, status.Error(codes.Internal, err.Error()) + return nil, createError + } + if useNFS { + arr.NASCooldownTracker.ResetFailure(selectedNasName) } - } else { volumeResponse = getCSIVolume(resp.ID, sizeInBytes) } - // Fetch the service tag - var serviceTag = GetServiceTag(ctx, req, arr, resp.ID, protocol) + metroVolumeIDSuffix := "" + if isMetroVolume { + // Configure Metro on volume + volID := volumeResponse.VolumeId + log.Infof("Configuring Metro on volume %s", volID) + + metroSession, err := arr.GetClient().ConfigureMetroVolume(ctx, volID, &gopowerstore.MetroConfig{ + RemoteSystemID: remoteSystem.ID, + }) + if err != nil { + if apiError, ok := err.(gopowerstore.APIError); ok && apiError.ReplicationSessionAlreadyCreated() { // idempotency check + log.Debugf("Metro has already been configured on volume %s", volID) + } else { + return nil, status.Errorf(codes.Internal, "can't configure metro on volume: %s", err.Error()) + } + } else { + log.Infof("Metro Session %s created for volume %s", metroSession.ID, volID) + } + + // Get the remote volume ID from the replication session. + replicationSession, err := arr.GetClient().GetReplicationSessionByLocalResourceID(ctx, volID) + if err != nil { + return nil, status.Errorf(codes.Internal, "could not get metro replication session: %s", err.Error()) + } + // Confirm the replication session is of the 'volume' type + if strings.ToLower(replicationSession.ResourceType) != "volume" { + return nil, status.Errorf(codes.FailedPrecondition, "replication session %s has a resource type %s, wanted type 'volume'", + replicationSession.ID, replicationSession.ResourceType) + } + // Build the metro volume handle suffix + metroVolumeIDSuffix = ":" + replicationSession.RemoteResourceID + "/" + remoteSystem.SerialNumber + } + // Fetch the service tag + serviceTag := GetServiceTag(ctx, req, arr, volumeResponse.VolumeId, protocol) volumeResponse.VolumeContext = req.Parameters - volumeResponse.VolumeContext[common.KeyArrayID] = arr.GetGlobalID() - volumeResponse.VolumeContext[common.KeyArrayVolumeName] = req.Name - volumeResponse.VolumeContext[common.KeyProtocol] = protocol - volumeResponse.VolumeContext[common.KeyServiceTag] = serviceTag + volumeResponse.VolumeContext[identifiers.KeyArrayID] = arr.GetGlobalID() + volumeResponse.VolumeContext[identifiers.KeyArrayVolumeName] = req.Name + volumeResponse.VolumeContext[identifiers.KeyProtocol] = protocol + volumeResponse.VolumeContext[identifiers.KeyServiceTag] = serviceTag if useNFS { - volumeResponse.VolumeContext[common.KeyNfsACL] = nfsAcls - volumeResponse.VolumeContext[common.KeyNasName] = arr.GetNasName() - topology = common.GetNfsTopology(arr.GetIP()) + volumeResponse.VolumeContext[identifiers.KeyNfsACL] = nfsAcls + volumeResponse.VolumeContext[identifiers.KeyNasName] = creator.(*NfsCreator).nasName + topology = identifiers.GetNfsTopology(arr.GetIP()) log.Infof("Modified topology to nfs for %s", req.GetName()) } - volumeResponse.VolumeId = volumeResponse.VolumeId + "/" + arr.GetGlobalID() + "/" + protocol + volumeResponse.VolumeId = volumeResponse.VolumeId + "/" + arr.GetGlobalID() + "/" + protocol + metroVolumeIDSuffix + volumeResponse.AccessibleTopology = topology return &csi.CreateVolumeResponse{ Volume: volumeResponse, @@ -379,12 +590,13 @@ func (s *Service) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest // DeleteVolume deletes either FileSystem or Volume from storage array. func (s *Service) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) { + log := log.WithContext(ctx) id := req.GetVolumeId() if id == "" { return nil, status.Error(codes.InvalidArgument, "volume ID is required") } - id, arrayID, protocol, err := array.ParseVolumeID(ctx, id, s.DefaultArray(), nil) + volumeHandle, err := array.ParseVolumeID(ctx, id, s.DefaultArray(), nil) if err != nil { if apiError, ok := err.(gopowerstore.APIError); ok && apiError.NotFound() { return &csi.DeleteVolumeResponse{}, nil @@ -392,6 +604,10 @@ func (s *Service) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest return nil, err } + id = volumeHandle.LocalUUID + arrayID := volumeHandle.LocalArrayGlobalID + protocol := volumeHandle.Protocol + arr, ok := s.Arrays()[arrayID] if !ok { return nil, status.Errorf(codes.Internal, "can't find array with provided id %s", arrayID) @@ -418,9 +634,9 @@ func (s *Service) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest // if one entry is there for RWRootHosts or RWHosts, check if this is the same externalAccess defined in value.yaml // if yes modifyNFSExport and remove externalAccess from the HostAcceesList on the array if (len(nfsExportResp.RWRootHosts) == 1 || len(nfsExportResp.RWHosts) == 1) && s.externalAccess != "" { - externalAccess, err := common.ParseCIDR(s.externalAccess) + externalAccess, err := identifiers.ParseCIDR(s.externalAccess) if err != nil { - log.Debug("error occurred while parsing externalAccess: ", err.Error(), s.externalAccess) + log.Debugf("error occurred %s while parsing externalAccess: %s ", err.Error(), s.externalAccess) return nil, status.Errorf(codes.FailedPrecondition, "filesystem %s cannot be deleted as it has associated NFS or SMB shares.", id) @@ -430,12 +646,12 @@ func (s *Service) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest var modifyHostPayload gopowerstore.NFSExportModify // Removing externalAccess from RWHosts as well as RWRootHosts if len(nfsExportResp.RWRootHosts) == 1 && externalAccess == nfsExportResp.RWRootHosts[0] { - log.Debug("Trying to remove externalAccess IP with mask having RWRootHosts access while deleting the volume: ", externalAccess) + log.Debugf("Trying to remove externalAccess IP with mask having RWRootHosts access while deleting the volume: %s", externalAccess) modifyNFSExport = true modifyHostPayload.RemoveRWRootHosts = []string{externalAccess} } if len(nfsExportResp.RWHosts) == 1 && externalAccess == nfsExportResp.RWHosts[0] { - log.Debug("Trying to remove externalAccess IP with mask having RWHosts access while deleting the volume: ", externalAccess) + log.Debugf("Trying to remove externalAccess IP with mask having RWHosts access while deleting the volume: %s", externalAccess) modifyNFSExport = true modifyHostPayload.RemoveRWHosts = []string{externalAccess} } @@ -443,7 +659,7 @@ func (s *Service) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest if modifyNFSExport { _, err = arr.GetClient().ModifyNFSExport(ctx, &modifyHostPayload, nfsExportResp.ID) if err != nil { - log.Debug("failure when removing externalAccess from nfs export: ", err.Error()) + log.Debugf("failure when removing externalAccess from nfs export: %s", err.Error()) if apiError, ok := err.(gopowerstore.APIError); !(ok && apiError.HostAlreadyRemovedFromNFSExport()) { return nil, status.Errorf(codes.FailedPrecondition, "filesystem %s cannot be deleted as it has associated NFS or SMB shares.", @@ -474,77 +690,190 @@ func (s *Service) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest return nil, err } else if protocol == "scsi" { - // query volume groups? - vgs, err := arr.GetClient().GetVolumeGroupsByVolumeID(ctx, id) - if err != nil { - if apiError, ok := err.(gopowerstore.APIError); !ok || !apiError.NotFound() { - return nil, err + localDeleted := false + remoteDeleted := false + + var metroResp *array.MetroFracturedResponse + var remoteArray *array.PowerStoreArray + remoteVolumeID := volumeHandle.RemoteUUID + remoteArrayID := volumeHandle.RemoteArrayGlobalID + if volumeHandle.IsMetro() { + if arr, ok := s.Arrays()[remoteArrayID]; ok { + remoteArray = arr + } else { + return nil, status.Errorf(codes.InvalidArgument, "failed to find remote array with ID %s", remoteArrayID) } - } - - if len(vgs.VolumeGroup) != 0 { - // Remove volume from volume group - // TODO: If volume has multiple volume group then how we should find ours? - // TODO: Maybe adding volumegroup id/name to volume id can help? - _, err := arr.GetClient().RemoveMembersFromVolumeGroup(ctx, &gopowerstore.VolumeGroupMembers{VolumeIds: []string{id}}, vgs.VolumeGroup[0].ID) + metroResp, _, err = array.CheckMetroState(ctx, volumeHandle, arr.GetClient(), remoteArray.GetClient()) if err != nil { - // TODO: check for idempotency cases + if err, ok := err.(gopowerstore.APIError); ok && err.NotFound() { + // both local and remote attempts to get the metro state failed with NotFound errors + // indicating the volume is already deleted + return &csi.DeleteVolumeResponse{}, nil + } return nil, err } + if metroResp.IsFractured { + log.Warnf("[METRO] metro volume %s is in a fractured state", req.GetVolumeId()) + } + } - // Unassign protection policy - _, err = arr.GetClient().ModifyVolume(ctx, &gopowerstore.VolumeModify{ProtectionPolicyID: ""}, id) - if err != nil { - return nil, err + // Delete local volume + err = deleteISCSIVolume(ctx, volumeHandle, arr, id) + if err == nil { + localDeleted = true + } else { + log.Errorf("failed to delete volume %s: %v", id, err) + } + + // non-metro or not fractured just return results + if !volumeHandle.IsMetro() || !metroResp.IsFractured { + if localDeleted { + return &csi.DeleteVolumeResponse{}, nil } + return nil, err } - // TODO: if len(vgs.VolumeGroup == 1) && it is the last volume : delete volume group - // TODO: What to do with RPO snaps? - // listSnaps, err := arr.GetClient().GetSnapshotsByVolumeID(ctx, id) - // if err != nil { - // return nil, status.Errorf(codes.Unknown, "failure getting snapshot: %s", err.Error()) - // } - // if len(listSnaps) > 0 { - // return nil, status.Errorf(codes.FailedPrecondition, - // "unable to delete volume -- snapshots based on this volume still exist: %v", - // listSnaps) - // } - _, err = arr.GetClient().DeleteVolume(ctx, nil, id) + // from here is metro and fractured array + err = deleteISCSIVolume(ctx, volumeHandle, remoteArray, remoteVolumeID) if err == nil { + remoteDeleted = true + } else { + log.Errorf("failed to delete remote volume %s: %v", id, err) + } + + if localDeleted && remoteDeleted { return &csi.DeleteVolumeResponse{}, nil } + // return error if anything not deleted and k8s will retry + return nil, err + } + + return nil, status.Errorf(codes.InvalidArgument, "can't figure out protocol") +} + +func deleteISCSIVolume(ctx context.Context, _ array.VolumeHandle, arr *array.PowerStoreArray, id string) error { + log := log.WithContext(ctx) + vgs, err := arr.GetClient().GetVolumeGroupsByVolumeID(ctx, id) + if err != nil { + if apiError, ok := err.(gopowerstore.APIError); !ok || !apiError.NotFound() { + return err + } + } + + if len(vgs.VolumeGroup) != 0 { + // Remove volume from volume group + // TODO: If volume has multiple volume group then how we should find ours? + // TODO: Maybe adding volumegroup id/name to volume id can help? + _, err := arr.GetClient().RemoveMembersFromVolumeGroup(ctx, &gopowerstore.VolumeGroupMembers{VolumeIDs: []string{id}}, vgs.VolumeGroup[0].ID) + if err != nil { + if apiError, ok := err.(gopowerstore.APIError); ok && apiError.VolumeAlreadyRemovedFromVolumeGroup() { // idempotency check + log.Debugf("Volume %s has already been removed from volume group %s", id, vgs.VolumeGroup[0].ID) // continue to delete volume + } else { + return status.Errorf(codes.Internal, "failed to remove volume %s from volume group: %s", id, err.Error()) + } + } + + // Unassign protection policy + _, err = arr.GetClient().ModifyVolume(ctx, &gopowerstore.VolumeModify{ProtectionPolicyID: ""}, id) + if err != nil { + return err + } + } + + volume, err := arr.GetClient().GetVolume(ctx, id) + if err != nil { + if apiError, ok := err.(gopowerstore.APIError); ok && apiError.NotFound() { + log.Infof("Volume %s not found, it may have been deleted.", id) + return nil + } + return status.Errorf(codes.Internal, "failure getting volume: %s", err.Error()) + } + + // TODO: if len(vgs.VolumeGroup == 1) && it is the last volume : delete volume group + // TODO: What to do with RPO snaps? + listSnaps, err := arr.GetClient().GetSnapshotsByVolumeID(ctx, id) + if err != nil { + if apiError, ok := err.(gopowerstore.APIError); !ok || !apiError.NotFound() { + return status.Errorf(codes.Unknown, "failure getting snapshot: %s", err.Error()) + } + } + + blockingDeleteSnapshotCount := 0 + for _, snap := range listSnaps { + // Note: There is no other way to check if it is a metro user snapshot from the response. + regex := regexp.MustCompile(array.MetroPrefixRegex + volume.Name) + if !regex.MatchString(snap.Name) { + log.Warnf("Snapshot detected for metro volume %s, delete this to finish cleanup: %s", volume.Name, snap.Name) + blockingDeleteSnapshotCount++ + } + } + + if blockingDeleteSnapshotCount > 0 { + return status.Errorf(codes.FailedPrecondition, + "unable to delete volume %s -- %d snapshots based on this volume still exist.", volume.Name, blockingDeleteSnapshotCount) + } + + // Check if volume has metro session and end it + if volume.MetroReplicationSessionID != "" { + _, err = arr.GetClient().EndMetroVolume(ctx, id, &gopowerstore.EndMetroVolumeOptions{ + DeleteRemoteVolume: true, // delete remote volume when deleting local volume + }) + if err != nil { + return status.Errorf(codes.Internal, "failure ending metro session on volume: %s", err.Error()) + } + } + + _, err = arr.GetClient().DeleteVolume(ctx, nil, id) + if err != nil { if apiError, ok := err.(gopowerstore.APIError); ok { if apiError.NotFound() { - return &csi.DeleteVolumeResponse{}, nil + return nil } if apiError.VolumeAttachedToHost() { - return nil, status.Errorf(codes.Internal, + return status.Errorf(codes.Internal, "volume with ID '%s' is still attached to host: %s", id, apiError.Error()) } } - return nil, err } - - return nil, status.Errorf(codes.InvalidArgument, "can't figure out protocol") + return err } // ControllerPublishVolume prepares Volume/FileSystem to be consumed by node by attaching/allowing access to the host. func (s *Service) ControllerPublishVolume(ctx context.Context, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) { + log := log.WithContext(ctx) id := req.GetVolumeId() + kubeNodeID := req.GetNodeId() + if id == "" { return nil, status.Error(codes.InvalidArgument, "volume ID is required") } - id, arrayID, protocol, err := array.ParseVolumeID(ctx, id, s.DefaultArray(), req.VolumeCapability) + if kubeNodeID == "" { + return nil, status.Error(codes.InvalidArgument, "node ID is required") + } + + volumeHandle, err := array.ParseVolumeID(ctx, id, s.DefaultArray(), req.VolumeCapability) if err != nil { - log.Info(err) + log.Error(err.Error()) return nil, err } + + id = volumeHandle.LocalUUID + arrayID := volumeHandle.LocalArrayGlobalID + protocol := volumeHandle.Protocol + remoteVolumeID := volumeHandle.RemoteUUID + remoteArrayID := volumeHandle.RemoteArrayGlobalID + arr, ok := s.Arrays()[arrayID] if !ok { - log.Info("ip is nil") - return nil, status.Error(codes.InvalidArgument, "failed to find array with given ID") + return nil, status.Errorf(codes.InvalidArgument, "failed to find array with ID %s", arrayID) + } + remoteArray := &array.PowerStoreArray{} + if volumeHandle.IsMetro() { + remoteArray, ok = s.Arrays()[remoteArrayID] + if !ok { + return nil, status.Errorf(codes.InvalidArgument, "failed to find remote array with ID %s", remoteArrayID) + } } vc := req.GetVolumeCapability() @@ -560,30 +889,126 @@ func (s *Service) ControllerPublishVolume(ctx context.Context, req *csi.Controll return nil, status.Error(codes.InvalidArgument, ErrUnknownAccessMode) } - kubeNodeID := req.GetNodeId() - if kubeNodeID == "" { - return nil, status.Error(codes.InvalidArgument, "node ID is required") - } - var publisher VolumePublisher - if protocol == "nfs" { publisher = &NfsPublisher{ - ExternalAccess: s.externalAccess, + ExternalAccess: s.externalAccess, + ExclusiveAccess: s.exclusiveAccess, } } else { publisher = &SCSIPublisher{} } - if err := publisher.CheckIfVolumeExists(ctx, arr.GetClient(), id); err != nil { - return nil, err + localDemoted := false + var metroResp *array.MetroFracturedResponse + isMetroFractured := false + + if remoteVolumeID != "" { + ctxLocal, cancelLocal := context.WithTimeout(context.Background(), array.MediumTimeout) + defer cancelLocal() + metroResp, localDemoted, err = array.CheckMetroState(ctxLocal, volumeHandle, arr.GetClient(), remoteArray.GetClient()) + if err != nil { + return nil, err + } + isMetroFractured = metroResp.IsFractured + if isMetroFractured { + log.Warnf("[METRO] metro volume %s is in a fractured state", req.GetVolumeId()) + } + if localDemoted { + log.Warnf("[METRO] metro volume %s has been demoted", req.GetVolumeId()) + } + } else { + if err := publisher.CheckIfVolumeExists(ctx, arr.GetClient(), id); err != nil { + return nil, err + } } - return publisher.Publish(ctx, req, arr.GetClient(), kubeNodeID, id) + publishContext := make(map[string]string) + publishVolumeResponse := &csi.ControllerPublishVolumeResponse{} + localPublished, remotePublished := false, false + + hostRegisteredLocalArray := arr.CheckConnectivity(ctx, kubeNodeID) + if hostRegisteredLocalArray { + log.Infof("Volume is being published on node %s for array %s", kubeNodeID, arr.Endpoint) + ctxLocal, cancelLocal := context.WithTimeout(context.Background(), array.MediumTimeout) + defer cancelLocal() + publishReponse, publishErr := publisher.Publish(ctxLocal, publishContext, req, arr.GetClient(), kubeNodeID, id, false) + if publishErr != nil { + if isMetroFractured && localDemoted { + log.Infof("[METRO] Could not publish volume %s on node %s for array %s due to Metro Session Fracture", id, kubeNodeID, arr.Endpoint) + } else { + log.Errorf("Failed to publish volume %s on node %s for array %s: %s", id, kubeNodeID, arr.Endpoint, publishErr) + return nil, publishErr + } + } else { + log.Infof("Local volume %s published", id) + publishVolumeResponse = publishReponse + localPublished = true + } + } else { + log.Infof("skipping volume publish on node %s for array %s, topology does not match", kubeNodeID, arr.Endpoint) + } + + hostRegisteredRemoteArray := false + if volumeHandle.IsMetro() { + if hostRegisteredRemoteArray = remoteArray.CheckConnectivity(ctx, kubeNodeID); hostRegisteredRemoteArray { + log.Infof("Volume is being published on node %s for remote array %s", kubeNodeID, remoteArray.Endpoint) + ctxRemote, cancelRemote := context.WithTimeout(context.Background(), array.MediumTimeout) + defer cancelRemote() + publishReponse, publishErr := publisher.Publish(ctxRemote, publishContext, req, remoteArray.GetClient(), kubeNodeID, remoteVolumeID, true) + if publishErr != nil { + if isMetroFractured && !localDemoted { + // localDemoted == false implies local is Promoted and Remote is Demoted. + log.Infof("[METRO] Could not publish volume %s on node %s for array %s due to Metro Session Fracture", remoteVolumeID, kubeNodeID, remoteArray.Endpoint) + } else { + // remote is Promoted + log.Errorf("Failed to publish volume %s on node %s for array %s: %s", id, kubeNodeID, remoteArray.Endpoint, publishErr) + return nil, publishErr + } + } else { + log.Infof("Remote volume %s published", remoteVolumeID) + remotePublished = true + publishVolumeResponse = publishReponse + } + } else { + log.Debugf("skipping volume publish on node %s for remote array %s, topology does not match", kubeNodeID, remoteArray.Endpoint) + } + } + + // at least one publish should succeed for non-metro, non-uniform metro, and uniform metro + // if a publish fails for uniform metro, the failed request will be deferred by adding to the volume journal + if !localPublished && !remotePublished { + return nil, status.Error(codes.Internal, "failed to publish volume") + } + + // Handling Uniform Metro Fracture cases. Other failures would have returned error before this point. + // Deferred operations are relevant only for Uniform metro in a fractured state and when only one side of metro volume was published + // and other side was not. + // Uniform metro is confirmed by a metro volume handle, and host connectivity from the kubeNodeID to both arrays. + if volumeHandle.IsMetro() && hostRegisteredLocalArray && hostRegisteredRemoteArray { + if (localPublished && !remotePublished) || (!localPublished && remotePublished) { + deferredRequest, err := proto.Marshal(req) + if err != nil { + log.Errorf("[METRO] Error marshalling req: %s", err.Error()) + } + deferredArrayID := arrayID + if !remotePublished { + deferredArrayID = remoteArrayID + } + + err = createOrUpdateJournalEntryFunc(ctx, metroResp.VolumeName, volumeHandle, deferredArrayID, kubeNodeID, "ControllerPublishVolume", deferredRequest) + if err != nil { + log.Errorf("Could not create journal entry for operation %s for volume %s node %s array %s", "ControllerPublishVolume", id, kubeNodeID, arr.Endpoint) + } + } + } + + return publishVolumeResponse, nil } // ControllerUnpublishVolume prepares Volume/FileSystem to be deleted by unattaching/disabling access to the host. func (s *Service) ControllerUnpublishVolume(ctx context.Context, req *csi.ControllerUnpublishVolumeRequest) (*csi.ControllerUnpublishVolumeResponse, error) { + log := log.WithContext(ctx) id := req.GetVolumeId() if id == "" { return nil, status.Error(codes.InvalidArgument, "volume ID is required") @@ -593,49 +1018,206 @@ func (s *Service) ControllerUnpublishVolume(ctx context.Context, req *csi.Contro if kubeNodeID == "" { return nil, status.Error(codes.InvalidArgument, "node ID is required") } - - id, arrayID, protocol, err := array.ParseVolumeID(ctx, id, s.DefaultArray(), nil) + volumeHandle, err := array.ParseVolumeID(ctx, id, s.DefaultArray(), nil) if err != nil { - if apiError, ok := err.(gopowerstore.APIError); ok && apiError.NotFound() { - return &csi.ControllerUnpublishVolumeResponse{}, nil - } - return nil, status.Errorf(codes.Unknown, - "failure checking volume status for volume unpublishing: %s", err.Error()) + log.Error(err.Error()) + return nil, err } + arrayID := volumeHandle.LocalArrayGlobalID + remoteArrayID := volumeHandle.RemoteArrayGlobalID arr, ok := s.Arrays()[arrayID] if !ok { return nil, status.Errorf(codes.InvalidArgument, "cannot find array %s", arrayID) } + var remoteArray *array.PowerStoreArray + isMetroFractured := false + localDemoted := false + metroResp := &array.MetroFracturedResponse{ + IsFractured: false, + } - if protocol == "scsi" { - node, err := arr.GetClient().GetHostByName(ctx, kubeNodeID) + if volumeHandle.IsMetro() { + remoteArray, ok = s.Arrays()[remoteArrayID] + if !ok { + return nil, status.Errorf(codes.InvalidArgument, "cannot find remote array %s", remoteArrayID) + } + + metroResp, localDemoted, err = checkMetroStateFunc(ctx, volumeHandle, arr.GetClient(), remoteArray.GetClient()) if err != nil { - if apiError, ok := err.(gopowerstore.APIError); ok && apiError.HostIsNotExist() { - // We need additional check here since we can just have host without ip in it - ipList := common.GetIPListFromString(kubeNodeID) - if ipList == nil { - return nil, errors.New("can't find IP in nodeID") + if apiError, ok := err.(gopowerstore.APIError); ok { + if !apiError.NotFound() { + return nil, err } - ip := ipList[len(ipList)-1] - nodeID := kubeNodeID[:len(kubeNodeID)-len(ip)-1] - node, err = arr.GetClient().GetHostByName(ctx, nodeID) - if err != nil { - return nil, status.Errorf(codes.NotFound, "host with k8s node ID '%s' not found", kubeNodeID) + + // Not found due to potentially deleted volume through UI. Still need to unpublish. + log.Infof("[Metro] Volume with ID %s not found", id) + } + } + + if metroResp != nil { + isMetroFractured = metroResp.IsFractured + } + + if isMetroFractured { + log.Warnf("[METRO] metro volume %s is in a fractured state", req.GetVolumeId()) + } + + if localDemoted { + log.Warnf("[METRO] metro volume %s has been demoted", req.GetVolumeId()) + } + } + + localVolumeUnpublished := false + remoteVolumeUnpublished := false + response := &csi.ControllerUnpublishVolumeResponse{} + + // Check if it is Metro volume and with newer secret configuation + nodeConnectedToLocalArray := isNodeConnectedToArrayFunc(ctx, kubeNodeID, arr) + if nodeConnectedToLocalArray { + log.Debugf("Volume is being unpublished on node %s for array %s", kubeNodeID, arr.Endpoint) + ctxLocal, cancelLocal := context.WithTimeout(context.Background(), array.MediumTimeout) + defer cancelLocal() + resp, unpublishErr := unpublishVolumeFunc(ctxLocal, kubeNodeID, arr, &volumeHandle, nil) + if unpublishErr != nil { + if isMetroFractured && localDemoted { + // expected failure if Metro is Fractured and local array is down + log.Infof("[METRO] Could not unpublish volume %s on node %s for array %s due to Metro Session Fracture", id, kubeNodeID, arr.Endpoint) + } else { + log.Errorf("Failed to unpublish volume %s for array %s: %s", id, arr.Endpoint, err) + return nil, unpublishErr + } + } else { + log.Infof("Unpublished volume %s for array %s", id, arr.Endpoint) + localVolumeUnpublished = true + response = resp + } + } + nodeConnectedToRemoteArray := false + if volumeHandle.IsMetro() { + nodeConnectedToRemoteArray = isNodeConnectedToArrayFunc(ctx, kubeNodeID, remoteArray) + if nodeConnectedToRemoteArray { + log.Debugf("Volume is being unpublished on node %s for remote array %s", kubeNodeID, remoteArray.Endpoint) + ctxRemote, cancelRemote := context.WithTimeout(context.Background(), array.MediumTimeout) + defer cancelRemote() + resp, unpublishErr := unpublishVolumeFunc(ctxRemote, kubeNodeID, nil, &volumeHandle, remoteArray) + if unpublishErr != nil { + if isMetroFractured && !localDemoted { + // expected failure if Metro is Fractured and remote array is down + log.Infof("[METRO] Could not unpublish volume %s on node %s for array %s due to Metro Session Fracture", id, kubeNodeID, remoteArray.Endpoint) + } else { + log.Errorf("Failed to unpublish volume %s for array %s: %s", id, remoteArray.Endpoint, err) + return nil, unpublishErr } } else { - return nil, status.Errorf(codes.Internal, - "failure checking host '%s' status for volume unpublishing: %s", kubeNodeID, err.Error()) + log.Infof("Unpublished volume %s for array %s", id, remoteArray.Endpoint) + remoteVolumeUnpublished = true + response = resp } } + } + if !localVolumeUnpublished && !remoteVolumeUnpublished { + return nil, status.Error(codes.Internal, "failed to unpublish volume") + } - err = detachVolumeFromHost(ctx, node.ID, id, arr.GetClient()) - if err != nil { - return nil, err + if volumeHandle.IsMetro() && nodeConnectedToLocalArray && nodeConnectedToRemoteArray { + if (localVolumeUnpublished && !remoteVolumeUnpublished) || (!localVolumeUnpublished && remoteVolumeUnpublished) { + deferredRequest, err := proto.Marshal(req) + if err != nil { + log.Errorf("[METRO] Error marshalling req: %s", err.Error()) + return nil, err + } + + deferredArrayID := arrayID + if !remoteVolumeUnpublished { + deferredArrayID = remoteArrayID + } + + err = createOrUpdateJournalEntryFunc(ctx, metroResp.VolumeName, volumeHandle, deferredArrayID, kubeNodeID, "ControllerUnpublishVolume", deferredRequest) + if err != nil { + log.Errorf("Could not create journal entry for operation %s for volume %s node %s array %s", "ControllerUnpublishVolume", id, kubeNodeID, arrayID) + return nil, err + } + + log.Infof("[METRO] Metro volume %s created journal entry for operation %s for volume %s node %s array %s", id, "ControllerUnpublishVolume", id, kubeNodeID, arrayID) + } + } + + return response, nil +} + +func isNodeConnectedToArray(ctx context.Context, kubeNodeID string, arr *array.PowerStoreArray) bool { + return arr.CheckConnectivity(ctx, kubeNodeID) +} + +// unpublishVolume removes the mount to the target path and unpublishes the volume +func unpublishVolume(ctx context.Context, kubeNodeID string, arr *array.PowerStoreArray, volumeHandle *array.VolumeHandle, remoteArray *array.PowerStoreArray) (*csi.ControllerUnpublishVolumeResponse, error) { + if arr == nil && remoteArray == nil { + return &csi.ControllerUnpublishVolumeResponse{}, errors.New("no array information for controller unpublish") + } + + id := volumeHandle.LocalUUID + protocol := volumeHandle.Protocol + remoteVolumeID := volumeHandle.RemoteUUID + + switch protocol { + case "scsi": + // unpublish can be for remote array only + if arr != nil { + _, err := arr.GetClient().GetVolume(ctx, id) + if err != nil { + if apiError, ok := err.(gopowerstore.APIError); ok && apiError.NotFound() { + return &csi.ControllerUnpublishVolumeResponse{}, nil + } + } + node, err := arr.GetClient().GetHostByName(ctx, kubeNodeID) + if err != nil { + if apiError, ok := err.(gopowerstore.APIError); ok && apiError.HostIsNotExist() { + // We need additional check here since we can just have host without ip in it + ipList := identifiers.GetIPListFromString(kubeNodeID) + if ipList == nil { + return nil, errors.New("can't find IP in nodeID") + } + ip := ipList[len(ipList)-1] + nodeID := kubeNodeID[:len(kubeNodeID)-len(ip)-1] + node, err = arr.GetClient().GetHostByName(ctx, nodeID) + if err != nil { + return nil, status.Errorf(codes.NotFound, "host with k8s node ID '%s' not found", kubeNodeID) + } + } else { + return nil, status.Errorf(codes.Internal, + "failure checking host '%s' status for volume unpublishing: %s", kubeNodeID, err.Error()) + } + } + + err = detachVolumeFromHost(ctx, node.ID, id, arr.GetClient()) + if err != nil { + return nil, err + } + log.Debugf("Volume is unpublished on node %s for array %s", kubeNodeID, arr.Endpoint) + } + + if remoteVolumeID != "" && remoteArray != nil { // For Remote Metro volume + _, err := remoteArray.GetClient().GetVolume(ctx, remoteVolumeID) + if err != nil { + if apiError, ok := err.(gopowerstore.APIError); ok && apiError.NotFound() { + return &csi.ControllerUnpublishVolumeResponse{}, nil + } + } + node, err := remoteArray.GetClient().GetHostByName(ctx, kubeNodeID) + if err != nil { + return nil, status.Errorf(codes.Internal, + "failure checking host '%s' status for volume unpublishing on remote array: %s", kubeNodeID, err.Error()) + } + err = detachVolumeFromHost(ctx, node.ID, remoteVolumeID, remoteArray.GetClient()) + if err != nil { + return nil, err + } + log.Debugf("Volume is unpublished on node %s for array %s", kubeNodeID, remoteArray.Endpoint) } return &csi.ControllerUnpublishVolumeResponse{}, nil - } else if protocol == "nfs" { + case "nfs": fs, err := arr.GetClient().GetFS(ctx, id) if err != nil { if apiError, ok := err.(gopowerstore.APIError); ok && apiError.NotFound() { @@ -645,7 +1227,7 @@ func (s *Service) ControllerUnpublishVolume(ctx context.Context, req *csi.Contro } // Parse volumeID to get an IP - ipList := common.GetIPListFromString(kubeNodeID) + ipList := identifiers.GetIPListFromString(kubeNodeID) if ipList == nil { return nil, errors.New("can't find IP in nodeID") } @@ -667,7 +1249,7 @@ func (s *Service) ControllerUnpublishVolume(ctx context.Context, req *csi.Contro if len(export.ROHosts) > 0 { if index >= 0 { modifyHostPayload.RemoveROHosts = []string{ip + "/255.255.255.255"} // we can't remove without netmask - log.Debug("Going to remove IP from ROHosts: ", modifyHostPayload.RemoveROHosts[0]) + log.Debugf("Going to remove IP from ROHosts: %s ", modifyHostPayload.RemoveROHosts[0]) } } @@ -676,24 +1258,24 @@ func (s *Service) ControllerUnpublishVolume(ctx context.Context, req *csi.Contro if len(export.RORootHosts) > 0 { if index >= 0 { modifyHostPayload.RemoveRORootHosts = []string{ip + "/255.255.255.255"} // we can't remove without netmask - log.Debug("Going to remove IP from RORootHosts: ", modifyHostPayload.RemoveRORootHosts[0]) + log.Debugf("Going to remove IP from RORootHosts: %s", modifyHostPayload.RemoveRORootHosts[0]) } } - if common.Contains(export.RWHosts, ip+"/255.255.255.255") { + if identifiers.Contains(export.RWHosts, ip+"/255.255.255.255") { modifyHostPayload.RemoveRWHosts = []string{ip + "/255.255.255.255"} // we can't remove without netmask - log.Debug("Going to remove IP from RWHosts: ", modifyHostPayload.RemoveRWHosts[0]) + log.Debugf("Going to remove IP from RWHosts: %s", modifyHostPayload.RemoveRWHosts[0]) } - if common.Contains(export.RWRootHosts, ip+"/255.255.255.255") { + if identifiers.Contains(export.RWRootHosts, ip+"/255.255.255.255") { modifyHostPayload.RemoveRWRootHosts = []string{ip + "/255.255.255.255"} // we can't remove without netmask - log.Debug("Going to remove IP from RWRootHosts: ", modifyHostPayload.RemoveRWRootHosts[0]) + log.Debugf("Going to remove IP from RWRootHosts: %s ", modifyHostPayload.RemoveRWRootHosts[0]) } // Detach host from nfs export _, err = arr.GetClient().ModifyNFSExport(ctx, &modifyHostPayload, export.ID) if err != nil { if apiError, ok := err.(gopowerstore.APIError); !(ok && apiError.HostAlreadyRemovedFromNFSExport()) { - log.Debug("Error occured while modifying NFS export during UnPublishVolume", err.Error()) + log.Debugf("Error occured while modifying NFS export during UnPublishVolume %s", err.Error()) return nil, status.Errorf(codes.Internal, "failure when removing new host to nfs export: %s", err.Error()) } @@ -713,48 +1295,49 @@ func GetServiceTag(ctx context.Context, req *csi.CreateVolumeRequest, arr *array var applianceName string var err error + log := log.WithContext(ctx) // Check if appliance id is present in PVC manifest if applianceID, ok := (req.Parameters)["appliance_id"]; ok { // Fetching appliance information using the appliance id ap, err = arr.Client.GetAppliance(ctx, applianceID) if err != nil { - log.Warn("Received error while calling GetAppliance ", err.Error()) + log.Warnf("Received error while calling GetAppliance %s", err.Error()) } } else { if protocol != "nfs" { vol, err = arr.Client.GetVolume(ctx, volID) if err != nil { - log.Warn("Received error while calling GetVolume ", err.Error()) + log.Warnf("Received error while calling GetVolume %s", err.Error()) } if vol.ApplianceID == "" { log.Warn("Unable to fetch ApplianceID from the volume") } else { ap, err = arr.Client.GetAppliance(ctx, vol.ApplianceID) if err != nil { - log.Warn("Received error while calling GetAppliance ", err.Error()) + log.Warnf("Received error while calling GetAppliance %s", err.Error()) } } } else { f, err = arr.Client.GetFS(ctx, volID) if err != nil { - log.Warn("Received error while calling GetFS ", err.Error()) + log.Warnf("Received error while calling GetFS %s", err.Error()) } if f.NasServerID == "" { log.Warn("Unable to fetch the NasServerID from the file system") } else { nas, err = arr.Client.GetNAS(ctx, f.NasServerID) if err != nil { - log.Warn("Received error while calling GetNAS ", err.Error()) + log.Warnf("Received error while calling GetNAS %s", err.Error()) } - if nas.CurrentNodeId == "" { + if nas.CurrentNodeID == "" { log.Warn("Unable to fetch the CurrentNodeId from the nas server") } else { // Removing "-node-X" from the end of CurrentNodeId to get Appliance Name - applianceName = strings.Split(nas.CurrentNodeId, "-node-")[0] + applianceName = strings.Split(nas.CurrentNodeID, "-node-")[0] // Fetching appliance information using the appliance name ap, err = arr.Client.GetApplianceByName(ctx, applianceName) if err != nil { - log.Warn("Received error while calling GetApplianceByName ", err.Error()) + log.Warnf("Received error while calling GetApplianceByName %s", err.Error()) } } } @@ -815,10 +1398,15 @@ func (s *Service) ValidateVolumeCapabilities(ctx context.Context, req *csi.Valid } // for sanity id := req.GetVolumeId() - id, arrayID, proto, err := array.ParseVolumeID(ctx, id, s.DefaultArray(), nil) + volumeHandle, err := array.ParseVolumeID(ctx, id, s.DefaultArray(), nil) if err != nil { return &csi.ValidateVolumeCapabilitiesResponse{}, status.Error(codes.NotFound, "No such volume") } + + id = volumeHandle.LocalUUID + arrayID := volumeHandle.LocalArrayGlobalID + proto := volumeHandle.Protocol + if proto == "nfs" { _, err := s.Arrays()[arrayID].Client.GetFS(ctx, id) if err != nil { @@ -870,19 +1458,11 @@ func (s *Service) ListVolumes(ctx context.Context, req *csi.ListVolumesRequest) } // Call the common listVolumes code - source, nextToken, err := s.listPowerStoreVolumes(ctx, startToken, maxEntries) + entries, nextToken, err := s.listPowerStoreVolumes(ctx, startToken, maxEntries) if err != nil { return nil, err } - // Process the source volumes and make CSI Volumes - entries := make([]*csi.ListVolumesResponse_Entry, len(source)) - for i, vol := range source { - entries[i] = &csi.ListVolumesResponse_Entry{ - Volume: getCSIVolume(vol.ID, vol.Size), - } - } - return &csi.ListVolumesResponse{ Entries: entries, NextToken: nextToken, @@ -894,7 +1474,7 @@ func (s *Service) GetCapacity(ctx context.Context, req *csi.GetCapacityRequest) params := req.GetParameters() // Get array from map - arrayID, ok := params[common.KeyArrayID] + arrayID, ok := params[identifiers.KeyArrayID] var arr *array.PowerStoreArray // If no ArrayIP was provided in storage class we just use default array @@ -924,11 +1504,12 @@ func (s *Service) GetCapacity(ctx context.Context, req *csi.GetCapacityRequest) } func getMaximumVolumeSize(ctx context.Context, arr *array.PowerStoreArray) int64 { + log := log.WithContext(ctx) valueInCache, found := getCachedMaximumVolumeSize(arr.GlobalID) if !found || valueInCache < 0 { defaultHeaders := arr.Client.GetCustomHTTPHeaders() if defaultHeaders == nil { - defaultHeaders = make(http.Header) + defaultHeaders = api.NewSafeHeader().GetHeader() } customHeaders := defaultHeaders customHeaders.Add("DELL-VISIBILITY", "internal") @@ -964,12 +1545,12 @@ func cacheMaximumVolumeSize(key string, value int64) { } // ControllerGetCapabilities returns list of capabilities that are supported by the driver. -func (s *Service) ControllerGetCapabilities(ctx context.Context, request *csi.ControllerGetCapabilitiesRequest) (*csi.ControllerGetCapabilitiesResponse, error) { - newCap := func(cap csi.ControllerServiceCapability_RPC_Type) *csi.ControllerServiceCapability { +func (s *Service) ControllerGetCapabilities(_ context.Context, _ *csi.ControllerGetCapabilitiesRequest) (*csi.ControllerGetCapabilitiesResponse, error) { + newCap := func(capability csi.ControllerServiceCapability_RPC_Type) *csi.ControllerServiceCapability { return &csi.ControllerServiceCapability{ Type: &csi.ControllerServiceCapability_Rpc{ Rpc: &csi.ControllerServiceCapability_RPC{ - Type: cap, + Type: capability, }, }, } @@ -992,6 +1573,7 @@ func (s *Service) ControllerGetCapabilities(ctx context.Context, request *csi.Co if s.isHealthMonitorEnabled { for _, capability := range []csi.ControllerServiceCapability_RPC_Type{ csi.ControllerServiceCapability_RPC_GET_VOLUME, + csi.ControllerServiceCapability_RPC_LIST_VOLUMES, csi.ControllerServiceCapability_RPC_LIST_VOLUMES_PUBLISHED_NODES, csi.ControllerServiceCapability_RPC_VOLUME_CONDITION, } { @@ -1017,11 +1599,15 @@ func (s *Service) CreateSnapshot(ctx context.Context, req *csi.CreateSnapshotReq return nil, status.Errorf(codes.InvalidArgument, "volume ID to be snapped is required") } - id, arrayID, protocol, err := array.ParseVolumeID(ctx, sourceVolID, s.DefaultArray(), nil) + volumeHandle, err := array.ParseVolumeID(ctx, sourceVolID, s.DefaultArray(), nil) if err != nil { return nil, err } + id := volumeHandle.LocalUUID + arrayID := volumeHandle.LocalArrayGlobalID + protocol := volumeHandle.Protocol + arr, ok := s.Arrays()[arrayID] if !ok { return nil, status.Error(codes.InvalidArgument, "failed to find array with given ID") @@ -1090,7 +1676,7 @@ func (s *Service) DeleteSnapshot(ctx context.Context, req *csi.DeleteSnapshotReq return nil, status.Errorf(codes.InvalidArgument, "snapshot ID to be deleted is required") } - id, arrayID, protocol, err := array.ParseVolumeID(ctx, snapID, s.DefaultArray(), nil) + volumeHandle, err := array.ParseVolumeID(ctx, snapID, s.DefaultArray(), nil) if err != nil { if apiError, ok := err.(gopowerstore.APIError); ok && apiError.NotFound() { return &csi.DeleteSnapshotResponse{}, nil @@ -1098,6 +1684,10 @@ func (s *Service) DeleteSnapshot(ctx context.Context, req *csi.DeleteSnapshotReq return nil, err } + id := volumeHandle.LocalUUID + arrayID := volumeHandle.LocalArrayGlobalID + protocol := volumeHandle.Protocol + arr, ok := s.Arrays()[arrayID] if !ok { return nil, status.Error(codes.InvalidArgument, "failed to find array with given ID") @@ -1201,36 +1791,79 @@ func (s *Service) ListSnapshots(ctx context.Context, req *csi.ListSnapshotsReque }, nil } +func GetMetroSessionState(ctx context.Context, metroSessionID string, arr *array.PowerStoreArray) (gopowerstore.RSStateEnum, error) { + metroSession, err := arr.Client.GetReplicationSessionByID(ctx, metroSessionID) + if err != nil { + return "", fmt.Errorf("could not get metro replication session %s: %w", metroSessionID, err) + } + return metroSession.State, nil +} + // ControllerExpandVolume resizes Volume or FileSystem by increasing available volume capacity in the storage array. func (s *Service) ControllerExpandVolume(ctx context.Context, req *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error) { - id, arrayID, protocol, err := array.ParseVolumeID(ctx, req.VolumeId, s.DefaultArray(), nil) + volumeHandle, err := array.ParseVolumeID(ctx, req.VolumeId, s.DefaultArray(), nil) if err != nil { return nil, status.Errorf(codes.OutOfRange, "unable to parse the volume id") } + id := volumeHandle.LocalUUID + arrayID := volumeHandle.LocalArrayGlobalID + protocol := volumeHandle.Protocol + remoteVolumeID := volumeHandle.RemoteUUID + requiredBytes := req.GetCapacityRange().GetRequiredBytes() if requiredBytes > MaxVolumeSizeBytes { return nil, status.Errorf(codes.OutOfRange, "volume exceeds allowed limit") } + array, ok := s.Arrays()[arrayID] + if !ok { + return nil, status.Errorf(codes.InvalidArgument, "unable to find array with ID %s", arrayID) + } + client := array.Client + if protocol == "scsi" { - vol, err := s.Arrays()[arrayID].Client.GetVolume(ctx, id) + vol, err := client.GetVolume(ctx, id) if err != nil { - return nil, status.Errorf(codes.OutOfRange, "detected SCSI protocol but wasn't able to fetch the volume info") + return nil, status.Error(codes.NotFound, "detected SCSI protocol but wasn't able to fetch the volume info") } + + isMetro := remoteVolumeID != "" + if isMetro && vol.MetroReplicationSessionID == "" { + return nil, status.Errorf(codes.Internal, + "failed to expand the volume %s because the metro replication session ID is empty for metro volume", vol.Name) + } + if vol.Size < requiredBytes { - _, err = s.Arrays()[arrayID].Client.ModifyVolume(context.Background(), &gopowerstore.VolumeModify{Size: requiredBytes}, id) + if isMetro { + // must pause metro session before modifying the volume + state, err := GetMetroSessionState(ctx, vol.MetroReplicationSessionID, array) + if err != nil { + return nil, status.Errorf(codes.Internal, + "failed to expand the volume %q: could not retrieve metro session state: %v", vol.Name, err) + } + + if state != gopowerstore.RsStatePaused { + return nil, status.Errorf(codes.Aborted, + "failed to expand the volume %q because the metro replication session is in state %q. Please pause the metro replication session manually.", + vol.Name, state) + } + } + + _, err = client.ModifyVolume(context.Background(), &gopowerstore.VolumeModify{Size: requiredBytes}, id) if err != nil { - return nil, err + return nil, status.Errorf(codes.Internal, "unable to modify volume size: %s", err.Error()) } return &csi.ControllerExpandVolumeResponse{CapacityBytes: requiredBytes, NodeExpansionRequired: true}, nil } + return &csi.ControllerExpandVolumeResponse{}, nil } - fs, err := s.Arrays()[arrayID].Client.GetFS(ctx, id) + + fs, err := client.GetFS(ctx, id) if err == nil { if fs.SizeTotal < requiredBytes { - _, err = s.Arrays()[arrayID].Client.ModifyFS(context.Background(), &gopowerstore.FSModify{Size: int(requiredBytes + ReservedSize)}, id) + _, err = client.ModifyFS(context.Background(), &gopowerstore.FSModify{Size: int(requiredBytes + ReservedSize)}, id) if err != nil { return nil, err } @@ -1241,10 +1874,15 @@ func (s *Service) ControllerExpandVolume(ctx context.Context, req *csi.Controlle // ControllerGetVolume fetch current information about a volume func (s *Service) ControllerGetVolume(ctx context.Context, req *csi.ControllerGetVolumeRequest) (*csi.ControllerGetVolumeResponse, error) { - id, arrayID, protocol, err := array.ParseVolumeID(ctx, req.VolumeId, s.DefaultArray(), nil) + volumeHandle, err := array.ParseVolumeID(ctx, req.VolumeId, s.DefaultArray(), nil) if err != nil { return nil, status.Errorf(codes.OutOfRange, "unable to parse the volume id") } + + id := volumeHandle.LocalUUID + arrayID := volumeHandle.LocalArrayGlobalID + protocol := volumeHandle.Protocol + var hosts []string abnormal := false message := "" @@ -1327,60 +1965,126 @@ func (s *Service) RegisterAdditionalServers(server *grpc.Server) { } // ProbeController probes the controller service -func (s *Service) ProbeController(ctx context.Context, req *commonext.ProbeControllerRequest) (*commonext.ProbeControllerResponse, error) { - ready := new(wrappers.BoolValue) +func (s *Service) ProbeController(ctx context.Context, _ *commonext.ProbeControllerRequest) (*commonext.ProbeControllerResponse, error) { + log := log.WithContext(ctx) + ready := new(wrapperspb.BoolValue) ready.Value = true rep := new(commonext.ProbeControllerResponse) rep.Ready = ready - rep.Name = common.Name - rep.VendorVersion = core.SemVer - rep.Manifest = common.Manifest + rep.Name = identifiers.Name + rep.VendorVersion = identifiers.ManifestSemver + identifiers.Manifest["semver"] = identifiers.ManifestSemver + rep.Manifest = identifiers.Manifest log.Debug(fmt.Sprintf("ProbeController returning: %v", rep.Ready.GetValue())) return rep, nil } -func (s *Service) listPowerStoreVolumes(ctx context.Context, startToken, maxEntries int) ([]gopowerstore.Volume, string, error) { - var volumes []gopowerstore.Volume +func (s *Service) listPowerStoreVolumes(ctx context.Context, startToken, maxEntries int) ([]*csi.ListVolumesResponse_Entry, string, error) { + var volResponse []*csi.ListVolumesResponse_Entry - // Get the volumes from the cache if we can - for _, arr := range s.Arrays() { - v, err := arr.GetClient().GetVolumes(ctx) + // Pre-fetch host-volume mappings for every array (so we only call mapping API once) + mappingsByArray := make(map[string][]gopowerstore.HostVolumeMapping) + for arrayID, arr := range s.Arrays() { + maps, err := arr.GetClient().GetHostVolumeMappings(ctx) + if err != nil { + log.Warnf("ListVolumes: failed to fetch host-volume mappings for array %s: %v", arrayID, err) + continue + } + mappingsByArray[arrayID] = maps + } + + // --------------------------- + // Block Volumes (SCSI) + // --------------------------- + for arrayID, arr := range s.Arrays() { + vols, err := arr.GetClient().GetVolumes(ctx) if err != nil { return nil, "", status.Errorf(codes.Internal, "unable to list volumes: %s", err.Error()) } - volumes = append(volumes, v...) + // for each volume build CSI-style volume id and populate published node ids + maps := mappingsByArray[arrayID] + for _, vol := range vols { + // build CSI volumeID so it matches PV.spec.csi.volumeHandle + // format: "//scsi" + fullVolumeID := fmt.Sprintf("%s/%s/scsi", vol.ID, arrayID) + + entry := &csi.ListVolumesResponse_Entry{ + Volume: &csi.Volume{ + CapacityBytes: int64(vol.Size), + VolumeId: fullVolumeID, + }, + } + + // Populate PublishedNodeIds from host_volume_mapping -> host -> host.Name + var nodes []string + for _, m := range maps { + if m.VolumeID == vol.ID && m.HostID != "" { + if host, err := arr.GetClient().GetHost(ctx, m.HostID); err == nil && host.Name != "" { + nodes = append(nodes, host.Name) + } + } + } + if len(nodes) > 0 { + entry.Status = &csi.ListVolumesResponse_VolumeStatus{ + PublishedNodeIds: nodes, + } + } + volResponse = append(volResponse, entry) + } + } + + // --------------------------- + // FileSystems (NFS) + // --------------------------- + for arrayID, arr := range s.Arrays() { + fsList, err := arr.GetClient().ListFS(ctx) + if err != nil { + return nil, "", status.Errorf(codes.Internal, "unable to list filesystems: %s", err.Error()) + } + for _, fs := range fsList { + // NFS volumeID format: "//nfs" + fullVolumeID := fmt.Sprintf("%s/%s/nfs", fs.ID, arrayID) + entry := &csi.ListVolumesResponse_Entry{ + Volume: &csi.Volume{ + CapacityBytes: int64(fs.SizeTotal), + VolumeId: fullVolumeID, + }, + } + // NFS does not have host mappings in the same way: leaving Status nil is fine + volResponse = append(volResponse, entry) + } } - if startToken > len(volumes) { - return nil, "", status.Errorf(codes.Aborted, "startingToken=%d > len(volumes)=%d", startToken, len(volumes)) + if startToken > len(volResponse) { + return nil, "", status.Errorf(codes.Aborted, "startingToken=%d > len(volumes)=%d", startToken, len(volResponse)) } - // Discern the number of remaining entries. - rem := len(volumes) - startToken - // If maxEntries is 0 or greater than the number of remaining entries then - // set max entries to the number of remaining entries. - if maxEntries == 0 || maxEntries > rem { - maxEntries = rem + remaining := len(volResponse) - startToken + + if maxEntries == 0 || maxEntries > remaining { + maxEntries = remaining } - // We can't really return more per page + // Cap the number of entries returned in a single response. + // This prevents extremely large CSI responses, which can increase memory usage + // and impact ListVolumes performance on clusters with many volumes. if maxEntries > 700 { maxEntries = 700 } - // Compute the next starting point; if at end reset nextToken := startToken + maxEntries nextTokenStr := "" - if nextToken < (startToken + rem) { + if nextToken < len(volResponse) { nextTokenStr = fmt.Sprintf("%d", nextToken) } - return volumes[startToken : startToken+maxEntries], nextTokenStr, nil + return volResponse[startToken:nextToken], nextTokenStr, nil } func (s *Service) listPowerStoreSnapshots(ctx context.Context, startToken, maxEntries int, snapID, srcID string) ([]GeneralSnapshot, string, error) { + log := log.WithContext(ctx) var generalSnapshots []GeneralSnapshot if snapID == "" && srcID == "" { @@ -1408,12 +2112,16 @@ func (s *Service) listPowerStoreSnapshots(ctx context.Context, startToken, maxEn } } else if snapID != "" { log.Infof("Requested snapshot via snapshot id %s", snapID) - id, arrayID, protocol, err := array.ParseVolumeID(ctx, snapID, s.DefaultArray(), nil) + volumeHandle, err := array.ParseVolumeID(ctx, snapID, s.DefaultArray(), nil) if err != nil { - log.Error(err) + log.Error(err.Error()) return []GeneralSnapshot{}, "", nil } + id := volumeHandle.LocalUUID + arrayID := volumeHandle.LocalArrayGlobalID + protocol := volumeHandle.Protocol + arr, ok := s.Arrays()[arrayID] if !ok { return nil, "", status.Errorf(codes.Internal, "unable to get array with arrayID %s", arrayID) @@ -1429,7 +2137,7 @@ func (s *Service) listPowerStoreSnapshots(ctx context.Context, startToken, maxEn return nil, "", status.Errorf(codes.Internal, "unable to get filesystem snapshot: %s", getErr.Error()) } - log.Info(fsSnapshot) + log.Infof("%+v", fsSnapshot) fsSnapshot.ID = fsSnapshot.ID + "/" + arrayID + "/" + protocol generalSnapshots = append(generalSnapshots, FilesystemSnapshot(fsSnapshot)) @@ -1448,12 +2156,16 @@ func (s *Service) listPowerStoreSnapshots(ctx context.Context, startToken, maxEn } else { log.Infof("Requested snapshot via source id %s", srcID) // This works VGS on single default array, But for multiple array scenario this default array should be changed to dynamic array - id, arrayID, protocol, err := array.ParseVolumeID(ctx, srcID, s.DefaultArray(), nil) + volumeHandle, err := array.ParseVolumeID(ctx, srcID, s.DefaultArray(), nil) if err != nil { - log.Error(err) + log.Error(err.Error()) return []GeneralSnapshot{}, "", nil } + id := volumeHandle.LocalUUID + arrayID := volumeHandle.LocalArrayGlobalID + protocol := volumeHandle.Protocol + arr, ok := s.Arrays()[arrayID] if !ok { return nil, "", status.Errorf(codes.Internal, "unable to get array with arrayID %s", arrayID) diff --git a/pkg/controller/controller_node_to_array_connectivity.go b/pkg/controller/controller_node_to_array_connectivity.go index 543b5a1a..8ed63399 100644 --- a/pkg/controller/controller_node_to_array_connectivity.go +++ b/pkg/controller/controller_node_to_array_connectivity.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2022-2023 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2022-2025 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,21 +27,21 @@ import ( "net/http" "time" - log "github.com/sirupsen/logrus" - - "github.com/dell/csi-powerstore/v2/pkg/common" + "github.com/dell/csi-powerstore/v2/pkg/identifiers" ) // QueryArrayStatus make API call to the specified url to retrieve connection status func (s *Service) QueryArrayStatus(ctx context.Context, url string) (bool, error) { + log := log.WithContext(ctx) defer func() { if err := recover(); err != nil { - log.Println("panic occurred in queryStatus:", err) + log.Debugf("panic occurred in queryStatus: %v", err) } }() client := http.Client{ - Timeout: common.Timeout, + Timeout: identifiers.PodmonArrayConnectivityTimeout, } + resp, err := client.Get(url) log.Debugf("Received response %+v for url %s", resp, url) @@ -59,26 +59,26 @@ func (s *Service) QueryArrayStatus(ctx context.Context, url string) (bool, error log.Errorf("Found unexpected response from the server while fetching array status %d ", resp.StatusCode) return false, fmt.Errorf("unexpected response from the server") } - var statusResponse common.ArrayConnectivityStatus + var statusResponse identifiers.ArrayConnectivityStatus err = json.Unmarshal(bodyBytes, &statusResponse) if err != nil { log.Errorf("unable to unmarshal and determine connectivity due to %s ", err) return false, err } log.Infof("API Response received is %+v\n", statusResponse) - //responseObject has last success and last attempt timestamp in Unix format + // responseObject has last success and last attempt timestamp in Unix format timeDiff := statusResponse.LastAttempt - statusResponse.LastSuccess - tolerance := common.SetPollingFrequency(ctx) + tolerance := identifiers.SetPollingFrequency(ctx) currTime := time.Now().Unix() - //checking if the status response is stale and connectivity test is still running - //since nodeProbe is run at frequency tolerance/2, ideally below check should never be true + // checking if the status response is stale and connectivity test is still running + // since nodeProbe is run at frequency tolerance/2, ideally below check should never be true if (currTime - statusResponse.LastAttempt) > tolerance*2 { log.Errorf("seems like connectivity test is not being run, current time is %d and last run was at %d", currTime, statusResponse.LastAttempt) - //considering connectivity is broken + // considering connectivity is broken return false, nil } log.Debugf("last connectivity was %d sec back, tolerance is %d sec", timeDiff, tolerance) - //give 2s leeway for tolerance check + // give 2s leeway for tolerance check if timeDiff <= tolerance+2 { return true, nil } diff --git a/pkg/controller/controller_test.go b/pkg/controller/controller_test.go index 512bc98d..2d430ff9 100644 --- a/pkg/controller/controller_test.go +++ b/pkg/controller/controller_test.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2021-2023 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2021-2024 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ * */ -package controller_test +package controller import ( "context" @@ -24,73 +24,143 @@ import ( "fmt" "net/http" "path/filepath" + "strings" "testing" + "time" csiext "github.com/dell/dell-csi-extensions/replication" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "github.com/dell/csi-powerstore/v2/mocks" - "github.com/dell/csi-powerstore/v2/pkg/controller" csictx "github.com/dell/gocsi/context" - "github.com/container-storage-interface/spec/lib/go/csi" "github.com/dell/csi-powerstore/v2/pkg/array" - "github.com/dell/csi-powerstore/v2/pkg/common" + "github.com/dell/csi-powerstore/v2/pkg/identifiers" + "github.com/dell/csi-powerstore/v2/pkg/identifiers/k8sutils" "github.com/dell/gopowerstore" "github.com/dell/gopowerstore/api" gopowerstoremock "github.com/dell/gopowerstore/mocks" - . "github.com/onsi/ginkgo" + "github.com/container-storage-interface/spec/lib/go/csi" + ginkgo "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" - . "github.com/onsi/gomega" + gomega "github.com/onsi/gomega" "github.com/stretchr/testify/mock" + k8score "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/rest" ) const ( validBaseVolID = "39bb1b5f-5624-490d-9ece-18f7b28a904e" - validBlockVolumeID = "39bb1b5f-5624-490d-9ece-18f7b28a904e/globalvolid1/scsi" - validNfsVolumeID = "39bb1b5f-5624-490d-9ece-18f7b28a904e/globalvolid2/nfs" - invalidBlockVolumeID = "39bb1b5f-5624-490d-9ece-18f7b28a904e/globalvolid3/scsi" + validLegacyVolID = validBaseVolID validNasID = "24aefac2-a796-47dc-886a-c73ff8c1a671" - validVolSize = 16 * 1024 * 1024 * 1024 + KiB = 1024 + MiB = 1024 * KiB + GiB = 1024 * MiB + validVolSize = 16 * GiB firstValidID = "globalvolid1" secondValidID = "globalvolid2" validNasName = "my-nas-name" validSnapName = "my-snap" - validNodeID = "csi-node-1a47a1b91c444a8a90193d8066669603-127.0.0.1" validHostName = "csi-node-1a47a1b91c444a8a90193d8066669603" validHostID = "24aefac2-a796-47dc-886a-c73ff8c1a671" validClusterName = "localSystemName" - validRemoteVolId = "9f840c56-96e6-4de9-b5a3-27e7c20eaa77" + validRemoteVolID = "9f840c56-96e6-4de9-b5a3-27e7c20eaa77" validRemoteSystemName = "remoteName" validRemoteSystemID = "df7f804c-6373-4659-b197-36654d17979c" + validSessionID = "9abd0198-2733-4e46-b5fa-456e9c367184" validRPO = "Five_Minutes" + zeroRPO = "Zero" + replicationModeSync = "SYNC" + replicationModeAsync = "ASYNC" validGroupID = "610adaef-4f0a-4dff-9812-29ffa5daf185" validRemoteGroupID = "62ed932b-329b-4ba6-b0e0-3f51c34c4701" validNamespaceName = "default" - validGroupName = "csi-" + validRemoteSystemName + "-" + validRPO - validNamespacedGroupName = "csi-" + validNamespaceName + "-" + validRemoteSystemName + "-" + validRPO validPolicyID = "e74f6cfd-ae2a-4cde-ad6b-529b40edee5e" - validPolicyName = "pp-" + validGroupName validRuleID = "c721f30b-0b37-4aaf-a3a2-ef99caba2100" - validRuleName = "rr-" + validGroupName - validReplicationPrefix = "/" + controller.KeyReplicationEnabled + validReplicationPrefix = "/" + KeyReplicationEnabled validVolumeGroupName = "VGName" validRemoteSystemGlobalID = "PS111111111111" validNfsAcls = "A::OWNER@:RWX" validNfsServerID = "24aefac2-a796-47dc-886a-c73ff8c1a671" validApplianceID = "my-appliance" + validRemoteApplianceID = "my-appliance2" validServiceTag = "service-tag" + replicationSessionID = "123456" +) + +var ( + // format: //scsi + validBlockVolumeID = filepath.Join(validBaseVolID, firstValidID, "scsi") + + // format: //scsi:/ + validMetroBlockVolumeID = filepath.Join(validBaseVolID, firstValidID, "scsi:"+validRemoteVolID, secondValidID) + + // format: //scsi:/ + invalidMetroBlockVolumeID = filepath.Join(validBaseVolID, firstValidID, "scsi:"+validRemoteVolID, "globalvolid3") + + // format: //nfs + validNfsVolumeID = filepath.Join(validBaseVolID, secondValidID, "nfs") + + // format: //scsi. + // + // should expect not to find this global ID in the list of arrays + invalidBlockVolumeID = filepath.Join(validBaseVolID, "globalvolid3", "scsi") + + // format: csi-node--127.0.0.1 + validNodeID = strings.Join([]string{validHostName, "127.0.0.1"}, "-") + + // format: csi-- + validGroupName = strings.Join([]string{"csi", validRemoteSystemName, validRPO}, "-") + + // format: pp-csi-- + validPolicyName = "pp-" + validGroupName + + // format: rr-csi-- + validRuleName = "rr-" + validGroupName + + // format: csi--Zero + validGroupNameSync = strings.Join([]string{"csi", validRemoteSystemName, zeroRPO}, "-") + + // format: pp-csi--Zero + validPolicyNameSync = "pp-" + validGroupNameSync + + // format: csi--- + validNamespacedGroupName = strings.Join([]string{"csi", validNamespaceName, validRemoteSystemName, validRPO}, "-") + + // format: csi---Zero + validNamespacedGroupNameSync = strings.Join([]string{"csi", validNamespaceName, validRemoteSystemName, zeroRPO}, "-") ) var ( clientMock *gopowerstoremock.Client fsMock *mocks.FsInterface - ctrlSvc *controller.Service + ctrlSvc *Service ) func TestCSIControllerService(t *testing.T) { - RegisterFailHandler(Fail) + defaultK8sConfigFunc := k8sutils.InClusterConfigFunc + defaultK8sClientsetFunc := k8sutils.NewForConfigFunc + + k8sutils.InClusterConfigFunc = func() (*rest.Config, error) { + return &rest.Config{}, nil + } + k8sutils.NewForConfigFunc = func(_ *rest.Config) (kubernetes.Interface, error) { + return fake.NewClientset(), nil + } + + defer func() { + k8sutils.InClusterConfigFunc = defaultK8sConfigFunc + k8sutils.NewForConfigFunc = defaultK8sClientsetFunc + }() + + gomega.RegisterFailHandler(ginkgo.Fail) junitReporter := reporters.NewJUnitReporter("ctrl-svc.xml") - RunSpecsWithDefaultAndCustomReporters(t, "CSIControllerService testing suite", []Reporter{junitReporter}) + ginkgo.RunSpecsWithDefaultAndCustomReporters(t, "CSIControllerService testing suite", []ginkgo.Reporter{junitReporter}) } func setVariables() { @@ -99,37 +169,45 @@ func setVariables() { arrays := make(map[string]*array.PowerStoreArray) first := &array.PowerStoreArray{ - Endpoint: "https://192.168.0.1/api/rest", - Username: "admin", - GlobalID: firstValidID, - Password: "pass", - BlockProtocol: common.ISCSITransport, - Insecure: true, - IsDefault: true, - Client: clientMock, - IP: "192.168.0.1", + Endpoint: "https://192.168.0.1/api/rest", + Username: "admin", + GlobalID: firstValidID, + Password: "pass", + BlockProtocol: identifiers.ISCSITransport, + Insecure: true, + IsDefault: true, + Client: clientMock, + IP: "192.168.0.1", + NASCooldownTracker: array.NewNASCooldown(time.Minute, 5), } second := &array.PowerStoreArray{ - Endpoint: "https://192.168.0.2/api/rest", - Username: "admin", - GlobalID: secondValidID, - Password: "pass", - NasName: validNasName, - BlockProtocol: common.NoneTransport, - Insecure: true, - Client: clientMock, - IP: "192.168.0.2", + Endpoint: "https://192.168.0.2/api/rest", + Username: "admin", + GlobalID: secondValidID, + Password: "pass", + NasName: validNasName, + BlockProtocol: identifiers.NoneTransport, + Insecure: true, + Client: clientMock, + IP: "192.168.0.2", + NASCooldownTracker: array.NewNASCooldown(time.Minute, 5), } arrays[firstValidID] = first arrays[secondValidID] = second - csictx.Setenv(context.Background(), common.EnvReplicationPrefix, "replication.storage.dell.com") - csictx.Setenv(context.Background(), common.EnvNfsAcls, "A::OWNER@:RWX") + csictx.Setenv(context.Background(), identifiers.EnvReplicationPrefix, "replication.storage.dell.com") + csictx.Setenv(context.Background(), identifiers.EnvNfsAcls, "A::OWNER@:RWX") - ctrlSvc = &controller.Service{Fs: fsMock} + ctrlSvc = &Service{ + Fs: fsMock, + IsCSMDREnabled: true, + } ctrlSvc.SetArrays(arrays) ctrlSvc.SetDefaultArray(first) + k8sutils.Kubeclient = &k8sutils.K8sClient{ + Clientset: fake.NewSimpleClientset(), + } ctrlSvc.Init() } @@ -137,23 +215,25 @@ func addMetaData(createParams interface{}) { if t, ok := createParams.(interface { MetaData() http.Header }); ok { - t.MetaData().Set(controller.HeaderPersistentVolumeName, "") - t.MetaData().Set(controller.HeaderPersistentVolumeClaimName, "") - t.MetaData().Set(controller.HeaderPersistentVolumeClaimNamespace, "") + t.MetaData().Set(HeaderPersistentVolumeName, "") + t.MetaData().Set(HeaderPersistentVolumeClaimName, "") + t.MetaData().Set(HeaderPersistentVolumeClaimNamespace, "") } else { fmt.Printf("warning: %T: no MetaData method exists, consider updating gopowerstore library.", createParams) } } -var _ = Describe("CSIControllerService", func() { - BeforeEach(func() { +var _ = ginkgo.Describe("CSIControllerService", func() { + ginkgo.BeforeEach(func() { setVariables() }) - Describe("calling CreateVolume()", func() { - When("creating block volume", func() { - It("should successfully create block volume", func() { - clientMock.On("GetCustomHTTPHeaders").Return(make(http.Header)) + ginkgo.Describe("calling CreateVolume()", func() { + ginkgo.When("creating block volume", func() { + ginkgo.It("should successfully create block volume", func() { + clientMock.On("GetCustomHTTPHeaders").Return(api.NewSafeHeader().GetHeader()) + clientMock.On("GetVolumeByName", mock.Anything, mock.Anything).Return( + gopowerstore.Volume{}, errors.New("no vol found")) clientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(3.0), nil) clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) clientMock.On("CreateVolume", mock.Anything, mock.Anything).Return(gopowerstore.CreateResponse{ID: validBaseVolID}, nil) @@ -161,89 +241,93 @@ var _ = Describe("CSIControllerService", func() { clientMock.On("GetAppliance", context.Background(), mock.Anything).Return(gopowerstore.ApplianceInstance{ServiceTag: validServiceTag}, nil) req := getTypicalCreateVolumeRequest("my-vol", validVolSize) - req.Parameters[common.KeyArrayID] = firstValidID - req.Parameters[controller.KeyCSIPVCName] = req.Name - req.Parameters[controller.KeyCSIPVCNamespace] = validNamespaceName + req.Parameters[identifiers.KeyArrayID] = firstValidID + req.Parameters[KeyCSIPVCName] = req.Name + req.Parameters[KeyCSIPVCNamespace] = validNamespaceName res, err := ctrlSvc.CreateVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.CreateVolumeResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.CreateVolumeResponse{ Volume: &csi.Volume{ CapacityBytes: validVolSize, VolumeId: filepath.Join(validBaseVolID, firstValidID, "scsi"), VolumeContext: map[string]string{ - common.KeyArrayID: firstValidID, - common.KeyArrayVolumeName: "my-vol", - common.KeyProtocol: "scsi", - common.KeyVolumeDescription: req.Name + "-" + validNamespaceName, - controller.KeyCSIPVCName: req.Name, - controller.KeyCSIPVCNamespace: validNamespaceName, - common.KeyServiceTag: validServiceTag, + identifiers.KeyArrayID: firstValidID, + identifiers.KeyArrayVolumeName: "my-vol", + identifiers.KeyProtocol: "scsi", + identifiers.KeyVolumeDescription: req.Name + "-" + validNamespaceName, + KeyCSIPVCName: req.Name, + KeyCSIPVCNamespace: validNamespaceName, + identifiers.KeyServiceTag: validServiceTag, }, }, })) }) }) - It("should successfully create block volume and vol attributes should be set", func() { - clientMock.On("GetCustomHTTPHeaders").Return(make(http.Header)) + ginkgo.It("should successfully create block volume and vol attributes should be set", func() { + clientMock.On("GetCustomHTTPHeaders").Return(api.NewSafeHeader().GetHeader()) clientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(3.0), nil) clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) + clientMock.On("GetVolumeByName", mock.Anything, mock.Anything).Return( + gopowerstore.Volume{}, errors.New("no vol found")) clientMock.On("CreateVolume", mock.Anything, mock.Anything).Return(gopowerstore.CreateResponse{ID: validBaseVolID}, nil) clientMock.On("GetVolume", context.Background(), mock.Anything).Return(gopowerstore.Volume{ApplianceID: validApplianceID}, nil) clientMock.On("GetAppliance", context.Background(), mock.Anything).Return(gopowerstore.ApplianceInstance{ServiceTag: validServiceTag}, nil) req := getTypicalCreateVolumeRequest("my-vol", validVolSize) - req.Parameters[common.KeyArrayID] = firstValidID - req.Parameters[controller.KeyCSIPVCName] = req.Name - req.Parameters[controller.KeyCSIPVCNamespace] = validNamespaceName - req.Parameters[common.KeyVolumeDescription] = "Vol-description" - req.Parameters[common.KeyAppType] = "Other" - req.Parameters[common.KeyAppTypeOther] = "Android" - req.Parameters[common.KeyApplianceID] = "12345" - req.Parameters[common.KeyProtectionPolicyID] = "xyz" - req.Parameters[common.KeyPerformancePolicyID] = "abc" + req.Parameters[identifiers.KeyArrayID] = firstValidID + req.Parameters[KeyCSIPVCName] = req.Name + req.Parameters[KeyCSIPVCNamespace] = validNamespaceName + req.Parameters[identifiers.KeyVolumeDescription] = "Vol-description" + req.Parameters[identifiers.KeyAppType] = "Other" + req.Parameters[identifiers.KeyAppTypeOther] = "Android" + req.Parameters[identifiers.KeyApplianceID] = "12345" + req.Parameters[identifiers.KeyProtectionPolicyID] = "xyz" + req.Parameters[identifiers.KeyPerformancePolicyID] = "abc" res, err := ctrlSvc.CreateVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.CreateVolumeResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.CreateVolumeResponse{ Volume: &csi.Volume{ CapacityBytes: validVolSize, VolumeId: filepath.Join(validBaseVolID, firstValidID, "scsi"), VolumeContext: map[string]string{ - common.KeyArrayID: firstValidID, - common.KeyArrayVolumeName: "my-vol", - common.KeyProtocol: "scsi", - common.KeyVolumeDescription: "Vol-description", - common.KeyAppType: "Other", - common.KeyAppTypeOther: "Android", - common.KeyApplianceID: "12345", - common.KeyProtectionPolicyID: "xyz", - common.KeyPerformancePolicyID: "abc", - controller.KeyCSIPVCName: req.Name, - controller.KeyCSIPVCNamespace: validNamespaceName, - common.KeyServiceTag: validServiceTag, + identifiers.KeyArrayID: firstValidID, + identifiers.KeyArrayVolumeName: "my-vol", + identifiers.KeyProtocol: "scsi", + identifiers.KeyVolumeDescription: "Vol-description", + identifiers.KeyAppType: "Other", + identifiers.KeyAppTypeOther: "Android", + identifiers.KeyApplianceID: "12345", + identifiers.KeyProtectionPolicyID: "xyz", + identifiers.KeyPerformancePolicyID: "abc", + KeyCSIPVCName: req.Name, + KeyCSIPVCNamespace: validNamespaceName, + identifiers.KeyServiceTag: validServiceTag, }, }, })) }) }) - When("create block volume with replication properties", func() { + ginkgo.When("creating a block volume with replication properties", func() { var req *csi.CreateVolumeRequest - BeforeEach(func() { + ginkgo.BeforeEach(func() { req = getTypicalCreateVolumeRequest("my-vol", validVolSize) - req.Parameters[common.KeyArrayID] = firstValidID - req.Parameters[ctrlSvc.WithRP(controller.KeyReplicationEnabled)] = "true" - req.Parameters[ctrlSvc.WithRP(controller.KeyReplicationRPO)] = validRPO - req.Parameters[ctrlSvc.WithRP(controller.KeyReplicationRemoteSystem)] = validRemoteSystemName - req.Parameters[ctrlSvc.WithRP(controller.KeyReplicationIgnoreNamespaces)] = "true" - req.Parameters[ctrlSvc.WithRP(controller.KeyReplicationVGPrefix)] = "csi" - req.Parameters[controller.KeyCSIPVCName] = req.Name - req.Parameters[controller.KeyCSIPVCNamespace] = validNamespaceName - }) - - It("should create volume and volumeGroup if policy exists", func() { - clientMock.On("GetCustomHTTPHeaders").Return(make(http.Header)) + req.Parameters[identifiers.KeyArrayID] = firstValidID + req.Parameters[ctrlSvc.WithRP(KeyReplicationEnabled)] = "true" + req.Parameters[ctrlSvc.WithRP(KeyReplicationRPO)] = validRPO + req.Parameters[ctrlSvc.WithRP(KeyReplicationRemoteSystem)] = validRemoteSystemName + req.Parameters[ctrlSvc.WithRP(KeyReplicationIgnoreNamespaces)] = "true" + req.Parameters[ctrlSvc.WithRP(KeyReplicationVGPrefix)] = "csi" + req.Parameters[KeyCSIPVCName] = req.Name + req.Parameters[KeyCSIPVCNamespace] = validNamespaceName + }) + + ginkgo.It("should create volume and volumeGroup if policy exists - ASYNC", func() { + clientMock.On("GetCustomHTTPHeaders").Return(api.NewSafeHeader().GetHeader()) + clientMock.On("GetVolumeByName", mock.Anything, mock.Anything).Return( + gopowerstore.Volume{}, errors.New("no vol found")) clientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(3.0), nil) clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) // all entities not exists @@ -261,39 +345,91 @@ var _ = Describe("CSIControllerService", func() { clientMock.On("GetAppliance", context.Background(), mock.Anything).Return(gopowerstore.ApplianceInstance{ServiceTag: validServiceTag}, nil) res, err := ctrlSvc.CreateVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.CreateVolumeResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.CreateVolumeResponse{ + Volume: &csi.Volume{ + CapacityBytes: validVolSize, + VolumeId: filepath.Join(validBaseVolID, firstValidID, "scsi"), + VolumeContext: map[string]string{ + identifiers.KeyArrayVolumeName: "my-vol", + identifiers.KeyProtocol: "scsi", + identifiers.KeyArrayID: firstValidID, + identifiers.KeyVolumeDescription: req.Name + "-" + validNamespaceName, + identifiers.KeyServiceTag: validServiceTag, + KeyCSIPVCName: req.Name, + KeyCSIPVCNamespace: validNamespaceName, + ctrlSvc.WithRP(KeyReplicationEnabled): "true", + ctrlSvc.WithRP(KeyReplicationRPO): validRPO, + ctrlSvc.WithRP(KeyReplicationRemoteSystem): validRemoteSystemName, + ctrlSvc.WithRP(KeyReplicationIgnoreNamespaces): "true", + ctrlSvc.WithRP(KeyReplicationVGPrefix): "csi", + }, + }, + })) + }) + + ginkgo.It("should create volume and volumeGroup if policy exists - SYNC", func() { + clientMock.On("GetCustomHTTPHeaders").Return(api.NewSafeHeader().GetHeader()) + clientMock.On("GetVolumeByName", mock.Anything, mock.Anything).Return( + gopowerstore.Volume{}, errors.New("no vol found")) + clientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(3.0), nil) + clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) + // Setting Replciation mode and corresponding attributes for SYNC + req.Parameters[ctrlSvc.WithRP(KeyReplicationMode)] = replicationModeSync + req.Parameters[ctrlSvc.WithRP(KeyReplicationRPO)] = zeroRPO + + // all entities not exists + clientMock.On("GetVolumeGroupByName", mock.Anything, validGroupNameSync). + Return(gopowerstore.VolumeGroup{}, gopowerstore.NewNotFoundError()) + + EnsureProtectionPolicyExistsMockSync() + + createGroupRequest := &gopowerstore.VolumeGroupCreate{Name: validGroupNameSync, ProtectionPolicyID: validPolicyID} + clientMock.On("CreateVolumeGroup", mock.Anything, createGroupRequest).Return(gopowerstore.CreateResponse{ID: validGroupID}, nil) + clientMock.On("GetVolumeGroup", mock.Anything, validGroupID).Return(gopowerstore.VolumeGroup{ID: validGroupID, ProtectionPolicyID: validPolicyID}, nil) + + clientMock.On("CreateVolume", mock.Anything, mock.Anything).Return(gopowerstore.CreateResponse{ID: validBaseVolID}, nil) + clientMock.On("GetVolume", context.Background(), mock.Anything).Return(gopowerstore.Volume{ApplianceID: validApplianceID}, nil) + clientMock.On("GetAppliance", context.Background(), mock.Anything).Return(gopowerstore.ApplianceInstance{ServiceTag: validServiceTag}, nil) + + res, err := ctrlSvc.CreateVolume(context.Background(), req) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.CreateVolumeResponse{ Volume: &csi.Volume{ CapacityBytes: validVolSize, VolumeId: filepath.Join(validBaseVolID, firstValidID, "scsi"), VolumeContext: map[string]string{ - common.KeyArrayVolumeName: "my-vol", - common.KeyProtocol: "scsi", - common.KeyArrayID: firstValidID, - common.KeyVolumeDescription: req.Name + "-" + validNamespaceName, - common.KeyServiceTag: validServiceTag, - controller.KeyCSIPVCName: req.Name, - controller.KeyCSIPVCNamespace: validNamespaceName, - ctrlSvc.WithRP(controller.KeyReplicationEnabled): "true", - ctrlSvc.WithRP(controller.KeyReplicationRPO): validRPO, - ctrlSvc.WithRP(controller.KeyReplicationRemoteSystem): validRemoteSystemName, - ctrlSvc.WithRP(controller.KeyReplicationIgnoreNamespaces): "true", - ctrlSvc.WithRP(controller.KeyReplicationVGPrefix): "csi", + identifiers.KeyArrayVolumeName: "my-vol", + identifiers.KeyProtocol: "scsi", + identifiers.KeyArrayID: firstValidID, + identifiers.KeyVolumeDescription: req.Name + "-" + validNamespaceName, + identifiers.KeyServiceTag: validServiceTag, + KeyCSIPVCName: req.Name, + KeyCSIPVCNamespace: validNamespaceName, + ctrlSvc.WithRP(KeyReplicationMode): replicationModeSync, + ctrlSvc.WithRP(KeyReplicationEnabled): "true", + ctrlSvc.WithRP(KeyReplicationRPO): zeroRPO, + ctrlSvc.WithRP(KeyReplicationRemoteSystem): validRemoteSystemName, + ctrlSvc.WithRP(KeyReplicationIgnoreNamespaces): "true", + ctrlSvc.WithRP(KeyReplicationVGPrefix): "csi", }, }, })) }) - It("should create vg with namespace if namespaces not ignored", func() { - req.Parameters[ctrlSvc.WithRP(controller.KeyReplicationIgnoreNamespaces)] = "false" - req.Parameters[controller.KeyCSIPVCName] = req.Name - req.Parameters[controller.KeyCSIPVCNamespace] = validNamespaceName + ginkgo.It("should create vg with namespace if namespaces not ignored - ASYNC", func() { + req.Parameters[ctrlSvc.WithRP(KeyReplicationIgnoreNamespaces)] = "false" + req.Parameters[KeyCSIPVCName] = req.Name + req.Parameters[KeyCSIPVCNamespace] = validNamespaceName defer func() { - req.Parameters[ctrlSvc.WithRP(controller.KeyReplicationIgnoreNamespaces)] = "true" - req.Parameters[controller.KeyCSIPVCNamespace] = "" + req.Parameters[ctrlSvc.WithRP(KeyReplicationIgnoreNamespaces)] = "true" + req.Parameters[KeyCSIPVCNamespace] = "" }() + clientMock.On("GetVolumeByName", mock.Anything, mock.Anything).Return( + gopowerstore.Volume{}, errors.New("no vol found")) + clientMock.On("GetVolumeGroupByName", mock.Anything, validNamespacedGroupName). Return(gopowerstore.VolumeGroup{}, gopowerstore.NewNotFoundError()) @@ -308,7 +444,68 @@ var _ = Describe("CSIControllerService", func() { createGroupRequest := &gopowerstore.VolumeGroupCreate{Name: validNamespacedGroupName, ProtectionPolicyID: validPolicyID} clientMock.On("CreateVolumeGroup", mock.Anything, createGroupRequest).Return(gopowerstore.CreateResponse{ID: validGroupID}, nil) clientMock.On("GetVolumeGroup", mock.Anything, validGroupID).Return(gopowerstore.VolumeGroup{ID: validGroupID, ProtectionPolicyID: validPolicyID}, nil) - clientMock.On("GetCustomHTTPHeaders").Return(make(http.Header)) + clientMock.On("GetCustomHTTPHeaders").Return(api.NewSafeHeader().GetHeader()) + clientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(3.0), nil) + clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) + clientMock.On("CreateVolume", mock.Anything, mock.Anything).Return(gopowerstore.CreateResponse{ID: validBaseVolID}, nil) + clientMock.On("GetVolume", context.Background(), mock.Anything).Return(gopowerstore.Volume{ApplianceID: validApplianceID}, nil) + clientMock.On("GetAppliance", context.Background(), mock.Anything).Return(gopowerstore.ApplianceInstance{ServiceTag: validServiceTag}, nil) + + res, err := ctrlSvc.CreateVolume(context.Background(), req) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.CreateVolumeResponse{ + Volume: &csi.Volume{ + CapacityBytes: validVolSize, + VolumeId: filepath.Join(validBaseVolID, firstValidID, "scsi"), + VolumeContext: map[string]string{ + identifiers.KeyArrayVolumeName: "my-vol", + identifiers.KeyProtocol: "scsi", + identifiers.KeyArrayID: firstValidID, + identifiers.KeyVolumeDescription: req.Name + "-" + validNamespaceName, + identifiers.KeyServiceTag: validServiceTag, + KeyCSIPVCName: req.Name, + KeyCSIPVCNamespace: validNamespaceName, + ctrlSvc.WithRP(KeyReplicationEnabled): "true", + ctrlSvc.WithRP(KeyReplicationRPO): validRPO, + ctrlSvc.WithRP(KeyReplicationRemoteSystem): validRemoteSystemName, + ctrlSvc.WithRP(KeyReplicationIgnoreNamespaces): "false", + ctrlSvc.WithRP(KeyReplicationVGPrefix): "csi", + }, + }, + })) + }) + + ginkgo.It("should create vg with namespace if namespaces not ignored - SYNC", func() { + req.Parameters[ctrlSvc.WithRP(KeyReplicationIgnoreNamespaces)] = "false" + req.Parameters[KeyCSIPVCName] = req.Name + req.Parameters[KeyCSIPVCNamespace] = validNamespaceName + // Setting Replciation mode and corresponding attributes for SYNC + req.Parameters[ctrlSvc.WithRP(KeyReplicationMode)] = replicationModeSync + req.Parameters[ctrlSvc.WithRP(KeyReplicationRPO)] = zeroRPO + + defer func() { + req.Parameters[ctrlSvc.WithRP(KeyReplicationIgnoreNamespaces)] = "true" + req.Parameters[KeyCSIPVCNamespace] = "" + }() + + clientMock.On("GetVolumeByName", mock.Anything, mock.Anything).Return( + gopowerstore.Volume{}, errors.New("no vol found")) + + clientMock.On("GetVolumeGroupByName", mock.Anything, validNamespacedGroupNameSync). + Return(gopowerstore.VolumeGroup{}, gopowerstore.NewNotFoundError()) + + clientMock.On("GetRemoteSystemByName", mock.Anything, validRemoteSystemName).Return(gopowerstore.RemoteSystem{ + Name: validRemoteSystemName, + ID: validRemoteSystemID, + }, nil) + + clientMock.On("GetProtectionPolicyByName", mock.Anything, "pp-"+validNamespacedGroupNameSync). + Return(gopowerstore.ProtectionPolicy{ID: validPolicyID}, nil) + + createGroupRequest := &gopowerstore.VolumeGroupCreate{Name: validNamespacedGroupNameSync, ProtectionPolicyID: validPolicyID} + clientMock.On("CreateVolumeGroup", mock.Anything, createGroupRequest).Return(gopowerstore.CreateResponse{ID: validGroupID}, nil) + clientMock.On("GetVolumeGroup", mock.Anything, validGroupID).Return(gopowerstore.VolumeGroup{ID: validGroupID, ProtectionPolicyID: validPolicyID}, nil) + clientMock.On("GetCustomHTTPHeaders").Return(api.NewSafeHeader().GetHeader()) clientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(3.0), nil) clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) clientMock.On("CreateVolume", mock.Anything, mock.Anything).Return(gopowerstore.CreateResponse{ID: validBaseVolID}, nil) @@ -316,36 +513,81 @@ var _ = Describe("CSIControllerService", func() { clientMock.On("GetAppliance", context.Background(), mock.Anything).Return(gopowerstore.ApplianceInstance{ServiceTag: validServiceTag}, nil) res, err := ctrlSvc.CreateVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.CreateVolumeResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.CreateVolumeResponse{ Volume: &csi.Volume{ CapacityBytes: validVolSize, VolumeId: filepath.Join(validBaseVolID, firstValidID, "scsi"), VolumeContext: map[string]string{ - common.KeyArrayVolumeName: "my-vol", - common.KeyProtocol: "scsi", - common.KeyArrayID: firstValidID, - common.KeyVolumeDescription: req.Name + "-" + validNamespaceName, - common.KeyServiceTag: validServiceTag, - controller.KeyCSIPVCName: req.Name, - controller.KeyCSIPVCNamespace: validNamespaceName, - ctrlSvc.WithRP(controller.KeyReplicationEnabled): "true", - ctrlSvc.WithRP(controller.KeyReplicationRPO): validRPO, - ctrlSvc.WithRP(controller.KeyReplicationRemoteSystem): validRemoteSystemName, - ctrlSvc.WithRP(controller.KeyReplicationIgnoreNamespaces): "false", - ctrlSvc.WithRP(controller.KeyReplicationVGPrefix): "csi", + identifiers.KeyArrayVolumeName: "my-vol", + identifiers.KeyProtocol: "scsi", + identifiers.KeyArrayID: firstValidID, + identifiers.KeyVolumeDescription: req.Name + "-" + validNamespaceName, + identifiers.KeyServiceTag: validServiceTag, + KeyCSIPVCName: req.Name, + KeyCSIPVCNamespace: validNamespaceName, + ctrlSvc.WithRP(KeyReplicationMode): replicationModeSync, + ctrlSvc.WithRP(KeyReplicationEnabled): "true", + ctrlSvc.WithRP(KeyReplicationRPO): zeroRPO, + ctrlSvc.WithRP(KeyReplicationRemoteSystem): validRemoteSystemName, + ctrlSvc.WithRP(KeyReplicationIgnoreNamespaces): "false", + ctrlSvc.WithRP(KeyReplicationVGPrefix): "csi", }, }, })) }) - It("should create new volume with existing volumeGroup with policy", func() { + ginkgo.It("should create new volume with existing volumeGroup with policy - ASYNC", func() { + clientMock.On("GetVolumeByName", mock.Anything, mock.Anything).Return( + gopowerstore.Volume{}, errors.New("no vol found")) clientMock.On("GetVolumeGroupByName", mock.Anything, validGroupName). Return(gopowerstore.VolumeGroup{ID: validGroupID, ProtectionPolicyID: validPolicyID}, nil) - req.Parameters[controller.KeyCSIPVCName] = req.Name - req.Parameters[controller.KeyCSIPVCNamespace] = validNamespaceName - clientMock.On("GetCustomHTTPHeaders").Return(make(http.Header)) + req.Parameters[KeyCSIPVCName] = req.Name + req.Parameters[KeyCSIPVCNamespace] = validNamespaceName + clientMock.On("GetCustomHTTPHeaders").Return(api.NewSafeHeader().GetHeader()) + clientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(3.0), nil) + clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) + clientMock.On("CreateVolume", mock.Anything, mock.Anything).Return(gopowerstore.CreateResponse{ID: validBaseVolID}, nil) + clientMock.On("GetVolume", context.Background(), mock.Anything).Return(gopowerstore.Volume{ApplianceID: validApplianceID}, nil) + clientMock.On("GetAppliance", context.Background(), mock.Anything).Return(gopowerstore.ApplianceInstance{ServiceTag: validServiceTag}, nil) + + res, err := ctrlSvc.CreateVolume(context.Background(), req) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.CreateVolumeResponse{ + Volume: &csi.Volume{ + CapacityBytes: validVolSize, + VolumeId: filepath.Join(validBaseVolID, firstValidID, "scsi"), + VolumeContext: map[string]string{ + identifiers.KeyArrayVolumeName: "my-vol", + identifiers.KeyProtocol: "scsi", + identifiers.KeyArrayID: firstValidID, + identifiers.KeyVolumeDescription: req.Name + "-" + validNamespaceName, + identifiers.KeyServiceTag: validServiceTag, + KeyCSIPVCName: req.Name, + KeyCSIPVCNamespace: validNamespaceName, + ctrlSvc.WithRP(KeyReplicationEnabled): "true", + ctrlSvc.WithRP(KeyReplicationRPO): validRPO, + ctrlSvc.WithRP(KeyReplicationRemoteSystem): validRemoteSystemName, + ctrlSvc.WithRP(KeyReplicationIgnoreNamespaces): "true", + ctrlSvc.WithRP(KeyReplicationVGPrefix): "csi", + }, + }, + })) + }) + + ginkgo.It("should create new volume with existing volumeGroup with policy - SYNC", func() { + clientMock.On("GetVolumeByName", mock.Anything, mock.Anything).Return( + gopowerstore.Volume{}, errors.New("no vol found")) + clientMock.On("GetVolumeGroupByName", mock.Anything, validGroupNameSync). + Return(gopowerstore.VolumeGroup{ID: validGroupID, ProtectionPolicyID: validPolicyID, IsWriteOrderConsistent: true}, nil) + // Setting Replciation mode and corresponding attributes for SYNC + req.Parameters[ctrlSvc.WithRP(KeyReplicationMode)] = replicationModeSync + req.Parameters[ctrlSvc.WithRP(KeyReplicationRPO)] = zeroRPO + + req.Parameters[KeyCSIPVCName] = req.Name + req.Parameters[KeyCSIPVCNamespace] = validNamespaceName + clientMock.On("GetCustomHTTPHeaders").Return(api.NewSafeHeader().GetHeader()) clientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(3.0), nil) clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) clientMock.On("CreateVolume", mock.Anything, mock.Anything).Return(gopowerstore.CreateResponse{ID: validBaseVolID}, nil) @@ -353,39 +595,107 @@ var _ = Describe("CSIControllerService", func() { clientMock.On("GetAppliance", context.Background(), mock.Anything).Return(gopowerstore.ApplianceInstance{ServiceTag: validServiceTag}, nil) res, err := ctrlSvc.CreateVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.CreateVolumeResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.CreateVolumeResponse{ Volume: &csi.Volume{ CapacityBytes: validVolSize, VolumeId: filepath.Join(validBaseVolID, firstValidID, "scsi"), VolumeContext: map[string]string{ - common.KeyArrayVolumeName: "my-vol", - common.KeyProtocol: "scsi", - common.KeyArrayID: firstValidID, - common.KeyVolumeDescription: req.Name + "-" + validNamespaceName, - common.KeyServiceTag: validServiceTag, - controller.KeyCSIPVCName: req.Name, - controller.KeyCSIPVCNamespace: validNamespaceName, - ctrlSvc.WithRP(controller.KeyReplicationEnabled): "true", - ctrlSvc.WithRP(controller.KeyReplicationRPO): validRPO, - ctrlSvc.WithRP(controller.KeyReplicationRemoteSystem): validRemoteSystemName, - ctrlSvc.WithRP(controller.KeyReplicationIgnoreNamespaces): "true", - ctrlSvc.WithRP(controller.KeyReplicationVGPrefix): "csi", + identifiers.KeyArrayVolumeName: "my-vol", + identifiers.KeyProtocol: "scsi", + identifiers.KeyArrayID: firstValidID, + identifiers.KeyVolumeDescription: req.Name + "-" + validNamespaceName, + identifiers.KeyServiceTag: validServiceTag, + KeyCSIPVCName: req.Name, + KeyCSIPVCNamespace: validNamespaceName, + ctrlSvc.WithRP(KeyReplicationEnabled): "true", + ctrlSvc.WithRP(KeyReplicationMode): replicationModeSync, + ctrlSvc.WithRP(KeyReplicationRPO): zeroRPO, + ctrlSvc.WithRP(KeyReplicationRemoteSystem): validRemoteSystemName, + ctrlSvc.WithRP(KeyReplicationIgnoreNamespaces): "true", + ctrlSvc.WithRP(KeyReplicationVGPrefix): "csi", }, }, })) }) - It("should create volume and update volumeGroup without policy, but policy exists", func() { + ginkgo.It("should fail create new volume with existing volumeGroup with policy and when IsWriteOrderConsistent is false - SYNC", func() { + clientMock.On("GetVolumeGroupByName", mock.Anything, validGroupNameSync). + Return(gopowerstore.VolumeGroup{ID: validGroupID, ProtectionPolicyID: validPolicyID, IsWriteOrderConsistent: false}, nil) + // Setting Replciation mode and corresponding attributes for SYNC + req.Parameters[ctrlSvc.WithRP(KeyReplicationMode)] = replicationModeSync + req.Parameters[ctrlSvc.WithRP(KeyReplicationRPO)] = zeroRPO + + req.Parameters[KeyCSIPVCName] = req.Name + req.Parameters[KeyCSIPVCNamespace] = validNamespaceName + clientMock.On("GetCustomHTTPHeaders").Return(api.NewSafeHeader().GetHeader()) + clientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(3.0), nil) + clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) + clientMock.On("CreateVolume", mock.Anything, mock.Anything).Return(gopowerstore.CreateResponse{ID: validBaseVolID}, nil) + clientMock.On("GetVolume", context.Background(), mock.Anything).Return(gopowerstore.Volume{ApplianceID: validApplianceID}, nil) + clientMock.On("GetAppliance", context.Background(), mock.Anything).Return(gopowerstore.ApplianceInstance{ServiceTag: validServiceTag}, nil) + + res, err := ctrlSvc.CreateVolume(context.Background(), req) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("can't apply protection policy with sync rule if volume group is not write-order consistent")) + }) + ginkgo.It("should create volume and update volumeGroup without policy, but policy exists - ASYNC", func() { + clientMock.On("GetVolumeByName", mock.Anything, mock.Anything).Return( + gopowerstore.Volume{}, errors.New("no vol found")) clientMock.On("GetVolumeGroupByName", mock.Anything, validGroupName). Return(gopowerstore.VolumeGroup{ID: validGroupID, ProtectionPolicyID: validPolicyID}, nil) EnsureProtectionPolicyExistsMock() - req.Parameters[controller.KeyCSIPVCName] = req.Name - req.Parameters[controller.KeyCSIPVCNamespace] = validNamespaceName - clientMock.On("GetCustomHTTPHeaders").Return(make(http.Header)) + req.Parameters[KeyCSIPVCName] = req.Name + req.Parameters[KeyCSIPVCNamespace] = validNamespaceName + clientMock.On("GetCustomHTTPHeaders").Return(api.NewSafeHeader().GetHeader()) + clientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(3.0), nil) + clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) + clientMock.On("CreateVolume", mock.Anything, mock.Anything).Return(gopowerstore.CreateResponse{ID: validBaseVolID}, nil) + clientMock.On("GetVolume", context.Background(), mock.Anything).Return(gopowerstore.Volume{ApplianceID: validApplianceID}, nil) + clientMock.On("GetAppliance", context.Background(), mock.Anything).Return(gopowerstore.ApplianceInstance{ServiceTag: validServiceTag}, nil) + + res, err := ctrlSvc.CreateVolume(context.Background(), req) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.CreateVolumeResponse{ + Volume: &csi.Volume{ + CapacityBytes: validVolSize, + VolumeId: filepath.Join(validBaseVolID, firstValidID, "scsi"), + VolumeContext: map[string]string{ + identifiers.KeyArrayVolumeName: "my-vol", + identifiers.KeyProtocol: "scsi", + identifiers.KeyArrayID: firstValidID, + identifiers.KeyVolumeDescription: req.Name + "-" + validNamespaceName, + identifiers.KeyServiceTag: validServiceTag, + KeyCSIPVCName: req.Name, + KeyCSIPVCNamespace: validNamespaceName, + ctrlSvc.WithRP(KeyReplicationEnabled): "true", + ctrlSvc.WithRP(KeyReplicationRPO): validRPO, + ctrlSvc.WithRP(KeyReplicationRemoteSystem): validRemoteSystemName, + ctrlSvc.WithRP(KeyReplicationIgnoreNamespaces): "true", + ctrlSvc.WithRP(KeyReplicationVGPrefix): "csi", + }, + }, + })) + }) + + ginkgo.It("should create volume and update volumeGroup without policy, but policy exists - SYNC", func() { + clientMock.On("GetVolumeByName", mock.Anything, mock.Anything).Return( + gopowerstore.Volume{}, errors.New("no vol found")) + clientMock.On("GetVolumeGroupByName", mock.Anything, validGroupNameSync). + Return(gopowerstore.VolumeGroup{ID: validGroupID, ProtectionPolicyID: validPolicyID, IsWriteOrderConsistent: true}, nil) + + EnsureProtectionPolicyExistsMockSync() + + req.Parameters[KeyCSIPVCName] = req.Name + req.Parameters[KeyCSIPVCNamespace] = validNamespaceName + // Setting Replciation mode and corresponding attributes for SYNC + req.Parameters[ctrlSvc.WithRP(KeyReplicationMode)] = replicationModeSync + req.Parameters[ctrlSvc.WithRP(KeyReplicationRPO)] = zeroRPO + clientMock.On("GetCustomHTTPHeaders").Return(api.NewSafeHeader().GetHeader()) clientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(3.0), nil) clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) clientMock.On("CreateVolume", mock.Anything, mock.Anything).Return(gopowerstore.CreateResponse{ID: validBaseVolID}, nil) @@ -393,30 +703,55 @@ var _ = Describe("CSIControllerService", func() { clientMock.On("GetAppliance", context.Background(), mock.Anything).Return(gopowerstore.ApplianceInstance{ServiceTag: validServiceTag}, nil) res, err := ctrlSvc.CreateVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.CreateVolumeResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.CreateVolumeResponse{ Volume: &csi.Volume{ CapacityBytes: validVolSize, VolumeId: filepath.Join(validBaseVolID, firstValidID, "scsi"), VolumeContext: map[string]string{ - common.KeyArrayVolumeName: "my-vol", - common.KeyProtocol: "scsi", - common.KeyArrayID: firstValidID, - common.KeyVolumeDescription: req.Name + "-" + validNamespaceName, - common.KeyServiceTag: validServiceTag, - controller.KeyCSIPVCName: req.Name, - controller.KeyCSIPVCNamespace: validNamespaceName, - ctrlSvc.WithRP(controller.KeyReplicationEnabled): "true", - ctrlSvc.WithRP(controller.KeyReplicationRPO): validRPO, - ctrlSvc.WithRP(controller.KeyReplicationRemoteSystem): validRemoteSystemName, - ctrlSvc.WithRP(controller.KeyReplicationIgnoreNamespaces): "true", - ctrlSvc.WithRP(controller.KeyReplicationVGPrefix): "csi", + identifiers.KeyArrayVolumeName: "my-vol", + identifiers.KeyProtocol: "scsi", + identifiers.KeyArrayID: firstValidID, + identifiers.KeyVolumeDescription: req.Name + "-" + validNamespaceName, + identifiers.KeyServiceTag: validServiceTag, + KeyCSIPVCName: req.Name, + KeyCSIPVCNamespace: validNamespaceName, + ctrlSvc.WithRP(KeyReplicationEnabled): "true", + ctrlSvc.WithRP(KeyReplicationMode): replicationModeSync, + ctrlSvc.WithRP(KeyReplicationRPO): zeroRPO, + ctrlSvc.WithRP(KeyReplicationRemoteSystem): validRemoteSystemName, + ctrlSvc.WithRP(KeyReplicationIgnoreNamespaces): "true", + ctrlSvc.WithRP(KeyReplicationVGPrefix): "csi", }, }, })) }) - It("should fail create volume and update volumeGroup if we can't ensure that policy exists", func() { + ginkgo.It("should fail create volume and update volumeGroup without policy, but policy exists when IsWriteOrderConsistent is false - SYNC", func() { + clientMock.On("GetVolumeGroupByName", mock.Anything, validGroupNameSync). + Return(gopowerstore.VolumeGroup{ID: validGroupID, ProtectionPolicyID: validPolicyID, IsWriteOrderConsistent: false}, nil) + + EnsureProtectionPolicyExistsMockSync() + + req.Parameters[KeyCSIPVCName] = req.Name + req.Parameters[KeyCSIPVCNamespace] = validNamespaceName + // Setting Replciation mode and corresponding attributes for SYNC + req.Parameters[ctrlSvc.WithRP(KeyReplicationMode)] = replicationModeSync + req.Parameters[ctrlSvc.WithRP(KeyReplicationRPO)] = zeroRPO + clientMock.On("GetCustomHTTPHeaders").Return(api.NewSafeHeader().GetHeader()) + clientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(3.0), nil) + clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) + clientMock.On("CreateVolume", mock.Anything, mock.Anything).Return(gopowerstore.CreateResponse{ID: validBaseVolID}, nil) + clientMock.On("GetVolume", context.Background(), mock.Anything).Return(gopowerstore.Volume{ApplianceID: validApplianceID}, nil) + clientMock.On("GetAppliance", context.Background(), mock.Anything).Return(gopowerstore.ApplianceInstance{ServiceTag: validServiceTag}, nil) + + res, err := ctrlSvc.CreateVolume(context.Background(), req) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("can't apply protection policy with sync rule if volume group is not write-order consistent")) + }) + + ginkgo.It("should fail create volume and update volumeGroup if we can't ensure that policy exists - ASYNC", func() { clientMock.On("GetVolumeGroupByName", mock.Anything, validGroupName). Return(gopowerstore.VolumeGroup{}, gopowerstore.NewNotFoundError()) @@ -424,373 +759,683 @@ var _ = Describe("CSIControllerService", func() { Return(gopowerstore.RemoteSystem{}, gopowerstore.NewHostIsNotExistError()) res, err := ctrlSvc.CreateVolume(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).NotTo(BeNil()) - Expect(err.Error()).To(ContainSubstring("can't ensure protection policy exists")) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("can't ensure protection policy exists")) + }) + + ginkgo.It("should fail create volume and update volumeGroup if we can't ensure that policy exists - SYNC", func() { + clientMock.On("GetVolumeGroupByName", mock.Anything, validGroupNameSync). + Return(gopowerstore.VolumeGroup{}, gopowerstore.NewNotFoundError()) + + clientMock.On("GetRemoteSystemByName", mock.Anything, validRemoteSystemName). + Return(gopowerstore.RemoteSystem{}, gopowerstore.NewHostIsNotExistError()) + + // Setting Replciation mode and corresponding attributes for SYNC + req.Parameters[ctrlSvc.WithRP(KeyReplicationMode)] = replicationModeSync + req.Parameters[ctrlSvc.WithRP(KeyReplicationRPO)] = zeroRPO + res, err := ctrlSvc.CreateVolume(context.Background(), req) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("can't ensure protection policy exists")) }) - It("should fail when rpo incorrect", func() { + ginkgo.It("should fail when rpo incorrect", func() { clientMock.On("CreateVolume", mock.Anything, mock.Anything).Return(gopowerstore.CreateResponse{ID: validBaseVolID}, nil) - req.Parameters[ctrlSvc.WithRP(controller.KeyReplicationRPO)] = "invalidRpo" + req.Parameters[ctrlSvc.WithRP(KeyReplicationRPO)] = "invalidRpo" + + res, err := ctrlSvc.CreateVolume(context.Background(), req) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("invalid RPO value")) + }) + + ginkgo.It("should fail when rpo not declared in parameters -ASYNC", func() { + delete(req.Parameters, ctrlSvc.WithRP(KeyReplicationRPO)) + + res, err := ctrlSvc.CreateVolume(context.Background(), req) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("replication mode is ASYNC but no RPO specified in storage class")) + }) + + ginkgo.It("should default RPO to Zero when mode is SYNC and RPO is not specified", func() { + delete(req.Parameters, ctrlSvc.WithRP(KeyReplicationRPO)) + clientMock.On("GetVolumeByName", mock.Anything, mock.Anything).Return( + gopowerstore.Volume{}, errors.New("no vol found")) + clientMock.On("GetVolumeGroupByName", mock.Anything, validGroupNameSync). + Return(gopowerstore.VolumeGroup{ID: validGroupID, ProtectionPolicyID: validPolicyID, IsWriteOrderConsistent: true}, nil) + + EnsureProtectionPolicyExistsMockSync() + + clientMock.On("GetCustomHTTPHeaders").Return(api.NewSafeHeader().GetHeader()) + clientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(3.0), nil) + clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) + clientMock.On("CreateVolume", mock.Anything, mock.Anything).Return(gopowerstore.CreateResponse{ID: validBaseVolID}, nil) + clientMock.On("GetVolume", context.Background(), mock.Anything).Return(gopowerstore.Volume{ApplianceID: validApplianceID}, nil) + clientMock.On("GetAppliance", context.Background(), mock.Anything).Return(gopowerstore.ApplianceInstance{ServiceTag: validServiceTag}, nil) + req.Parameters[ctrlSvc.WithRP(KeyReplicationMode)] = "SYNC" res, err := ctrlSvc.CreateVolume(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).NotTo(BeNil()) - Expect(err.Error()).To(ContainSubstring("invalid rpo value")) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.CreateVolumeResponse{ + Volume: &csi.Volume{ + CapacityBytes: validVolSize, + VolumeId: filepath.Join(validBaseVolID, firstValidID, "scsi"), + VolumeContext: map[string]string{ + identifiers.KeyArrayVolumeName: "my-vol", + identifiers.KeyProtocol: "scsi", + identifiers.KeyArrayID: firstValidID, + identifiers.KeyVolumeDescription: req.Name + "-" + validNamespaceName, + identifiers.KeyServiceTag: validServiceTag, + KeyCSIPVCName: req.Name, + KeyCSIPVCNamespace: validNamespaceName, + ctrlSvc.WithRP(KeyReplicationEnabled): "true", + ctrlSvc.WithRP(KeyReplicationMode): replicationModeSync, + ctrlSvc.WithRP(KeyReplicationRemoteSystem): validRemoteSystemName, + ctrlSvc.WithRP(KeyReplicationIgnoreNamespaces): "true", + ctrlSvc.WithRP(KeyReplicationVGPrefix): "csi", + }, + }, + })) + }) + ginkgo.It("should fail when remote system not declared in parameters", func() { + delete(req.Parameters, ctrlSvc.WithRP(KeyReplicationRemoteSystem)) + + res, err := ctrlSvc.CreateVolume(context.Background(), req) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("replication enabled but no remote system specified in storage class")) }) - It("should fail when rpo not declared in parameters", func() { + ginkgo.It("should fail when mode is incorrect", func() { + clientMock.On("CreateVolume", mock.Anything, mock.Anything).Return(gopowerstore.CreateResponse{ID: validBaseVolID}, nil) + + req.Parameters[ctrlSvc.WithRP(KeyReplicationMode)] = "SYNCMETRO" - delete(req.Parameters, ctrlSvc.WithRP(controller.KeyReplicationRPO)) + res, err := ctrlSvc.CreateVolume(context.Background(), req) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("invalid replication mode")) + }) + + ginkgo.It("should fail when mode is ASYNC and RPO is Zero", func() { + clientMock.On("CreateVolume", mock.Anything, mock.Anything).Return(gopowerstore.CreateResponse{ID: validBaseVolID}, nil) + req.Parameters[ctrlSvc.WithRP(KeyReplicationMode)] = replicationModeAsync + req.Parameters[ctrlSvc.WithRP(KeyReplicationRPO)] = zeroRPO res, err := ctrlSvc.CreateVolume(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).NotTo(BeNil()) - Expect(err.Error()).To(ContainSubstring("replication enabled but no RPO specified in storage class")) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("replication mode ASYNC requires RPO value to be non Zero")) + }) + + ginkgo.It("should fail when mode is SYNC and RPO is not Zero", func() { + clientMock.On("CreateVolume", mock.Anything, mock.Anything).Return(gopowerstore.CreateResponse{ID: validBaseVolID}, nil) + req.Parameters[ctrlSvc.WithRP(KeyReplicationMode)] = "SYNC" + req.Parameters[ctrlSvc.WithRP(KeyReplicationRPO)] = validRPO + res, err := ctrlSvc.CreateVolume(context.Background(), req) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("replication mode SYNC requires RPO value to be Zero")) }) - It("should fail when remote system not declared in parameters", func() { - delete(req.Parameters, ctrlSvc.WithRP(controller.KeyReplicationRemoteSystem)) + ginkgo.It("should fail when volume group prefix not declared in parameters", func() { + delete(req.Parameters, ctrlSvc.WithRP(KeyReplicationVGPrefix)) res, err := ctrlSvc.CreateVolume(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).NotTo(BeNil()) - Expect(err.Error()).To(ContainSubstring("replication enabled but no remote system specified in storage class")) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("replication enabled but no volume group prefix specified in storage class")) }) - It("should fail when volume group prefix not declared in parameters", func() { - delete(req.Parameters, ctrlSvc.WithRP(controller.KeyReplicationVGPrefix)) + ginkgo.It("should fail when invalid remote system is specified in parameters for metro volume", func() { + req.Parameters[ctrlSvc.WithRP(KeyReplicationMode)] = "METRO" + req.Parameters[ctrlSvc.WithRP(KeyReplicationRemoteSystem)] = "invalid" + + clientMock.On("GetRemoteSystemByName", mock.Anything, "invalid").Return(gopowerstore.RemoteSystem{}, gopowerstore.NewNotFoundError()) res, err := ctrlSvc.CreateVolume(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).NotTo(BeNil()) - Expect(err.Error()).To(ContainSubstring("replication enabled but no volume group prefix specified in storage class")) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("can't query remote system by name")) + }) + + ginkgo.Context("replication type is metro", func() { + var configureMetroRequest *gopowerstore.MetroConfig + + ginkgo.BeforeEach(func() { + // Default mock function functionality for metro replication. + // This base functionality can be overridden in the individual test implementation. + configureMetroRequest = &gopowerstore.MetroConfig{RemoteSystemID: validRemoteSystemID} + req.Parameters[ctrlSvc.WithRP(KeyReplicationMode)] = "METRO" + + clientMock.On("GetCustomHTTPHeaders").Return(api.NewSafeHeader().GetHeader()) + clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) + clientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(3.0), nil) + clientMock.On("GetRemoteSystemByName", mock.Anything, validRemoteSystemName).Return(gopowerstore.RemoteSystem{ + Name: validRemoteSystemName, + ID: validRemoteSystemID, + SerialNumber: secondValidID, + }, nil) + clientMock.On("CreateVolume", mock.Anything, mock.Anything).Return(gopowerstore.CreateResponse{ID: validBaseVolID}, nil) + }) + + ginkgo.It("should configure metro replication on volume", func() { + delete(req.Parameters, ctrlSvc.WithRP(KeyReplicationRPO)) + delete(req.Parameters, ctrlSvc.WithRP(KeyReplicationIgnoreNamespaces)) + delete(req.Parameters, ctrlSvc.WithRP(KeyReplicationVGPrefix)) + + clientMock.On("GetVolumeByName", mock.Anything, mock.Anything).Return( + gopowerstore.Volume{}, errors.New("no vol found")) + clientMock.On("ConfigureMetroVolume", mock.Anything, validBaseVolID, configureMetroRequest). + Return(gopowerstore.MetroSessionResponse{ID: validSessionID}, nil) + clientMock.On("GetVolume", context.Background(), mock.Anything). + Return(gopowerstore.Volume{ApplianceID: validApplianceID, MetroReplicationSessionID: validSessionID}, nil) + clientMock.On("GetAppliance", context.Background(), mock.Anything).Return(gopowerstore.ApplianceInstance{ServiceTag: validServiceTag}, nil) + clientMock.On("GetReplicationSessionByLocalResourceID", mock.Anything, validBaseVolID).Return(gopowerstore.ReplicationSession{ + LocalResourceID: validBaseVolID, + RemoteResourceID: validRemoteVolID, + ResourceType: "volume", + }, nil) + + res, err := ctrlSvc.CreateVolume(context.Background(), req) + + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.CreateVolumeResponse{ + Volume: &csi.Volume{ + CapacityBytes: validVolSize, + VolumeId: fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, firstValidID, "scsi", validRemoteVolID, secondValidID), + VolumeContext: map[string]string{ + identifiers.KeyArrayVolumeName: "my-vol", + identifiers.KeyProtocol: "scsi", + identifiers.KeyArrayID: firstValidID, + identifiers.KeyVolumeDescription: req.Name + "-" + validNamespaceName, + identifiers.KeyServiceTag: validServiceTag, + KeyCSIPVCName: req.Name, + KeyCSIPVCNamespace: validNamespaceName, + ctrlSvc.WithRP(KeyReplicationEnabled): "true", + ctrlSvc.WithRP(KeyReplicationMode): "METRO", + ctrlSvc.WithRP(KeyReplicationRemoteSystem): validRemoteSystemName, + }, + }, + })) + }) + + ginkgo.It("should continue metro replication on volume to support idempotency when metro was previously configured", func() { + delete(req.Parameters, ctrlSvc.WithRP(KeyReplicationRPO)) + delete(req.Parameters, ctrlSvc.WithRP(KeyReplicationIgnoreNamespaces)) + delete(req.Parameters, ctrlSvc.WithRP(KeyReplicationVGPrefix)) + + clientMock.On("GetVolumeByName", mock.Anything, mock.Anything).Return( + gopowerstore.Volume{}, errors.New("no vol found")) + + clientMock.On("ConfigureMetroVolume", mock.Anything, validBaseVolID, configureMetroRequest). + Return(gopowerstore.MetroSessionResponse{}, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{ + StatusCode: http.StatusBadRequest, + }, + }) + clientMock.On("GetVolume", context.Background(), mock.Anything). + Return(gopowerstore.Volume{ApplianceID: validApplianceID, MetroReplicationSessionID: validSessionID}, nil) + clientMock.On("GetAppliance", context.Background(), mock.Anything).Return(gopowerstore.ApplianceInstance{ServiceTag: validServiceTag}, nil) + clientMock.On("GetReplicationSessionByLocalResourceID", mock.Anything, validBaseVolID).Return(gopowerstore.ReplicationSession{ + LocalResourceID: validBaseVolID, + RemoteResourceID: validRemoteVolID, + ResourceType: "volume", + }, nil) + + res, err := ctrlSvc.CreateVolume(context.Background(), req) + + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.CreateVolumeResponse{ + Volume: &csi.Volume{ + CapacityBytes: validVolSize, + VolumeId: fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, firstValidID, "scsi", validRemoteVolID, secondValidID), + VolumeContext: map[string]string{ + identifiers.KeyArrayVolumeName: "my-vol", + identifiers.KeyProtocol: "scsi", + identifiers.KeyArrayID: firstValidID, + identifiers.KeyVolumeDescription: req.Name + "-" + validNamespaceName, + identifiers.KeyServiceTag: validServiceTag, + KeyCSIPVCName: req.Name, + KeyCSIPVCNamespace: validNamespaceName, + ctrlSvc.WithRP(KeyReplicationEnabled): "true", + ctrlSvc.WithRP(KeyReplicationMode): "METRO", + ctrlSvc.WithRP(KeyReplicationRemoteSystem): validRemoteSystemName, + }, + }, + })) + }) + + ginkgo.It("should configure not metro replication on volume due to CSM DR not avilable", func() { + // setting the value of IsCSMDREnabled parameter of ControllerService to false + ctrlSvc.IsCSMDREnabled = false + _ = ctrlSvc.Init() + delete(req.Parameters, ctrlSvc.WithRP(KeyReplicationRPO)) + delete(req.Parameters, ctrlSvc.WithRP(KeyReplicationIgnoreNamespaces)) + delete(req.Parameters, ctrlSvc.WithRP(KeyReplicationVGPrefix)) + + clientMock.On("GetVolumeByName", mock.Anything, mock.Anything).Return( + gopowerstore.Volume{}, errors.New("no vol found")) + ctrlSvc.IsCSMDREnabled = false + res, err := ctrlSvc.CreateVolume(context.Background(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("Metro replication mode requires CSM-DR to be enabled")) + // Resetting the value of IsCSMDREnabled parameter of ControllerService to true after our tests are completed + ctrlSvc.IsCSMDREnabled = true + _ = ctrlSvc.Init() + }) + + ginkgo.It("should fail to configure metro replication on volume if the volume cannot be found", func() { + clientMock.On("GetVolumeByName", mock.Anything, mock.Anything).Return( + gopowerstore.Volume{}, errors.New("no vol found")) + // Return volume not found error when trying to configure a metro session for that volume + clientMock.On("ConfigureMetroVolume", mock.Anything, validBaseVolID, configureMetroRequest). + Return(gopowerstore.MetroSessionResponse{}, gopowerstore.NewNotFoundError()) + clientMock.On("GetVolume", context.Background(), mock.Anything).Return(gopowerstore.Volume{ID: validBaseVolID, MetroReplicationSessionID: validSessionID}, nil) + + res, err := ctrlSvc.CreateVolume(context.Background(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("can't configure metro on volume")) + }) + + ginkgo.It("should fail when invalid remote system is specified in parameters for metro volume", func() { + req.Parameters[ctrlSvc.WithRP(KeyReplicationRemoteSystem)] = "invalid" + + // return 404 Not Found error when querying for the remote system + clientMock.On("GetRemoteSystemByName", mock.Anything, "invalid").Return(gopowerstore.RemoteSystem{}, gopowerstore.NewNotFoundError()) + + res, err := ctrlSvc.CreateVolume(context.Background(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("can't query remote system by name")) + }) + + ginkgo.It("should fail if it can't find the replication session", func() { + clientMock.On("GetVolumeByName", mock.Anything, mock.Anything).Return( + gopowerstore.Volume{}, errors.New("no vol found")) + clientMock.On("ConfigureMetroVolume", mock.Anything, validBaseVolID, configureMetroRequest). + Return(gopowerstore.MetroSessionResponse{ID: validSessionID}, nil) + clientMock.On("GetVolume", context.Background(), mock.Anything). + Return(gopowerstore.Volume{ApplianceID: validApplianceID, MetroReplicationSessionID: validSessionID}, nil) + clientMock.On("GetAppliance", context.Background(), mock.Anything).Return(gopowerstore.ApplianceInstance{ServiceTag: validServiceTag}, nil) + + // Return 404 Not Found error when querying for the replication session + clientMock.On("GetReplicationSessionByLocalResourceID", mock.Anything, validBaseVolID).Return(gopowerstore.ReplicationSession{}, gopowerstore.NewNotFoundError()) + + res, err := ctrlSvc.CreateVolume(context.Background(), req) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("could not get metro replication session")) + }) + + ginkgo.It("should fail if the replication session resource type is incorrect", func() { + clientMock.On("GetVolumeByName", mock.Anything, mock.Anything).Return( + gopowerstore.Volume{}, errors.New("no vol found")) + clientMock.On("ConfigureMetroVolume", mock.Anything, validBaseVolID, configureMetroRequest). + Return(gopowerstore.MetroSessionResponse{ID: validSessionID}, nil) + clientMock.On("GetVolume", context.Background(), mock.Anything). + Return(gopowerstore.Volume{ApplianceID: validApplianceID, MetroReplicationSessionID: validSessionID}, nil) + clientMock.On("GetAppliance", context.Background(), mock.Anything).Return(gopowerstore.ApplianceInstance{ServiceTag: validServiceTag}, nil) + + // Return a bad resource type for the replication session; "file_system" + resourceType := "file_system" + clientMock.On("GetReplicationSessionByLocalResourceID", mock.Anything, validBaseVolID). + Return(gopowerstore.ReplicationSession{ResourceType: resourceType, ID: validSessionID}, nil) + + res, err := ctrlSvc.CreateVolume(context.Background(), req) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring((fmt.Sprintf("replication session %s has a resource type %s, wanted type 'volume'", + validSessionID, resourceType)))) + }) }) }) - When("creating nfs volume", func() { - It("should successfully create nfs volume", func() { + ginkgo.When("creating nfs volume", func() { + ginkgo.It("should successfully create nfs volume", func() { clientMock.On("GetNASByName", mock.Anything, validNasName).Return(gopowerstore.NAS{ID: validNasID}, nil) clientMock.On("CreateFS", mock.Anything, mock.Anything).Return(gopowerstore.CreateResponse{ID: validBaseVolID}, nil) clientMock.On("GetFS", context.Background(), mock.Anything).Return(gopowerstore.FileSystem{NasServerID: validNasID}, nil) - clientMock.On("GetNAS", context.Background(), mock.Anything).Return(gopowerstore.NAS{CurrentNodeId: validNodeID}, nil) + clientMock.On("GetNAS", context.Background(), mock.Anything).Return(gopowerstore.NAS{CurrentNodeID: validNodeID}, nil) clientMock.On("GetApplianceByName", context.Background(), mock.Anything).Return(gopowerstore.ApplianceInstance{ServiceTag: validServiceTag}, nil) + clientMock.On("GetInProgressJobsByFsName", context.Background(), mock.Anything).Return([]gopowerstore.Job{}, nil) + clientMock.On("GetFSByName", mock.Anything, mock.Anything).Return(gopowerstore.FileSystem{}, errors.New("not found")) req := getTypicalCreateVolumeNFSRequest("my-vol", validVolSize) - req.Parameters[common.KeyArrayID] = secondValidID + req.Parameters[identifiers.KeyArrayID] = secondValidID - req.Parameters[controller.KeyCSIPVCName] = req.Name - req.Parameters[controller.KeyCSIPVCNamespace] = validNamespaceName + req.Parameters[KeyCSIPVCName] = req.Name + req.Parameters[KeyCSIPVCNamespace] = validNamespaceName res, err := ctrlSvc.CreateVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.CreateVolumeResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.CreateVolumeResponse{ Volume: &csi.Volume{ CapacityBytes: validVolSize, VolumeId: filepath.Join(validBaseVolID, secondValidID, "nfs"), VolumeContext: map[string]string{ - common.KeyArrayVolumeName: "my-vol", - common.KeyProtocol: "nfs", - common.KeyArrayID: secondValidID, - common.KeyNfsACL: "A::OWNER@:RWX", - common.KeyNasName: validNasName, - common.KeyVolumeDescription: req.Name + "-" + validNamespaceName, - common.KeyServiceTag: validServiceTag, - controller.KeyCSIPVCName: req.Name, - controller.KeyCSIPVCNamespace: validNamespaceName, + identifiers.KeyArrayVolumeName: "my-vol", + identifiers.KeyProtocol: "nfs", + identifiers.KeyArrayID: secondValidID, + identifiers.KeyNfsACL: "A::OWNER@:RWX", + identifiers.KeyNasName: validNasName, + identifiers.KeyVolumeDescription: req.Name + "-" + validNamespaceName, + identifiers.KeyServiceTag: validServiceTag, + KeyCSIPVCName: req.Name, + KeyCSIPVCNamespace: validNamespaceName, }, - AccessibleTopology: []*csi.Topology{{Segments: map[string]string{common.Name + "/" + ctrlSvc.Arrays()[secondValidID].GetIP() + "-nfs": "true"}}}, + AccessibleTopology: []*csi.Topology{{Segments: map[string]string{identifiers.Name + "/" + ctrlSvc.Arrays()[secondValidID].GetIP() + "-nfs": "true"}}}, }, })) }) - It("should successfully create nfs volume & all vol attribute should get set", func() { + ginkgo.It("should successfully create nfs volume & all vol attribute should get set", func() { clientMock.On("GetNASByName", mock.Anything, validNasName).Return(gopowerstore.NAS{ID: validNasID}, nil) clientMock.On("CreateFS", mock.Anything, mock.Anything).Return(gopowerstore.CreateResponse{ID: validBaseVolID}, nil) clientMock.On("GetFS", context.Background(), mock.Anything).Return(gopowerstore.FileSystem{NasServerID: validNasID}, nil) - clientMock.On("GetNAS", context.Background(), mock.Anything).Return(gopowerstore.NAS{CurrentNodeId: validNodeID}, nil) + clientMock.On("GetNAS", context.Background(), mock.Anything).Return(gopowerstore.NAS{CurrentNodeID: validNodeID}, nil) clientMock.On("GetApplianceByName", context.Background(), mock.Anything).Return(gopowerstore.ApplianceInstance{ServiceTag: validServiceTag}, nil) + clientMock.On("GetInProgressJobsByFsName", context.Background(), mock.Anything).Return([]gopowerstore.Job{}, nil) + clientMock.On("GetFSByName", mock.Anything, mock.Anything).Return(gopowerstore.FileSystem{}, errors.New("not found")) req := getTypicalCreateVolumeNFSRequest("my-vol", validVolSize) - req.Parameters[common.KeyArrayID] = secondValidID - - req.Parameters[controller.KeyCSIPVCName] = req.Name - req.Parameters[controller.KeyCSIPVCNamespace] = validNamespaceName - req.Parameters[common.KeyVolumeDescription] = "Vol-description" - req.Parameters[common.KeyConfigType] = "ConfigType_A" - req.Parameters[common.KeyAccessPolicy] = "AccessPolicy_A" - req.Parameters[common.KeyLockingPolicy] = "KeyLockingPolicy_A" - req.Parameters[common.KeyFolderRenamePolicy] = "KeyFolderRenamePolicy" - req.Parameters[common.KeyIsAsyncMtimeEnabled] = "true" - req.Parameters[common.KeyProtectionPolicyID] = "KeyProtectionPolicyID" - req.Parameters[common.KeyFileEventsPublishingMode] = "KeyFileEventsPublishingMode" - req.Parameters[common.KeyHostIoSize] = "VMware_16K" - req.Parameters[common.KeyFlrCreateMode] = "KeyFlrCreateMode" - req.Parameters[common.KeyFlrDefaultRetention] = "KeyFlrDefaultRetention" - req.Parameters[common.KeyFlrMinRetention] = "KeyFlrMinRetention" - req.Parameters[common.KeyFlrMaxRetention] = "KeyFlrMaxRetention" + req.Parameters[identifiers.KeyArrayID] = secondValidID + + req.Parameters[KeyCSIPVCName] = req.Name + req.Parameters[KeyCSIPVCNamespace] = validNamespaceName + req.Parameters[identifiers.KeyVolumeDescription] = "Vol-description" + req.Parameters[identifiers.KeyConfigType] = "ConfigType_A" + req.Parameters[identifiers.KeyAccessPolicy] = "AccessPolicy_A" + req.Parameters[identifiers.KeyLockingPolicy] = "KeyLockingPolicy_A" + req.Parameters[identifiers.KeyFolderRenamePolicy] = "KeyFolderRenamePolicy" + req.Parameters[identifiers.KeyIsAsyncMtimeEnabled] = "true" + req.Parameters[identifiers.KeyProtectionPolicyID] = "KeyProtectionPolicyID" + req.Parameters[identifiers.KeyFileEventsPublishingMode] = "KeyFileEventsPublishingMode" + req.Parameters[identifiers.KeyHostIoSize] = "VMware_16K" + req.Parameters[identifiers.KeyFlrCreateMode] = "KeyFlrCreateMode" + req.Parameters[identifiers.KeyFlrDefaultRetention] = "KeyFlrDefaultRetention" + req.Parameters[identifiers.KeyFlrMinRetention] = "KeyFlrMinRetention" + req.Parameters[identifiers.KeyFlrMaxRetention] = "KeyFlrMaxRetention" res, err := ctrlSvc.CreateVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.CreateVolumeResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.CreateVolumeResponse{ Volume: &csi.Volume{ CapacityBytes: validVolSize, VolumeId: filepath.Join(validBaseVolID, secondValidID, "nfs"), VolumeContext: map[string]string{ - common.KeyArrayVolumeName: "my-vol", - common.KeyProtocol: "nfs", - common.KeyArrayID: secondValidID, - common.KeyNfsACL: "A::OWNER@:RWX", - common.KeyNasName: validNasName, - common.KeyVolumeDescription: "Vol-description", - common.KeyConfigType: "ConfigType_A", - common.KeyAccessPolicy: "AccessPolicy_A", - common.KeyLockingPolicy: "KeyLockingPolicy_A", - common.KeyFolderRenamePolicy: "KeyFolderRenamePolicy", - common.KeyIsAsyncMtimeEnabled: "true", - common.KeyProtectionPolicyID: "KeyProtectionPolicyID", - common.KeyFileEventsPublishingMode: "KeyFileEventsPublishingMode", - common.KeyHostIoSize: "VMware_16K", - common.KeyFlrCreateMode: "KeyFlrCreateMode", - common.KeyFlrDefaultRetention: "KeyFlrDefaultRetention", - common.KeyFlrMinRetention: "KeyFlrMinRetention", - common.KeyFlrMaxRetention: "KeyFlrMaxRetention", - common.KeyServiceTag: validServiceTag, - controller.KeyCSIPVCName: req.Name, - controller.KeyCSIPVCNamespace: validNamespaceName, + identifiers.KeyArrayVolumeName: "my-vol", + identifiers.KeyProtocol: "nfs", + identifiers.KeyArrayID: secondValidID, + identifiers.KeyNfsACL: "A::OWNER@:RWX", + identifiers.KeyNasName: validNasName, + identifiers.KeyVolumeDescription: "Vol-description", + identifiers.KeyConfigType: "ConfigType_A", + identifiers.KeyAccessPolicy: "AccessPolicy_A", + identifiers.KeyLockingPolicy: "KeyLockingPolicy_A", + identifiers.KeyFolderRenamePolicy: "KeyFolderRenamePolicy", + identifiers.KeyIsAsyncMtimeEnabled: "true", + identifiers.KeyProtectionPolicyID: "KeyProtectionPolicyID", + identifiers.KeyFileEventsPublishingMode: "KeyFileEventsPublishingMode", + identifiers.KeyHostIoSize: "VMware_16K", + identifiers.KeyFlrCreateMode: "KeyFlrCreateMode", + identifiers.KeyFlrDefaultRetention: "KeyFlrDefaultRetention", + identifiers.KeyFlrMinRetention: "KeyFlrMinRetention", + identifiers.KeyFlrMaxRetention: "KeyFlrMaxRetention", + identifiers.KeyServiceTag: validServiceTag, + KeyCSIPVCName: req.Name, + KeyCSIPVCNamespace: validNamespaceName, }, - AccessibleTopology: []*csi.Topology{{Segments: map[string]string{common.Name + "/" + ctrlSvc.Arrays()[secondValidID].GetIP() + "-nfs": "true"}}}, + AccessibleTopology: []*csi.Topology{{Segments: map[string]string{identifiers.Name + "/" + ctrlSvc.Arrays()[secondValidID].GetIP() + "-nfs": "true"}}}, }, })) }) - When("creating nfs volume with NFS acls in array config and storage class", func() { - It("should successfully create nfs volume with storage class NFS acls in volume response", func() { + ginkgo.When("creating nfs volume with NFS acls in array config and storage class", func() { + ginkgo.It("should successfully create nfs volume with storage class NFS acls in volume response", func() { clientMock.On("GetNASByName", mock.Anything, validNasName).Return(gopowerstore.NAS{ID: validNasID}, nil) clientMock.On("CreateFS", mock.Anything, mock.Anything).Return(gopowerstore.CreateResponse{ID: validBaseVolID}, nil) - clientMock.On("GetNfsServer", mock.Anything, mock.Anything).Return(gopowerstore.NFSServerInstance{Id: validNfsServerID, IsNFSv4Enabled: true}, nil) + clientMock.On("GetNfsServer", mock.Anything, mock.Anything).Return(gopowerstore.NFSServerInstance{ID: validNfsServerID, IsNFSv4Enabled: true}, nil) clientMock.On("GetFS", context.Background(), mock.Anything).Return(gopowerstore.FileSystem{NasServerID: validNasID}, nil) - clientMock.On("GetNAS", context.Background(), mock.Anything).Return(gopowerstore.NAS{CurrentNodeId: validNodeID}, nil) + clientMock.On("GetNAS", context.Background(), mock.Anything).Return(gopowerstore.NAS{CurrentNodeID: validNodeID}, nil) clientMock.On("GetApplianceByName", context.Background(), mock.Anything).Return(gopowerstore.ApplianceInstance{ServiceTag: validServiceTag}, nil) + clientMock.On("GetInProgressJobsByFsName", context.Background(), mock.Anything).Return([]gopowerstore.Job{}, nil) + clientMock.On("GetFSByName", mock.Anything, mock.Anything).Return(gopowerstore.FileSystem{}, errors.New("not found")) ctrlSvc.Arrays()[secondValidID].NfsAcls = "A::GROUP@:RWX" req := getTypicalCreateVolumeNFSRequest("my-vol", validVolSize) - req.Parameters[common.KeyNfsACL] = "0777" - req.Parameters[common.KeyArrayID] = secondValidID - req.Parameters[controller.KeyCSIPVCName] = req.Name - req.Parameters[controller.KeyCSIPVCNamespace] = validNamespaceName + req.Parameters[identifiers.KeyNfsACL] = "0777" + req.Parameters[identifiers.KeyArrayID] = secondValidID + req.Parameters[KeyCSIPVCName] = req.Name + req.Parameters[KeyCSIPVCNamespace] = validNamespaceName res, err := ctrlSvc.CreateVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.CreateVolumeResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.CreateVolumeResponse{ Volume: &csi.Volume{ CapacityBytes: validVolSize, VolumeId: filepath.Join(validBaseVolID, secondValidID, "nfs"), VolumeContext: map[string]string{ - common.KeyArrayVolumeName: "my-vol", - common.KeyProtocol: "nfs", - common.KeyArrayID: secondValidID, - common.KeyNfsACL: "0777", - common.KeyNasName: validNasName, - common.KeyVolumeDescription: req.Name + "-" + validNamespaceName, - common.KeyServiceTag: validServiceTag, - controller.KeyCSIPVCName: req.Name, - controller.KeyCSIPVCNamespace: validNamespaceName, - }, - AccessibleTopology: []*csi.Topology{{Segments: map[string]string{common.Name + "/" + ctrlSvc.Arrays()[secondValidID].GetIP() + "-nfs": "true"}}}, + identifiers.KeyArrayVolumeName: "my-vol", + identifiers.KeyProtocol: "nfs", + identifiers.KeyArrayID: secondValidID, + identifiers.KeyNfsACL: "0777", + identifiers.KeyNasName: validNasName, + identifiers.KeyVolumeDescription: req.Name + "-" + validNamespaceName, + identifiers.KeyServiceTag: validServiceTag, + KeyCSIPVCName: req.Name, + KeyCSIPVCNamespace: validNamespaceName, + }, + AccessibleTopology: []*csi.Topology{{Segments: map[string]string{identifiers.Name + "/" + ctrlSvc.Arrays()[secondValidID].GetIP() + "-nfs": "true"}}}, }, })) }) }) - When("creating nfs volume with NFS acls in array config and not in storage class", func() { - It("should successfully create nfs volume with array config NFS acls in volume response", func() { + ginkgo.When("creating nfs volume with NFS acls in array config and not in storage class", func() { + ginkgo.It("should successfully create nfs volume with array config NFS acls in volume response", func() { clientMock.On("GetNASByName", mock.Anything, validNasName).Return(gopowerstore.NAS{ID: validNasID}, nil) clientMock.On("CreateFS", mock.Anything, mock.Anything).Return(gopowerstore.CreateResponse{ID: validBaseVolID}, nil) - clientMock.On("GetNfsServer", mock.Anything, mock.Anything).Return(gopowerstore.NFSServerInstance{Id: validNfsServerID, IsNFSv4Enabled: true}, nil) + clientMock.On("GetNfsServer", mock.Anything, mock.Anything).Return(gopowerstore.NFSServerInstance{ID: validNfsServerID, IsNFSv4Enabled: true}, nil) clientMock.On("GetFS", context.Background(), mock.Anything).Return(gopowerstore.FileSystem{NasServerID: validNasID}, nil) - clientMock.On("GetNAS", context.Background(), mock.Anything).Return(gopowerstore.NAS{CurrentNodeId: validNodeID}, nil) + clientMock.On("GetNAS", context.Background(), mock.Anything).Return(gopowerstore.NAS{CurrentNodeID: validNodeID}, nil) clientMock.On("GetApplianceByName", context.Background(), mock.Anything).Return(gopowerstore.ApplianceInstance{ServiceTag: validServiceTag}, nil) + clientMock.On("GetInProgressJobsByFsName", context.Background(), mock.Anything).Return([]gopowerstore.Job{}, nil) + clientMock.On("GetFSByName", mock.Anything, mock.Anything).Return(gopowerstore.FileSystem{}, errors.New("not found")) ctrlSvc.Arrays()[secondValidID].NfsAcls = "A::GROUP@:RWX" req := getTypicalCreateVolumeNFSRequest("my-vol", validVolSize) - req.Parameters[common.KeyArrayID] = secondValidID - req.Parameters[controller.KeyCSIPVCName] = req.Name - req.Parameters[controller.KeyCSIPVCNamespace] = validNamespaceName + req.Parameters[identifiers.KeyArrayID] = secondValidID + req.Parameters[KeyCSIPVCName] = req.Name + req.Parameters[KeyCSIPVCNamespace] = validNamespaceName res, err := ctrlSvc.CreateVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.CreateVolumeResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.CreateVolumeResponse{ Volume: &csi.Volume{ CapacityBytes: validVolSize, VolumeId: filepath.Join(validBaseVolID, secondValidID, "nfs"), VolumeContext: map[string]string{ - common.KeyArrayVolumeName: "my-vol", - common.KeyProtocol: "nfs", - common.KeyArrayID: secondValidID, - common.KeyNfsACL: "A::GROUP@:RWX", - common.KeyNasName: validNasName, - common.KeyVolumeDescription: req.Name + "-" + validNamespaceName, - common.KeyServiceTag: validServiceTag, - controller.KeyCSIPVCName: req.Name, - controller.KeyCSIPVCNamespace: validNamespaceName, - }, - AccessibleTopology: []*csi.Topology{{Segments: map[string]string{common.Name + "/" + ctrlSvc.Arrays()[secondValidID].GetIP() + "-nfs": "true"}}}, + identifiers.KeyArrayVolumeName: "my-vol", + identifiers.KeyProtocol: "nfs", + identifiers.KeyArrayID: secondValidID, + identifiers.KeyNfsACL: "A::GROUP@:RWX", + identifiers.KeyNasName: validNasName, + identifiers.KeyVolumeDescription: req.Name + "-" + validNamespaceName, + identifiers.KeyServiceTag: validServiceTag, + KeyCSIPVCName: req.Name, + KeyCSIPVCNamespace: validNamespaceName, + }, + AccessibleTopology: []*csi.Topology{{Segments: map[string]string{identifiers.Name + "/" + ctrlSvc.Arrays()[secondValidID].GetIP() + "-nfs": "true"}}}, }, })) }) }) - When("creating nfs volume with NFS acls not in array config and not in storage class", func() { - It("should successfully create nfs volume with default NFS acls in volume response", func() { + ginkgo.When("creating nfs volume with NFS acls not in array config and not in storage class", func() { + ginkgo.It("should successfully create nfs volume with default NFS acls in volume response", func() { clientMock.On("GetNASByName", mock.Anything, validNasName).Return(gopowerstore.NAS{ID: validNasID}, nil) clientMock.On("CreateFS", mock.Anything, mock.Anything).Return(gopowerstore.CreateResponse{ID: validBaseVolID}, nil) - clientMock.On("GetNfsServer", mock.Anything, mock.Anything).Return(gopowerstore.NFSServerInstance{Id: validNfsServerID, IsNFSv4Enabled: true}, nil) + clientMock.On("GetNfsServer", mock.Anything, mock.Anything).Return(gopowerstore.NFSServerInstance{ID: validNfsServerID, IsNFSv4Enabled: true}, nil) clientMock.On("GetFS", context.Background(), mock.Anything).Return(gopowerstore.FileSystem{NasServerID: validNasID}, nil) - clientMock.On("GetNAS", context.Background(), mock.Anything).Return(gopowerstore.NAS{CurrentNodeId: validNodeID}, nil) + clientMock.On("GetNAS", context.Background(), mock.Anything).Return(gopowerstore.NAS{CurrentNodeID: validNodeID}, nil) clientMock.On("GetApplianceByName", context.Background(), mock.Anything).Return(gopowerstore.ApplianceInstance{ServiceTag: validServiceTag}, nil) + clientMock.On("GetInProgressJobsByFsName", context.Background(), mock.Anything).Return([]gopowerstore.Job{}, nil) + clientMock.On("GetFSByName", mock.Anything, mock.Anything).Return(gopowerstore.FileSystem{}, errors.New("not found")) req := getTypicalCreateVolumeNFSRequest("my-vol", validVolSize) - req.Parameters[common.KeyArrayID] = secondValidID - req.Parameters[controller.KeyCSIPVCName] = req.Name - req.Parameters[controller.KeyCSIPVCNamespace] = validNamespaceName + req.Parameters[identifiers.KeyArrayID] = secondValidID + req.Parameters[KeyCSIPVCName] = req.Name + req.Parameters[KeyCSIPVCNamespace] = validNamespaceName res, err := ctrlSvc.CreateVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.CreateVolumeResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.CreateVolumeResponse{ Volume: &csi.Volume{ CapacityBytes: validVolSize, VolumeId: filepath.Join(validBaseVolID, secondValidID, "nfs"), VolumeContext: map[string]string{ - common.KeyArrayVolumeName: "my-vol", - common.KeyProtocol: "nfs", - common.KeyArrayID: secondValidID, - common.KeyNfsACL: "A::OWNER@:RWX", - common.KeyNasName: validNasName, - common.KeyVolumeDescription: req.Name + "-" + validNamespaceName, - common.KeyServiceTag: validServiceTag, - controller.KeyCSIPVCName: req.Name, - controller.KeyCSIPVCNamespace: validNamespaceName, - }, - AccessibleTopology: []*csi.Topology{{Segments: map[string]string{common.Name + "/" + ctrlSvc.Arrays()[secondValidID].GetIP() + "-nfs": "true"}}}, + identifiers.KeyArrayVolumeName: "my-vol", + identifiers.KeyProtocol: "nfs", + identifiers.KeyArrayID: secondValidID, + identifiers.KeyNfsACL: "A::OWNER@:RWX", + identifiers.KeyNasName: validNasName, + identifiers.KeyVolumeDescription: req.Name + "-" + validNamespaceName, + identifiers.KeyServiceTag: validServiceTag, + KeyCSIPVCName: req.Name, + KeyCSIPVCNamespace: validNamespaceName, + }, + AccessibleTopology: []*csi.Topology{{Segments: map[string]string{identifiers.Name + "/" + ctrlSvc.Arrays()[secondValidID].GetIP() + "-nfs": "true"}}}, }, })) }) }) - When("creating nfs volume with NFS acls in not in secrets & default", func() { - It("should successfully create nfs volume with empty NFS acls in volume response", func() { + ginkgo.When("creating nfs volume with NFS acls in not in secrets & default", func() { + ginkgo.It("should successfully create nfs volume with empty NFS acls in volume response", func() { clientMock.On("GetNASByName", mock.Anything, validNasName).Return(gopowerstore.NAS{ID: validNasID}, nil) clientMock.On("CreateFS", mock.Anything, mock.Anything).Return(gopowerstore.CreateResponse{ID: validBaseVolID}, nil) - clientMock.On("GetNfsServer", mock.Anything, mock.Anything).Return(gopowerstore.NFSServerInstance{Id: validNfsServerID, IsNFSv4Enabled: true}, nil) + clientMock.On("GetNfsServer", mock.Anything, mock.Anything).Return(gopowerstore.NFSServerInstance{ID: validNfsServerID, IsNFSv4Enabled: true}, nil) clientMock.On("GetFS", context.Background(), mock.Anything).Return(gopowerstore.FileSystem{NasServerID: validNasID}, nil) - clientMock.On("GetNAS", context.Background(), mock.Anything).Return(gopowerstore.NAS{CurrentNodeId: validNodeID}, nil) + clientMock.On("GetNAS", context.Background(), mock.Anything).Return(gopowerstore.NAS{CurrentNodeID: validNodeID}, nil) clientMock.On("GetApplianceByName", context.Background(), mock.Anything).Return(gopowerstore.ApplianceInstance{ServiceTag: validServiceTag}, nil) + clientMock.On("GetInProgressJobsByFsName", context.Background(), mock.Anything).Return([]gopowerstore.Job{}, nil) + clientMock.On("GetFSByName", mock.Anything, mock.Anything).Return(gopowerstore.FileSystem{}, errors.New("not found")) req := getTypicalCreateVolumeNFSRequest("my-vol", validVolSize) - req.Parameters[common.KeyArrayID] = secondValidID - req.Parameters[controller.KeyCSIPVCName] = req.Name - req.Parameters[controller.KeyCSIPVCNamespace] = validNamespaceName + req.Parameters[identifiers.KeyArrayID] = secondValidID + req.Parameters[KeyCSIPVCName] = req.Name + req.Parameters[KeyCSIPVCNamespace] = validNamespaceName ctrlSvc.Arrays()[secondValidID].NfsAcls = "" - csictx.Setenv(context.Background(), common.EnvNfsAcls, "") + csictx.Setenv(context.Background(), identifiers.EnvNfsAcls, "") _ = ctrlSvc.Init() res, err := ctrlSvc.CreateVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.CreateVolumeResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.CreateVolumeResponse{ Volume: &csi.Volume{ CapacityBytes: validVolSize, VolumeId: filepath.Join(validBaseVolID, secondValidID, "nfs"), VolumeContext: map[string]string{ - common.KeyArrayVolumeName: "my-vol", - common.KeyProtocol: "nfs", - common.KeyArrayID: secondValidID, - common.KeyNfsACL: "", - common.KeyNasName: validNasName, - common.KeyVolumeDescription: req.Name + "-" + validNamespaceName, - common.KeyServiceTag: validServiceTag, - controller.KeyCSIPVCName: req.Name, - controller.KeyCSIPVCNamespace: validNamespaceName, - }, - AccessibleTopology: []*csi.Topology{{Segments: map[string]string{common.Name + "/" + ctrlSvc.Arrays()[secondValidID].GetIP() + "-nfs": "true"}}}, + identifiers.KeyArrayVolumeName: "my-vol", + identifiers.KeyProtocol: "nfs", + identifiers.KeyArrayID: secondValidID, + identifiers.KeyNfsACL: "", + identifiers.KeyNasName: validNasName, + identifiers.KeyVolumeDescription: req.Name + "-" + validNamespaceName, + identifiers.KeyServiceTag: validServiceTag, + KeyCSIPVCName: req.Name, + KeyCSIPVCNamespace: validNamespaceName, + }, + AccessibleTopology: []*csi.Topology{{Segments: map[string]string{identifiers.Name + "/" + ctrlSvc.Arrays()[secondValidID].GetIP() + "-nfs": "true"}}}, }, })) }) }) - When("creating nfs volume without nfs topology in AccessibilityRequirements", func() { - It("should fail", func() { + ginkgo.When("creating nfs volume without nfs topology in AccessibilityRequirements", func() { + ginkgo.It("should fail", func() { req := getTypicalCreateVolumeNFSRequest("my-vol", validVolSize) - req.Parameters[common.KeyArrayID] = secondValidID + req.Parameters[identifiers.KeyArrayID] = secondValidID - req.Parameters[controller.KeyCSIPVCName] = req.Name - req.Parameters[controller.KeyCSIPVCNamespace] = validNamespaceName + req.Parameters[KeyCSIPVCName] = req.Name + req.Parameters[KeyCSIPVCNamespace] = validNamespaceName - iscsiTopology := &csi.Topology{Segments: map[string]string{common.Name + "/" + ctrlSvc.Arrays()[secondValidID].GetIP() + "-iscsi": "true"}} + iscsiTopology := &csi.Topology{Segments: map[string]string{identifiers.Name + "/" + ctrlSvc.Arrays()[secondValidID].GetIP() + "-iscsi": "true"}} preferred := []*csi.Topology{iscsiTopology} accessibilityRequirements := &csi.TopologyRequirement{Preferred: preferred} req.AccessibilityRequirements = accessibilityRequirements res, err := ctrlSvc.CreateVolume(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring("invalid topology requested for NFS Volume. Please validate your storage class has nfs topology.")) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("invalid topology requested for NFS Volume. Please validate your storage class has nfs topology.")) }) }) - When("creating nfs volume with more than one topology in AccessibilityRequirements", func() { - It("should return only nfs topology", func() { + ginkgo.When("creating nfs volume with more than one topology in AccessibilityRequirements", func() { + ginkgo.It("should return only nfs topology", func() { clientMock.On("GetNASByName", mock.Anything, validNasName).Return(gopowerstore.NAS{ID: validNasID}, nil) clientMock.On("CreateFS", mock.Anything, mock.Anything).Return(gopowerstore.CreateResponse{ID: validBaseVolID}, nil) clientMock.On("GetFS", context.Background(), mock.Anything).Return(gopowerstore.FileSystem{NasServerID: validNasID}, nil) - clientMock.On("GetNAS", context.Background(), mock.Anything).Return(gopowerstore.NAS{CurrentNodeId: validNodeID}, nil) + clientMock.On("GetNAS", context.Background(), mock.Anything).Return(gopowerstore.NAS{CurrentNodeID: validNodeID}, nil) clientMock.On("GetApplianceByName", context.Background(), mock.Anything).Return(gopowerstore.ApplianceInstance{ServiceTag: validServiceTag}, nil) + clientMock.On("GetInProgressJobsByFsName", context.Background(), mock.Anything).Return([]gopowerstore.Job{}, nil) + clientMock.On("GetFSByName", mock.Anything, mock.Anything).Return(gopowerstore.FileSystem{}, errors.New("not found")) req := getTypicalCreateVolumeNFSRequest("my-vol", validVolSize) - req.Parameters[common.KeyArrayID] = secondValidID + req.Parameters[identifiers.KeyArrayID] = secondValidID - req.Parameters[controller.KeyCSIPVCName] = req.Name - req.Parameters[controller.KeyCSIPVCNamespace] = validNamespaceName + req.Parameters[KeyCSIPVCName] = req.Name + req.Parameters[KeyCSIPVCNamespace] = validNamespaceName - iscsiTopology := &csi.Topology{Segments: map[string]string{common.Name + "/" + ctrlSvc.Arrays()[secondValidID].GetIP() + "-iscis": "true"}} + iscsiTopology := &csi.Topology{Segments: map[string]string{identifiers.Name + "/" + ctrlSvc.Arrays()[secondValidID].GetIP() + "-iscis": "true"}} req.AccessibilityRequirements.Preferred = append(req.AccessibilityRequirements.Preferred, iscsiTopology) res, err := ctrlSvc.CreateVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.CreateVolumeResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.CreateVolumeResponse{ Volume: &csi.Volume{ CapacityBytes: validVolSize, VolumeId: filepath.Join(validBaseVolID, secondValidID, "nfs"), VolumeContext: map[string]string{ - common.KeyArrayVolumeName: "my-vol", - common.KeyProtocol: "nfs", - common.KeyArrayID: secondValidID, - common.KeyNfsACL: "A::OWNER@:RWX", - common.KeyNasName: validNasName, - common.KeyVolumeDescription: req.Name + "-" + validNamespaceName, - common.KeyServiceTag: validServiceTag, - controller.KeyCSIPVCName: req.Name, - controller.KeyCSIPVCNamespace: validNamespaceName, - }, - AccessibleTopology: []*csi.Topology{{Segments: map[string]string{common.Name + "/" + ctrlSvc.Arrays()[secondValidID].GetIP() + "-nfs": "true"}}}, + identifiers.KeyArrayVolumeName: "my-vol", + identifiers.KeyProtocol: "nfs", + identifiers.KeyArrayID: secondValidID, + identifiers.KeyNfsACL: "A::OWNER@:RWX", + identifiers.KeyNasName: validNasName, + identifiers.KeyVolumeDescription: req.Name + "-" + validNamespaceName, + identifiers.KeyServiceTag: validServiceTag, + KeyCSIPVCName: req.Name, + KeyCSIPVCNamespace: validNamespaceName, + }, + AccessibleTopology: []*csi.Topology{{Segments: map[string]string{identifiers.Name + "/" + ctrlSvc.Arrays()[secondValidID].GetIP() + "-nfs": "true"}}}, }, })) }) }) - When("volume name already in use", func() { - It("should return existing volume [Block]", func() { + ginkgo.When("volume name already in use", func() { + ginkgo.It("should return existing volume [Block]", func() { volName := "my-vol" - clientMock.On("GetCustomHTTPHeaders").Return(make(http.Header)) + clientMock.On("GetCustomHTTPHeaders").Return(api.NewSafeHeader().GetHeader()) clientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(3.0), nil) clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) clientMock.On("CreateVolume", mock.Anything, mock.Anything). @@ -809,30 +1454,30 @@ var _ = Describe("CSIControllerService", func() { clientMock.On("GetAppliance", context.Background(), mock.Anything).Return(gopowerstore.ApplianceInstance{ServiceTag: validServiceTag}, nil) req := getTypicalCreateVolumeRequest(volName, validVolSize) - req.Parameters[common.KeyArrayID] = firstValidID - req.Parameters[controller.KeyCSIPVCName] = req.Name - req.Parameters[controller.KeyCSIPVCNamespace] = validNamespaceName + req.Parameters[identifiers.KeyArrayID] = firstValidID + req.Parameters[KeyCSIPVCName] = req.Name + req.Parameters[KeyCSIPVCNamespace] = validNamespaceName res, err := ctrlSvc.CreateVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.CreateVolumeResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.CreateVolumeResponse{ Volume: &csi.Volume{ CapacityBytes: validVolSize, VolumeId: filepath.Join(validBaseVolID, firstValidID, "scsi"), VolumeContext: map[string]string{ - common.KeyArrayVolumeName: "my-vol", - common.KeyProtocol: "scsi", - common.KeyArrayID: firstValidID, - common.KeyVolumeDescription: req.Name + "-" + validNamespaceName, - common.KeyServiceTag: validServiceTag, - controller.KeyCSIPVCName: req.Name, - controller.KeyCSIPVCNamespace: validNamespaceName, + identifiers.KeyArrayVolumeName: "my-vol", + identifiers.KeyProtocol: "scsi", + identifiers.KeyArrayID: firstValidID, + identifiers.KeyVolumeDescription: req.Name + "-" + validNamespaceName, + identifiers.KeyServiceTag: validServiceTag, + KeyCSIPVCName: req.Name, + KeyCSIPVCNamespace: validNamespaceName, }, }, })) }) - It("should return existing volume [NFS]", func() { + ginkgo.It("should return existing volume [NFS]", func() { volName := "my-vol" clientMock.On("GetNASByName", mock.Anything, validNasName).Return(gopowerstore.NAS{ID: validNasID}, nil) @@ -848,40 +1493,50 @@ var _ = Describe("CSIControllerService", func() { SizeTotal: validVolSize, }, nil) clientMock.On("GetFS", context.Background(), mock.Anything).Return(gopowerstore.FileSystem{NasServerID: validNasID}, nil) - clientMock.On("GetNAS", context.Background(), mock.Anything).Return(gopowerstore.NAS{CurrentNodeId: validNodeID}, nil) + clientMock.On("GetNAS", mock.Anything, mock.Anything). + Return(gopowerstore.NAS{ + Name: validNasName, + CurrentNodeID: validNodeID, + }, nil) clientMock.On("GetApplianceByName", context.Background(), mock.Anything).Return(gopowerstore.ApplianceInstance{ServiceTag: validServiceTag}, nil) + clientMock.On("GetInProgressJobsByFsName", context.Background(), mock.Anything).Return([]gopowerstore.Job{}, nil) + clientMock.On("GetFSByName", mock.Anything, volName).Return(gopowerstore.FileSystem{ + ID: validBaseVolID, + Name: volName, + SizeTotal: validVolSize, + }, nil) req := getTypicalCreateVolumeNFSRequest(volName, validVolSize) - req.Parameters[common.KeyArrayID] = secondValidID - req.Parameters[controller.KeyCSIPVCName] = req.Name - req.Parameters[controller.KeyCSIPVCNamespace] = validNamespaceName + req.Parameters[identifiers.KeyArrayID] = secondValidID + req.Parameters[KeyCSIPVCName] = req.Name + req.Parameters[KeyCSIPVCNamespace] = validNamespaceName res, err := ctrlSvc.CreateVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.CreateVolumeResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.CreateVolumeResponse{ Volume: &csi.Volume{ CapacityBytes: validVolSize, VolumeId: filepath.Join(validBaseVolID, secondValidID, "nfs"), VolumeContext: map[string]string{ - common.KeyArrayVolumeName: "my-vol", - common.KeyProtocol: "nfs", - common.KeyArrayID: secondValidID, - common.KeyNfsACL: "A::OWNER@:RWX", - common.KeyNasName: validNasName, - common.KeyVolumeDescription: req.Name + "-" + validNamespaceName, - common.KeyServiceTag: validServiceTag, - controller.KeyCSIPVCName: req.Name, - controller.KeyCSIPVCNamespace: validNamespaceName, - }, - AccessibleTopology: []*csi.Topology{{Segments: map[string]string{common.Name + "/" + ctrlSvc.Arrays()[secondValidID].GetIP() + "-nfs": "true"}}}, + identifiers.KeyArrayVolumeName: "my-vol", + identifiers.KeyProtocol: "nfs", + identifiers.KeyArrayID: secondValidID, + identifiers.KeyNfsACL: "A::OWNER@:RWX", + identifiers.KeyNasName: validNasName, + identifiers.KeyVolumeDescription: req.Name + "-" + validNamespaceName, + identifiers.KeyServiceTag: validServiceTag, + KeyCSIPVCName: req.Name, + KeyCSIPVCNamespace: validNamespaceName, + }, + AccessibleTopology: []*csi.Topology{{Segments: map[string]string{identifiers.Name + "/" + ctrlSvc.Arrays()[secondValidID].GetIP() + "-nfs": "true"}}}, }, })) }) - When("existing volume size is smaller", func() { - It("should fail [Block]", func() { + ginkgo.When("existing volume size is smaller", func() { + ginkgo.It("should fail [Block]", func() { volName := "my-vol" - clientMock.On("GetCustomHTTPHeaders").Return(make(http.Header)) + clientMock.On("GetCustomHTTPHeaders").Return(api.NewSafeHeader().GetHeader()) clientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(3.0), nil) clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) clientMock.On("CreateVolume", mock.Anything, mock.Anything). @@ -898,20 +1553,20 @@ var _ = Describe("CSIControllerService", func() { }, nil) req := getTypicalCreateVolumeRequest(volName, validVolSize) - req.Parameters[common.KeyArrayID] = firstValidID + req.Parameters[identifiers.KeyArrayID] = firstValidID res, err := ctrlSvc.CreateVolume(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To( - ContainSubstring("volume '" + volName + "' already exists but is incompatible volume size"), + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To( + gomega.ContainSubstring("volume '" + volName + "' already exists but is incompatible volume size"), ) }) - It("should fail [NFS]", func() { + ginkgo.It("should fail [NFS]", func() { volName := "my-vol" clientMock.On("GetNASByName", mock.Anything, validNasName).Return(gopowerstore.NAS{ID: validNasID}, nil) - + clientMock.On("GetInProgressJobsByFsName", context.Background(), mock.Anything).Return([]gopowerstore.Job{}, nil) clientMock.On("CreateFS", mock.Anything, mock.Anything).Return(gopowerstore.CreateResponse{}, gopowerstore.APIError{ ErrorMsg: &api.ErrorMsg{ StatusCode: http.StatusUnprocessableEntity, @@ -925,20 +1580,20 @@ var _ = Describe("CSIControllerService", func() { }, nil) req := getTypicalCreateVolumeNFSRequest(volName, validVolSize) - req.Parameters[common.KeyArrayID] = secondValidID + req.Parameters[identifiers.KeyArrayID] = secondValidID res, err := ctrlSvc.CreateVolume(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To( - ContainSubstring("filesystem '" + volName + "' already exists but is incompatible volume size"), + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To( + gomega.ContainSubstring("filesystem '" + volName + "' already exists but is incompatible volume size"), ) }) }) }) - When("creating volume from snapshot", func() { - It("should create volume using snapshot as a source [Block]", func() { + ginkgo.When("creating volume from snapshot", func() { + ginkgo.It("should create volume using snapshot as a source [Block]", func() { snapID := validBlockVolumeID contentSource := &csi.VolumeContentSource{Type: &csi.VolumeContentSource_Snapshot{ @@ -957,24 +1612,44 @@ var _ = Describe("CSIControllerService", func() { req := getTypicalCreateVolumeRequest("my-vol", validVolSize) req.VolumeContentSource = contentSource - req.Parameters[common.KeyArrayID] = firstValidID + req.Parameters[identifiers.KeyArrayID] = firstValidID res, err := ctrlSvc.CreateVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.CreateVolumeResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.CreateVolumeResponse{ Volume: &csi.Volume{ CapacityBytes: validVolSize, VolumeId: filepath.Join(validBaseVolID, firstValidID, "scsi"), VolumeContext: map[string]string{ - common.KeyArrayID: firstValidID, + identifiers.KeyArrayID: firstValidID, }, ContentSource: contentSource, }, })) }) - It("should create volume using snapshot as a source [NFS]", func() { + ginkgo.It("should fail to create volume using Metro snapshot as a source with Metro storage class [Block]", func() { + contentSource := &csi.VolumeContentSource{Type: &csi.VolumeContentSource_Snapshot{ + Snapshot: &csi.VolumeContentSource_SnapshotSource{ + SnapshotId: validBlockVolumeID, + }, + }} + + req := getTypicalCreateVolumeRequest("my-vol", validVolSize) + req.VolumeContentSource = contentSource + req.Parameters[identifiers.KeyArrayID] = firstValidID + req.Parameters[ctrlSvc.WithRP(KeyReplicationEnabled)] = "true" + req.Parameters[ctrlSvc.WithRP(KeyReplicationMode)] = "METRO" + + res, err := ctrlSvc.CreateVolume(context.Background(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("Configuring Metro is not supported on clones or volumes created from Metro snapshot")) + }) + + ginkgo.It("should create volume using snapshot as a source [NFS]", func() { snapID := validNfsVolumeID volName := "my-vol" @@ -986,7 +1661,7 @@ var _ = Describe("CSIControllerService", func() { clientMock.On("GetFS", mock.Anything, validBaseVolID).Return(gopowerstore.FileSystem{ ID: validBaseVolID, - SizeTotal: validVolSize + controller.ReservedSize, + SizeTotal: validVolSize + ReservedSize, }, nil) fsClone := &gopowerstore.FsClone{ @@ -998,27 +1673,27 @@ var _ = Describe("CSIControllerService", func() { req := getTypicalCreateVolumeNFSRequest(volName, validVolSize) req.VolumeContentSource = contentSource - req.Parameters[common.KeyArrayID] = secondValidID + req.Parameters[identifiers.KeyArrayID] = secondValidID res, err := ctrlSvc.CreateVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.CreateVolumeResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.CreateVolumeResponse{ Volume: &csi.Volume{ CapacityBytes: validVolSize, VolumeId: filepath.Join(validBaseVolID, secondValidID, "nfs"), VolumeContext: map[string]string{ - common.KeyArrayID: secondValidID, + identifiers.KeyArrayID: secondValidID, }, ContentSource: contentSource, - AccessibleTopology: []*csi.Topology{{Segments: map[string]string{common.Name + "/" + ctrlSvc.Arrays()[secondValidID].GetIP() + "-nfs": "true"}}}, + AccessibleTopology: []*csi.Topology{{Segments: map[string]string{identifiers.Name + "/" + ctrlSvc.Arrays()[secondValidID].GetIP() + "-nfs": "true"}}}, }, })) }) }) - When("cloning volume", func() { - It("should create volume using volume as a source [Block]", func() { + ginkgo.When("cloning volume", func() { + ginkgo.It("should create volume using volume as a source [Block]", func() { srcID := validBlockVolumeID volName := "my-vol" @@ -1042,24 +1717,47 @@ var _ = Describe("CSIControllerService", func() { req := getTypicalCreateVolumeRequest(volName, validVolSize) req.VolumeContentSource = contentSource - req.Parameters[common.KeyArrayID] = firstValidID + req.Parameters[identifiers.KeyArrayID] = firstValidID res, err := ctrlSvc.CreateVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.CreateVolumeResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.CreateVolumeResponse{ Volume: &csi.Volume{ CapacityBytes: validVolSize, VolumeId: filepath.Join(validBaseVolID, firstValidID, "scsi"), VolumeContext: map[string]string{ - common.KeyArrayID: firstValidID, + identifiers.KeyArrayID: firstValidID, }, ContentSource: contentSource, }, })) }) - It("should create volume using volume as a source [NFS]", func() { + ginkgo.It("should fail to create volume using Metro volume as a source with Metro storage class [Block]", func() { + srcID := validBlockVolumeID + volName := "my-vol" + + contentSource := &csi.VolumeContentSource{Type: &csi.VolumeContentSource_Volume{ + Volume: &csi.VolumeContentSource_VolumeSource{ + VolumeId: srcID, + }, + }} + + req := getTypicalCreateVolumeRequest(volName, validVolSize) + req.VolumeContentSource = contentSource + req.Parameters[identifiers.KeyArrayID] = firstValidID + req.Parameters[ctrlSvc.WithRP(KeyReplicationEnabled)] = "true" + req.Parameters[ctrlSvc.WithRP(KeyReplicationMode)] = "METRO" + + res, err := ctrlSvc.CreateVolume(context.Background(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("Configuring Metro is not supported on clones or volumes created from Metro snapshot")) + }) + + ginkgo.It("should create volume using volume as a source [NFS]", func() { srcID := validNfsVolumeID volName := "my-vol" @@ -1071,7 +1769,7 @@ var _ = Describe("CSIControllerService", func() { clientMock.On("GetFS", mock.Anything, validBaseVolID).Return(gopowerstore.FileSystem{ ID: validBaseVolID, - SizeTotal: validVolSize + controller.ReservedSize, + SizeTotal: validVolSize + ReservedSize, }, nil) fsClone := &gopowerstore.FsClone{ @@ -1083,28 +1781,31 @@ var _ = Describe("CSIControllerService", func() { req := getTypicalCreateVolumeNFSRequest(volName, validVolSize) req.VolumeContentSource = contentSource - req.Parameters[common.KeyArrayID] = secondValidID + req.Parameters[identifiers.KeyArrayID] = secondValidID res, err := ctrlSvc.CreateVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.CreateVolumeResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.CreateVolumeResponse{ Volume: &csi.Volume{ CapacityBytes: validVolSize, VolumeId: filepath.Join(validBaseVolID, secondValidID, "nfs"), VolumeContext: map[string]string{ - common.KeyArrayID: secondValidID, + identifiers.KeyArrayID: secondValidID, }, ContentSource: contentSource, - AccessibleTopology: []*csi.Topology{{Segments: map[string]string{common.Name + "/" + ctrlSvc.Arrays()[secondValidID].GetIP() + "-nfs": "true"}}}, + AccessibleTopology: []*csi.Topology{{Segments: map[string]string{identifiers.Name + "/" + ctrlSvc.Arrays()[secondValidID].GetIP() + "-nfs": "true"}}}, }, })) }) }) - When("there is no array IP in storage class", func() { - It("should use default array", func() { - clientMock.On("GetCustomHTTPHeaders").Return(make(http.Header)) + ginkgo.When("there is no array IP in storage class", func() { + ginkgo.It("should use default array", func() { + clientMock.On("GetCustomHTTPHeaders").Return(api.NewSafeHeader().GetHeader()) + clientMock.On("GetVolumeByName", mock.Anything, mock.Anything).Return( + gopowerstore.Volume{}, errors.New("no vol found")) + clientMock.On("GetInProgressJobsByFsName", context.Background(), mock.Anything).Return([]gopowerstore.Job{}, nil) clientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(3.0), nil) clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) clientMock.On("CreateVolume", mock.Anything, mock.Anything).Return(gopowerstore.CreateResponse{ID: validBaseVolID}, nil) @@ -1114,97 +1815,418 @@ var _ = Describe("CSIControllerService", func() { req := getTypicalCreateVolumeRequest("my-vol", validVolSize) res, err := ctrlSvc.CreateVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.CreateVolumeResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.CreateVolumeResponse{ Volume: &csi.Volume{ CapacityBytes: validVolSize, VolumeId: filepath.Join(validBaseVolID, firstValidID, "scsi"), VolumeContext: map[string]string{ - common.KeyArrayVolumeName: "my-vol", - common.KeyProtocol: "scsi", - common.KeyArrayID: firstValidID, - common.KeyVolumeDescription: "-", - common.KeyServiceTag: validServiceTag, + identifiers.KeyArrayVolumeName: "my-vol", + identifiers.KeyProtocol: "scsi", + identifiers.KeyArrayID: firstValidID, + identifiers.KeyVolumeDescription: "-", + identifiers.KeyServiceTag: validServiceTag, }, }, })) }) }) - When("there array IP passed to storage class is not config", func() { - It("should fail", func() { + ginkgo.When("there array IP passed to storage class is not config", func() { + ginkgo.It("should fail", func() { req := getTypicalCreateVolumeRequest("my-vol", validVolSize) - req.Parameters[common.KeyArrayID] = "127.0.0.1" + req.Parameters[identifiers.KeyArrayID] = "127.0.0.1" res, err := ctrlSvc.CreateVolume(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring("can't find array with provided id")) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("can't find array with provided id")) }) }) - When("requesting block access from nfs volume", func() { - It("should fail [new key]", func() { + ginkgo.When("requesting block access from nfs volume", func() { + ginkgo.It("should fail [new key]", func() { req := getTypicalCreateVolumeNFSRequest("my-vol", validVolSize) req.VolumeCapabilities[0].AccessType = &csi.VolumeCapability_Block{ Block: &csi.VolumeCapability_BlockVolume{}, } - req.Parameters[common.KeyArrayID] = secondValidID - req.Parameters[controller.KeyFsType] = "nfs" + req.Parameters[identifiers.KeyArrayID] = secondValidID + req.Parameters[KeyFsType] = "nfs" res, err := ctrlSvc.CreateVolume(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring("raw block requested from NFS Volume")) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("raw block requested from NFS Volume")) }) - It("should fail [old key]", func() { + ginkgo.It("should fail [old key]", func() { req := getTypicalCreateVolumeNFSRequest("my-vol", validVolSize) req.VolumeCapabilities[0].AccessType = &csi.VolumeCapability_Block{ Block: &csi.VolumeCapability_BlockVolume{}, } - req.Parameters[common.KeyArrayID] = secondValidID - req.Parameters[controller.KeyFsTypeOld] = "nfs" + req.Parameters[identifiers.KeyArrayID] = secondValidID + req.Parameters[KeyFsTypeOld] = "nfs" res, err := ctrlSvc.CreateVolume(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring("raw block requested from NFS Volume")) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("raw block requested from NFS Volume")) }) }) - When("volume name is empty", func() { - It("should fail", func() { + ginkgo.When("volume name is empty", func() { + ginkgo.It("should fail", func() { req := getTypicalCreateVolumeRequest("", validVolSize) res, err := ctrlSvc.CreateVolume(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring("name cannot be empty")) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("name cannot be empty")) }) }) - When("volume size is incorrect", func() { - It("should fail", func() { + ginkgo.When("volume size is incorrect", func() { + ginkgo.It("should fail", func() { req := getTypicalCreateVolumeRequest("my-vol", validVolSize) req.CapacityRange.LimitBytes = -1000 req.CapacityRange.RequiredBytes = -1000 res, err := ctrlSvc.CreateVolume(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring( + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring( fmt.Sprintf("bad capacity: volume size bytes %d and limit size bytes: %d must not be negative", req.CapacityRange.RequiredBytes, req.CapacityRange.RequiredBytes), )) }) }) }) - Describe("calling DeleteVolume()", func() { - When("deleting block volume", func() { - It("should successfully delete block volume", func() { + ginkgo.When("multi-nas is enabled for NFS", func() { + validNAS1 := gopowerstore.NAS{ + Name: "nasA", + OperationalStatus: gopowerstore.Started, + HealthDetails: gopowerstore.HealthDetails{State: gopowerstore.None}, + FileSystems: make([]gopowerstore.FileSystem, 1), // 1 FS (should be chosen) + } + + validNAS2 := gopowerstore.NAS{ + Name: "nasB", + OperationalStatus: gopowerstore.Started, + HealthDetails: gopowerstore.HealthDetails{State: gopowerstore.Info}, + FileSystems: make([]gopowerstore.FileSystem, 2), // 2 FS, but lexicographically larger + } + + validNAS3 := gopowerstore.NAS{ + Name: "nasC", + OperationalStatus: gopowerstore.Started, + HealthDetails: gopowerstore.HealthDetails{State: gopowerstore.Info}, + FileSystems: make([]gopowerstore.FileSystem, 3), + } + + invalidNAS4 := gopowerstore.NAS{ + Name: "nasX", + OperationalStatus: gopowerstore.Stopped, // Inactive NAS + HealthDetails: gopowerstore.HealthDetails{State: gopowerstore.Info}, + FileSystems: make([]gopowerstore.FileSystem, 1), + } + + ginkgo.It("should successfully create nfs volume with least used NAS if multiple NAS exist in storage class", func() { + clientMock.On("GetNASServers", mock.Anything).Return([]gopowerstore.NAS{validNAS1, validNAS2, validNAS3, invalidNAS4}, nil) + clientMock.On("GetNASByName", mock.Anything, "nasA").Return(gopowerstore.NAS{ID: validNasID}, nil) + clientMock.On("CreateFS", mock.Anything, mock.Anything).Return(gopowerstore.CreateResponse{ID: validBaseVolID}, nil) + clientMock.On("GetFS", context.Background(), mock.Anything).Return(gopowerstore.FileSystem{NasServerID: validNasID}, nil) + clientMock.On("GetNAS", context.Background(), mock.Anything).Return(gopowerstore.NAS{CurrentNodeID: validNodeID}, nil) + clientMock.On("GetApplianceByName", context.Background(), mock.Anything).Return(gopowerstore.ApplianceInstance{ServiceTag: validServiceTag}, nil) + clientMock.On("GetInProgressJobsByFsName", context.Background(), mock.Anything).Return([]gopowerstore.Job{}, nil) + clientMock.On("GetFSByName", mock.Anything, mock.Anything).Return(gopowerstore.FileSystem{}, errors.New("not found")) + + req := getTypicalCreateVolumeNFSRequest("my-vol", validVolSize) + req.Parameters[identifiers.KeyArrayID] = secondValidID + + req.Parameters[KeyCSIPVCName] = req.Name + req.Parameters[KeyCSIPVCNamespace] = validNamespaceName + req.Parameters[identifiers.KeyNasName] = "nasA, nasB, nasC, nasX" + + res, err := ctrlSvc.CreateVolume(context.Background(), req) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.CreateVolumeResponse{ + Volume: &csi.Volume{ + CapacityBytes: validVolSize, + VolumeId: filepath.Join(validBaseVolID, secondValidID, "nfs"), + VolumeContext: map[string]string{ + identifiers.KeyArrayVolumeName: "my-vol", + identifiers.KeyProtocol: "nfs", + identifiers.KeyArrayID: secondValidID, + identifiers.KeyNfsACL: "A::OWNER@:RWX", + identifiers.KeyNasName: "nasA", + identifiers.KeyVolumeDescription: req.Name + "-" + validNamespaceName, + identifiers.KeyServiceTag: validServiceTag, + KeyCSIPVCName: req.Name, + KeyCSIPVCNamespace: validNamespaceName, + }, + AccessibleTopology: []*csi.Topology{{Segments: map[string]string{identifiers.Name + "/" + ctrlSvc.Arrays()[secondValidID].GetIP() + "-nfs": "true"}}}, + }, + })) + }) + + ginkgo.It("should successfully create nfs volume if only one NAS exist in storage class", func() { + clientMock.On("GetNASByName", mock.Anything, "nasA").Return(gopowerstore.NAS{ID: validNasID}, nil) + clientMock.On("CreateFS", mock.Anything, mock.Anything).Return(gopowerstore.CreateResponse{ID: validBaseVolID}, nil) + clientMock.On("GetFS", context.Background(), mock.Anything).Return(gopowerstore.FileSystem{NasServerID: validNasID}, nil) + clientMock.On("GetNAS", context.Background(), mock.Anything).Return(gopowerstore.NAS{CurrentNodeID: validNodeID}, nil) + clientMock.On("GetApplianceByName", context.Background(), mock.Anything).Return(gopowerstore.ApplianceInstance{ServiceTag: validServiceTag}, nil) + clientMock.On("GetInProgressJobsByFsName", context.Background(), mock.Anything).Return([]gopowerstore.Job{}, nil) + clientMock.On("GetFSByName", mock.Anything, mock.Anything).Return(gopowerstore.FileSystem{}, errors.New("not found")) + + req := getTypicalCreateVolumeNFSRequest("my-vol", validVolSize) + req.Parameters[identifiers.KeyArrayID] = secondValidID + + req.Parameters[KeyCSIPVCName] = req.Name + req.Parameters[KeyCSIPVCNamespace] = validNamespaceName + req.Parameters[identifiers.KeyNasName] = "nasA" + + res, err := ctrlSvc.CreateVolume(context.Background(), req) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.CreateVolumeResponse{ + Volume: &csi.Volume{ + CapacityBytes: validVolSize, + VolumeId: filepath.Join(validBaseVolID, secondValidID, "nfs"), + VolumeContext: map[string]string{ + identifiers.KeyArrayVolumeName: "my-vol", + identifiers.KeyProtocol: "nfs", + identifiers.KeyArrayID: secondValidID, + identifiers.KeyNfsACL: "A::OWNER@:RWX", + identifiers.KeyNasName: "nasA", + identifiers.KeyVolumeDescription: req.Name + "-" + validNamespaceName, + identifiers.KeyServiceTag: validServiceTag, + KeyCSIPVCName: req.Name, + KeyCSIPVCNamespace: validNamespaceName, + }, + AccessibleTopology: []*csi.Topology{{Segments: map[string]string{identifiers.Name + "/" + ctrlSvc.Arrays()[secondValidID].GetIP() + "-nfs": "true"}}}, + }, + })) + }) + + ginkgo.It("should fail when there is no least used NAS [Invalid NAS]", func() { + clientMock.On("GetNASServers", mock.Anything).Return([]gopowerstore.NAS{invalidNAS4}, nil) + + req := getTypicalCreateVolumeNFSRequest("my-vol", validVolSize) + req.Parameters[identifiers.KeyArrayID] = secondValidID + + req.Parameters[KeyCSIPVCName] = req.Name + req.Parameters[KeyCSIPVCNamespace] = validNamespaceName + req.Parameters[identifiers.KeyNasName] = "nasA, nasB, nasC, nasX" + + res, err := ctrlSvc.CreateVolume(context.Background(), req) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("no suitable NAS server found, please ensure the NAS is running and healthy")) + gomega.Expect(res).To(gomega.BeNil()) + }) + + ginkgo.It("should fail if CreateFS fails with NAS limit error & failure count should be incremented", func() { + clientMock.On("GetInProgressJobsByFsName", context.Background(), mock.Anything).Return([]gopowerstore.Job{}, nil) + clientMock.On("GetNASServers", mock.Anything).Return([]gopowerstore.NAS{validNAS1, validNAS2, validNAS3, invalidNAS4}, nil) + clientMock.On("GetNASByName", mock.Anything, "nasA").Return(gopowerstore.NAS{ID: validNasID}, nil) + clientMock.On("CreateFS", mock.Anything, mock.Anything). + Return(gopowerstore.CreateResponse{}, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{ + StatusCode: http.StatusUnprocessableEntity, + Message: "New file system can not be created. The limit of 125 file systems for the NAS server 24aefac2-a796-47dc-886a-c73ff8c1a671 has been reached.", + }, + }) + clientMock.On("GetFSByName", mock.Anything, "my-vol").Return(gopowerstore.FileSystem{}, errors.New("not nil")) + clientMock.On("GetInProgressJobsByFsName", context.Background(), mock.Anything).Return([]gopowerstore.Job{}, nil) + + req := getTypicalCreateVolumeNFSRequest("my-vol", validVolSize) + req.Parameters[identifiers.KeyArrayID] = secondValidID + + req.Parameters[KeyCSIPVCName] = req.Name + req.Parameters[KeyCSIPVCNamespace] = validNamespaceName + req.Parameters[identifiers.KeyNasName] = "nasA, nasB, nasC, nasX" + + res, err := ctrlSvc.CreateVolume(context.Background(), req) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("The limit of 125 file systems for the NAS server")) + gomega.Expect(res).To(gomega.BeNil()) + tracker := ctrlSvc.Arrays()[secondValidID].NASCooldownTracker.(*array.NASCooldown) + gomega.Expect(tracker.GetStatusMap()["nasA"].Failures).To(gomega.Equal(1)) + }) + + ginkgo.It("should fail if CreateFS fails with some other API error & failure count should be incremented", func() { + clientMock.On("GetNASServers", mock.Anything).Return([]gopowerstore.NAS{validNAS1, validNAS2, validNAS3, invalidNAS4}, nil) + clientMock.On("GetNASByName", mock.Anything, "nasA").Return(gopowerstore.NAS{ID: validNasID}, nil) + clientMock.On("CreateFS", mock.Anything, mock.Anything). + Return(gopowerstore.CreateResponse{}, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{ + StatusCode: http.StatusForbidden, + Message: "some error message", + }, + }) + clientMock.On("GetInProgressJobsByFsName", context.Background(), mock.Anything).Return([]gopowerstore.Job{}, nil) + clientMock.On("GetFSByName", mock.Anything, mock.Anything).Return(gopowerstore.FileSystem{}, errors.New("not found")) + + req := getTypicalCreateVolumeNFSRequest("my-vol", validVolSize) + req.Parameters[identifiers.KeyArrayID] = secondValidID + + req.Parameters[KeyCSIPVCName] = req.Name + req.Parameters[KeyCSIPVCNamespace] = validNamespaceName + req.Parameters[identifiers.KeyNasName] = "nasA, nasB, nasC, nasX" + + res, err := ctrlSvc.CreateVolume(context.Background(), req) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("some error message")) + gomega.Expect(res).To(gomega.BeNil()) + tracker := ctrlSvc.Arrays()[secondValidID].NASCooldownTracker.(*array.NASCooldown) + gomega.Expect(tracker.GetStatusMap()["nasA"].Failures).To(gomega.Equal(1)) + }) + + ginkgo.It("should fail if CreateFS fails with Non-API error & failure count should be incremented", func() { + clientMock.On("GetInProgressJobsByFsName", context.Background(), mock.Anything).Return([]gopowerstore.Job{}, nil) + clientMock.On("GetFSByName", mock.Anything, mock.Anything).Return( + gopowerstore.FileSystem{}, errors.New("no vol found")) + clientMock.On("GetNASServers", mock.Anything).Return([]gopowerstore.NAS{validNAS1, validNAS2, validNAS3, invalidNAS4}, nil) + clientMock.On("GetNASByName", mock.Anything, "nasA").Return(gopowerstore.NAS{ID: validNasID}, nil) + clientMock.On("CreateFS", mock.Anything, mock.Anything).Return(gopowerstore.CreateResponse{}, errors.New("some error message")) + + req := getTypicalCreateVolumeNFSRequest("my-vol", validVolSize) + req.Parameters[identifiers.KeyArrayID] = secondValidID + + req.Parameters[KeyCSIPVCName] = req.Name + req.Parameters[KeyCSIPVCNamespace] = validNamespaceName + req.Parameters[identifiers.KeyNasName] = "nasA, nasB, nasC, nasX" + + res, err := ctrlSvc.CreateVolume(context.Background(), req) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("some error message")) + gomega.Expect(res).To(gomega.BeNil()) + tracker := ctrlSvc.Arrays()[secondValidID].NASCooldownTracker.(*array.NASCooldown) + gomega.Expect(tracker.GetStatusMap()["nasA"].Failures).To(gomega.Equal(1)) + }) + + ginkgo.It("should fail when listing jobs returns error [NFS]", func() { + volName := "my-vol" + clientMock.On("GetNASByName", mock.Anything, validNasName).Return(gopowerstore.NAS{ID: validNasID}, nil) + clientMock.On("GetInProgressJobsByFsName", context.Background(), mock.Anything).Return([]gopowerstore.Job{}, errors.New("cannot list jobs")) + req := getTypicalCreateVolumeNFSRequest(volName, validVolSize) + req.Parameters[identifiers.KeyArrayID] = secondValidID + res, err := ctrlSvc.CreateVolume(context.Background(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To( + gomega.ContainSubstring("Error getting jobs that are in progress"), + ) + }) + + ginkgo.It("should fail when an in progress job is found- to prevent duplicate volumes", func() { + volName := "my-vol" + clientMock.On("GetNASByName", mock.Anything, validNasName).Return(gopowerstore.NAS{ID: validNasID}, nil) + clientMock.On("GetInProgressJobsByFsName", context.Background(), mock.Anything).Return([]gopowerstore.Job{ + { + ID: "1", + Action: "create", + Type: "file_system", + ResourceName: volName, + State: "InProgress", + }, + }, nil) + req := getTypicalCreateVolumeNFSRequest(volName, validVolSize) + req.Parameters[identifiers.KeyArrayID] = secondValidID + res, err := ctrlSvc.CreateVolume(context.Background(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To( + gomega.ContainSubstring("Job already in progress"), + ) + }) + + ginkgo.It("should enter a cooldown if the failure threshold (5) is reached & fallback to next best nas server", func() { + clientMock.On("GetInProgressJobsByFsName", context.Background(), mock.Anything).Return([]gopowerstore.Job{}, nil) + clientMock.On("GetNASServers", mock.Anything).Return([]gopowerstore.NAS{validNAS1, validNAS2, validNAS3, invalidNAS4}, nil) + // 1st to 5th call: return nasA + clientMock.On("GetNASByName", mock.Anything, "nasA").Return(gopowerstore.NAS{ID: validNasID}, nil).Times(5) + // 6th call: return nasB + clientMock.On("GetNASByName", mock.Anything, "nasB").Return(gopowerstore.NAS{ID: "nasB-id"}, nil).Once() + // Return the same error 6 times + clientMock.On("CreateFS", mock.Anything, mock.Anything). + Return(gopowerstore.CreateResponse{}, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{ + StatusCode: http.StatusUnprocessableEntity, + Message: "New file system can not be created. The limit of 125 file systems for the NAS server has been reached.", + }, + }).Times(5) + + clientMock.On("GetFSByName", mock.Anything, "my-vol").Return(gopowerstore.FileSystem{}, errors.New("not nil")) + clientMock.On("CreateFS", mock.Anything, mock.Anything).Return(gopowerstore.CreateResponse{ID: validBaseVolID}, nil).Once() + clientMock.On("GetFS", context.Background(), mock.Anything).Return(gopowerstore.FileSystem{NasServerID: validNasID}, nil) + clientMock.On("GetNAS", context.Background(), mock.Anything).Return(gopowerstore.NAS{CurrentNodeID: validNodeID}, nil) + clientMock.On("GetApplianceByName", context.Background(), mock.Anything).Return(gopowerstore.ApplianceInstance{ServiceTag: validServiceTag}, nil) + + req := getTypicalCreateVolumeNFSRequest("my-vol", validVolSize) + req.Parameters[identifiers.KeyArrayID] = secondValidID + req.Parameters[KeyCSIPVCName] = req.Name + req.Parameters[KeyCSIPVCNamespace] = validNamespaceName + req.Parameters[identifiers.KeyNasName] = "nasA, nasB, nasC, nasX" + + var err error + var res *csi.CreateVolumeResponse + + // Trigger 6 failures + for i := 0; i < 6; i++ { + res, err = ctrlSvc.CreateVolume(context.Background(), req) + } + + gomega.Expect(err).To(gomega.BeNil()) + tracker := ctrlSvc.Arrays()[secondValidID].NASCooldownTracker.(*array.NASCooldown) + gomega.Expect(tracker.IsInCooldown("nasA")).To(gomega.BeTrue()) + gomega.Expect(res.GetVolume().GetVolumeContext()[identifiers.KeyNasName]).To(gomega.Equal("nasB")) + }) + + ginkgo.It("should be eligible for NAS selection in FS creation, after cooldown period has ended", func() { + clientMock.On("GetNASServers", mock.Anything). + Return([]gopowerstore.NAS{validNAS1, validNAS2, validNAS3, invalidNAS4}, nil) + + clientMock.On("GetNASByName", mock.Anything, "nasA"). + Return(gopowerstore.NAS{ID: validNasID}, nil).Once() + + clientMock.On("CreateFS", mock.Anything, mock.Anything). + Return(gopowerstore.CreateResponse{}, errors.New("some error message")).Once() + clientMock.On("GetInProgressJobsByFsName", context.Background(), mock.Anything).Return([]gopowerstore.Job{}, nil) + clientMock.On("GetFSByName", mock.Anything, mock.Anything).Return(gopowerstore.FileSystem{}, errors.New("not found")) + + req := getTypicalCreateVolumeNFSRequest("my-vol", validVolSize) + req.Parameters[identifiers.KeyArrayID] = secondValidID + req.Parameters[KeyCSIPVCName] = req.Name + req.Parameters[KeyCSIPVCNamespace] = validNamespaceName + req.Parameters[identifiers.KeyNasName] = "nasA, nasB, nasC, nasX" + + // First attempt - fails and triggers cooldown + res, err := ctrlSvc.CreateVolume(context.Background(), req) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("some error message")) + gomega.Expect(res).To(gomega.BeNil()) + + tracker := ctrlSvc.Arrays()[secondValidID].NASCooldownTracker.(*array.NASCooldown) + gomega.Expect(tracker.GetStatusMap()["nasA"].Failures).To(gomega.Equal(1)) + + // Simulate cooldown expiry + tracker.GetStatusMap()["nasA"] = &array.NASStatus{} // reset manually or simulate time + + // Second attempt - NAS A should be eligible again + clientMock.On("GetNASByName", mock.Anything, "nasA").Return(gopowerstore.NAS{ID: validNasID}, nil).Once() + clientMock.On("CreateFS", mock.Anything, mock.Anything).Return(gopowerstore.CreateResponse{ID: validBaseVolID}, nil).Once() + clientMock.On("GetFS", context.Background(), mock.Anything).Return(gopowerstore.FileSystem{NasServerID: validNasID}, nil) + clientMock.On("GetNAS", context.Background(), mock.Anything).Return(gopowerstore.NAS{CurrentNodeID: validNodeID}, nil) + clientMock.On("GetApplianceByName", context.Background(), mock.Anything).Return(gopowerstore.ApplianceInstance{ServiceTag: validServiceTag}, nil) + + res, err = ctrlSvc.CreateVolume(context.Background(), req) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res.GetVolume().GetVolumeContext()[identifiers.KeyNasName]).To(gomega.Equal("nasA")) + }) + }) + + ginkgo.Describe("calling DeleteVolume()", func() { + ginkgo.When("deleting block volume", func() { + ginkgo.It("should successfully delete block volume", func() { clientMock.On("GetSnapshotsByVolumeID", mock.Anything, validBaseVolID).Return([]gopowerstore.Volume{}, nil) clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID).Return(gopowerstore.VolumeGroups{}, nil) + clientMock.On("GetVolume", mock.Anything, validBaseVolID).Return(gopowerstore.Volume{ID: validBaseVolID, Size: validVolSize}, nil) clientMock.On("DeleteVolume", mock.Anything, mock.AnythingOfType("*gopowerstore.VolumeDelete"), @@ -1215,14 +2237,15 @@ var _ = Describe("CSIControllerService", func() { res, err := ctrlSvc.DeleteVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.DeleteVolumeResponse{})) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.DeleteVolumeResponse{})) }) }) - When("deleting block volume with old volume handle naming", func() { - It("should successfully delete block volume", func() { + ginkgo.When("deleting block volume with old volume handle naming", func() { + ginkgo.It("should successfully delete block volume", func() { clientMock.On("GetSnapshotsByVolumeID", mock.Anything, validBaseVolID).Return([]gopowerstore.Volume{}, nil) clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID).Return(gopowerstore.VolumeGroups{}, nil) + clientMock.On("GetVolume", mock.Anything, validBaseVolID).Return(gopowerstore.Volume{ID: validBaseVolID, Size: validVolSize}, nil) clientMock.On("DeleteVolume", mock.Anything, mock.AnythingOfType("*gopowerstore.VolumeDelete"), @@ -1234,16 +2257,16 @@ var _ = Describe("CSIControllerService", func() { res, err := ctrlSvc.DeleteVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.DeleteVolumeResponse{})) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.DeleteVolumeResponse{})) }) }) - When("delete block volume with replication props", func() { - It("should successful delete block volume and remove it from group and unassigned policy", func() { + ginkgo.When("delete block volume with replication props", func() { + ginkgo.It("should successful delete block volume and remove it from group and unassigned policy", func() { clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID). Return(gopowerstore.VolumeGroups{VolumeGroup: []gopowerstore.VolumeGroup{{ID: validGroupID, ProtectionPolicyID: validPolicyID}}}, nil) - + clientMock.On("GetSnapshotsByVolumeID", mock.Anything, validBaseVolID).Return([]gopowerstore.Volume{}, nil) clientMock.On("RemoveMembersFromVolumeGroup", mock.Anything, mock.AnythingOfType("*gopowerstore.VolumeGroupMembers"), @@ -1254,6 +2277,7 @@ var _ = Describe("CSIControllerService", func() { &gopowerstore.VolumeModify{ProtectionPolicyID: ""}, validBaseVolID). Return(gopowerstore.EmptyResponse(""), nil) + clientMock.On("GetVolume", mock.Anything, validBaseVolID).Return(gopowerstore.Volume{ID: validBaseVolID, Size: validVolSize}, nil) req := &csi.DeleteVolumeRequest{VolumeId: validBlockVolumeID} @@ -1265,13 +2289,63 @@ var _ = Describe("CSIControllerService", func() { res, err := ctrlSvc.DeleteVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.DeleteVolumeResponse{})) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.DeleteVolumeResponse{})) + }) + + ginkgo.It("should fail to delete block volume", func() { + clientMock.On("GetSnapshotsByVolumeID", mock.Anything, validBaseVolID).Return([]gopowerstore.Volume{}, nil) + clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID). + Return(gopowerstore.VolumeGroups{VolumeGroup: []gopowerstore.VolumeGroup{{ID: validGroupID, ProtectionPolicyID: validPolicyID}}}, nil) + clientMock.On("RemoveMembersFromVolumeGroup", mock.Anything, mock.AnythingOfType("*gopowerstore.VolumeGroupMembers"), validGroupID). + Return(gopowerstore.EmptyResponse(""), gopowerstore.NewNotFoundError()) + + req := &csi.DeleteVolumeRequest{VolumeId: validBlockVolumeID} + res, err := ctrlSvc.DeleteVolume(context.Background(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("failed to remove volume")) + }) + }) + + ginkgo.When("delete block metro volume with replication props", func() { + ginkgo.It("should successfully delete block metro volume", func() { + clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID).Return(gopowerstore.VolumeGroups{}, nil) + clientMock.On("GetSnapshotsByVolumeID", mock.Anything, validBaseVolID).Return([]gopowerstore.Volume{}, nil) + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, MetroReplicationSessionID: validSessionID}, nil) + endMetroRequest := &gopowerstore.EndMetroVolumeOptions{DeleteRemoteVolume: true} + clientMock.On("EndMetroVolume", mock.Anything, validBaseVolID, endMetroRequest).Return(gopowerstore.EmptyResponse(""), nil) + clientMock.On("DeleteVolume", mock.Anything, mock.AnythingOfType("*gopowerstore.VolumeDelete"), validBaseVolID). + Return(gopowerstore.EmptyResponse(""), nil) + + req := &csi.DeleteVolumeRequest{VolumeId: validBlockVolumeID} + res, err := ctrlSvc.DeleteVolume(context.Background(), req) + + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.DeleteVolumeResponse{})) + }) + + ginkgo.It("should fail to delete block metro volume", func() { + clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID).Return(gopowerstore.VolumeGroups{}, nil) + clientMock.On("GetSnapshotsByVolumeID", mock.Anything, validBaseVolID).Return([]gopowerstore.Volume{}, nil) + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, MetroReplicationSessionID: validSessionID}, nil) + clientMock.On("EndMetroVolume", mock.Anything, validBaseVolID, mock.AnythingOfType("*gopowerstore.EndMetroVolumeOptions")). + Return(gopowerstore.EmptyResponse(""), gopowerstore.NewNotFoundError()) + + req := &csi.DeleteVolumeRequest{VolumeId: validBlockVolumeID} + res, err := ctrlSvc.DeleteVolume(context.Background(), req) + + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("failure ending metro session on volume")) }) }) - When("deleting nfs volume", func() { - It("should successfully delete nfs volume", func() { + ginkgo.When("deleting nfs volume", func() { + ginkgo.It("should successfully delete nfs volume", func() { clientMock.On("GetFsSnapshotsByVolumeID", mock.Anything, validBaseVolID).Return([]gopowerstore.FileSystem{}, nil) clientMock.On("DeleteFS", mock.Anything, @@ -1284,24 +2358,35 @@ var _ = Describe("CSIControllerService", func() { res, err := ctrlSvc.DeleteVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.DeleteVolumeResponse{})) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.DeleteVolumeResponse{})) }) }) - When("volume id is not specified", func() { - It("should fail", func() { + ginkgo.When("volume id is not specified", func() { + ginkgo.It("should fail", func() { req := &csi.DeleteVolumeRequest{} res, err := ctrlSvc.DeleteVolume(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring("volume ID is required")) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("volume ID is required")) + }) + }) + + ginkgo.When("array id is not found", func() { + ginkgo.It("should fail", func() { + req := &csi.DeleteVolumeRequest{VolumeId: invalidBlockVolumeID} + res, err := ctrlSvc.DeleteVolume(context.Background(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("can't find array with provided id")) }) }) - When("there is no array ip in volume id", func() { - It("should check storage using default array [no volume found]", func() { + ginkgo.When("there is no array ip in volume id", func() { + ginkgo.It("should check storage using default array [no volume found]", func() { clientMock.On("GetVolume", context.Background(), validBaseVolID). Return(gopowerstore.Volume{}, gopowerstore.APIError{ ErrorMsg: &api.ErrorMsg{ @@ -1318,11 +2403,11 @@ var _ = Describe("CSIControllerService", func() { req := &csi.DeleteVolumeRequest{VolumeId: validBaseVolID} res, err := ctrlSvc.DeleteVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.DeleteVolumeResponse{})) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.DeleteVolumeResponse{})) }) - It("should check storage using default array [unexpected api error]", func() { + ginkgo.It("should check storage using default array [unexpected api error]", func() { e := errors.New("api-error") clientMock.On("GetVolume", context.Background(), validBaseVolID). Return(gopowerstore.Volume{}, e) @@ -1333,17 +2418,51 @@ var _ = Describe("CSIControllerService", func() { res, err := ctrlSvc.DeleteVolume(context.Background(), req) - Expect(err).ToNot(BeNil()) - Expect(res).To(BeNil()) - Expect(err.Error()).To(ContainSubstring("failure checking volume status")) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("failure checking volume status")) }) }) - When("when trying delete volume with existing snapshots", func() { - // TODO: add test after explicitly define deletion behavior - // It("should fail [Block]", func() {}) + ginkgo.When("get block API call fails", func() { + ginkgo.It("should fail [GetSnapshotsByVolumeID]", func() { + clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID).Return(gopowerstore.VolumeGroups{}, nil) + clientMock.On("GetSnapshotsByVolumeID", mock.Anything, validBaseVolID).Return([]gopowerstore.Volume{}, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{StatusCode: http.StatusGatewayTimeout}, + }) + clientMock.On("GetVolume", mock.Anything, validBaseVolID).Return( + gopowerstore.Volume{ + ID: validBaseVolID, + Name: "name", + Size: validVolSize, + }, nil) + + req := &csi.DeleteVolumeRequest{VolumeId: validBlockVolumeID} + res, err := ctrlSvc.DeleteVolume(context.Background(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("failure getting snapshot")) + }) + + ginkgo.It("should fail [GetVolume]", func() { + clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID).Return(gopowerstore.VolumeGroups{}, nil) + clientMock.On("GetSnapshotsByVolumeID", mock.Anything, validBaseVolID).Return([]gopowerstore.Volume{}, nil) + clientMock.On("GetVolume", mock.Anything, validBaseVolID).Return(gopowerstore.Volume{}, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{StatusCode: http.StatusGatewayTimeout}, + }) + + req := &csi.DeleteVolumeRequest{VolumeId: validBlockVolumeID} + res, err := ctrlSvc.DeleteVolume(context.Background(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("failure getting volume")) + }) + }) - It("should fail [NFS]", func() { + ginkgo.When("when trying delete volume with existing snapshots", func() { + ginkgo.It("should fail [NFS]", func() { clientMock.On("GetFsSnapshotsByVolumeID", mock.Anything, validBaseVolID). Return([]gopowerstore.FileSystem{ { @@ -1356,16 +2475,33 @@ var _ = Describe("CSIControllerService", func() { res, err := ctrlSvc.DeleteVolume(context.Background(), req) - Expect(err).ToNot(BeNil()) - Expect(res).To(BeNil()) - Expect(err.Error()).To(ContainSubstring("snapshots based on this volume still exist")) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("snapshots based on this volume still exist")) }) }) - - When("volume does not exist", func() { - It("should succeed [Block]", func() { + ginkgo.When("when trying delete volume with existing snapshots", func() { + ginkgo.It("should fail [scsi]", func() { + clientMock.On("GetVolume", mock.Anything, validBaseVolID).Return( + gopowerstore.Volume{ + ID: validBaseVolID, + Name: "name", + Size: validVolSize, + }, nil) + clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID).Return(gopowerstore.VolumeGroups{}, nil) + clientMock.On("GetSnapshotsByVolumeID", mock.Anything, validBaseVolID).Return([]gopowerstore.Volume{{ID: "snap-id-1"}, {ID: "snap-id-2"}}, nil) + req := &csi.DeleteVolumeRequest{VolumeId: validBaseVolID} + res, err := ctrlSvc.DeleteVolume(context.Background(), req) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("snapshots based on this volume still exist")) + }) + }) + ginkgo.When("volume does not exist", func() { + ginkgo.It("should succeed [Block]", func() { clientMock.On("GetSnapshotsByVolumeID", mock.Anything, validBaseVolID).Return([]gopowerstore.Volume{}, nil) clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID).Return(gopowerstore.VolumeGroups{}, nil) + clientMock.On("GetVolume", mock.Anything, validBaseVolID).Return(gopowerstore.Volume{ID: validBaseVolID}, nil) clientMock.On("DeleteVolume", mock.Anything, mock.AnythingOfType("*gopowerstore.VolumeDelete"), @@ -1380,11 +2516,11 @@ var _ = Describe("CSIControllerService", func() { res, err := ctrlSvc.DeleteVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.DeleteVolumeResponse{})) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.DeleteVolumeResponse{})) }) - It("should succeed [NFS]", func() { + ginkgo.It("should succeed [NFS]", func() { clientMock.On("GetFsSnapshotsByVolumeID", mock.Anything, validBaseVolID).Return([]gopowerstore.FileSystem{}, nil) clientMock.On("GetNFSExportByFileSystemID", mock.Anything, validBaseVolID). Return(gopowerstore.NFSExport{}, nil) @@ -1401,15 +2537,16 @@ var _ = Describe("CSIControllerService", func() { res, err := ctrlSvc.DeleteVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.DeleteVolumeResponse{})) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.DeleteVolumeResponse{})) }) }) - When("block volume still attached to host", func() { - It("should fail", func() { + ginkgo.When("block volume still attached to host", func() { + ginkgo.It("should fail", func() { clientMock.On("GetSnapshotsByVolumeID", mock.Anything, validBaseVolID).Return([]gopowerstore.Volume{}, nil) clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID).Return(gopowerstore.VolumeGroups{}, nil) + clientMock.On("GetVolume", mock.Anything, validBaseVolID).Return(gopowerstore.Volume{ID: validBaseVolID}, nil) clientMock.On("DeleteVolume", mock.Anything, mock.AnythingOfType("*gopowerstore.VolumeDelete"), @@ -1424,14 +2561,14 @@ var _ = Describe("CSIControllerService", func() { res, err := ctrlSvc.DeleteVolume(context.Background(), req) - Expect(err).ToNot(BeNil()) - Expect(res).To(BeNil()) - Expect(err.Error()).To(ContainSubstring("volume with ID '" + validBaseVolID + "' is still attached to host")) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("volume with ID '" + validBaseVolID + "' is still attached to host")) }) }) - When("can not connect to API", func() { - It("should fail [Block]", func() { + ginkgo.When("can not connect to API", func() { + ginkgo.It("should fail [Block]", func() { e := errors.New("can't connect") clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID).Return(gopowerstore.VolumeGroups{}, e) clientMock.On("GetSnapshotsByVolumeID", mock.Anything, validBaseVolID). @@ -1441,12 +2578,12 @@ var _ = Describe("CSIControllerService", func() { res, err := ctrlSvc.DeleteVolume(context.Background(), req) - Expect(err).ToNot(BeNil()) - Expect(res).To(BeNil()) - Expect(err.Error()).To(ContainSubstring(e.Error())) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring(e.Error())) }) - It("should fail [NFS]", func() { + ginkgo.It("should fail [NFS]", func() { e := errors.New("can't connect") clientMock.On("GetFsSnapshotsByVolumeID", mock.Anything, validBaseVolID). Return([]gopowerstore.FileSystem{}, e) @@ -1455,57 +2592,256 @@ var _ = Describe("CSIControllerService", func() { res, err := ctrlSvc.DeleteVolume(context.Background(), req) - Expect(err).ToNot(BeNil()) - Expect(res).To(BeNil()) - Expect(err.Error()).To(ContainSubstring("failure getting snapshot")) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("failure getting snapshot")) }) }) - When("volume id contains unsupported protocol", func() { - It("should fail", func() { + ginkgo.When("volume id contains unsupported protocol", func() { + ginkgo.It("should fail", func() { req := &csi.DeleteVolumeRequest{VolumeId: validBaseVolID + "/" + firstValidID + "/smb"} res, err := ctrlSvc.DeleteVolume(context.Background(), req) - Expect(err).ToNot(BeNil()) - Expect(res).To(BeNil()) - Expect(err.Error()).To(ContainSubstring("can't figure out protocol")) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("can't figure out protocol")) }) }) - }) - Describe("calling CreateSnapshot()", func() { - When("parameters are correct", func() { - It("should successfully create new snapshot [Block]", func() { - snapName := validSnapName - clientMock.On("GetVolume", mock.Anything, validBaseVolID).Return( - gopowerstore.Volume{ - Name: "name", - Size: validVolSize, + ginkgo.When("MetroFractured: deleting metro volume when remote array is down", func() { + ginkgo.It("should succeed fail", func() { + // local info + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", Name: "vol1", MetroReplicationSessionID: replicationSessionID, ApplianceID: validApplianceID}, nil) + clientMock.On("GetReplicationSessionByID", mock.Anything, replicationSessionID). + Return(gopowerstore.ReplicationSession{ + ID: replicationSessionID, + State: "Fractured", + LocalResourceState: "Promoted", }, nil) + clientMock.On("GetSnapshotsByVolumeID", mock.Anything, mock.Anything).Return([]gopowerstore.Volume{}, nil) + clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, mock.Anything).Return(gopowerstore.VolumeGroups{}, nil) + clientMock.On("DeleteVolume", + mock.Anything, + mock.AnythingOfType("*gopowerstore.VolumeDelete"), + validBaseVolID). + Return(gopowerstore.EmptyResponse(""), nil) + clientMock.On("EndMetroVolume", mock.Anything, mock.Anything, mock.Anything).Return(gopowerstore.EmptyResponse(""), nil) - clientMock.On("GetVolumeByName", mock.Anything, validSnapName).Return( - gopowerstore.Volume{}, errors.New("not nil")) - - clientMock.On("CreateSnapshot", mock.Anything, &gopowerstore.SnapshotCreate{ - Name: &snapName, - Description: nil, - }, validBaseVolID).Return(gopowerstore.CreateResponse{ID: "new-snap-id"}, nil) + // remote info + clientMock.On("GetVolume", mock.Anything, validRemoteVolID). + Return(gopowerstore.Volume{ID: validRemoteVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", Name: "vol1", ApplianceID: validRemoteApplianceID}, errors.New("timeout")) + clientMock.On("DeleteVolume", + mock.Anything, + mock.AnythingOfType("*gopowerstore.VolumeDelete"), + validRemoteVolID). + Return(gopowerstore.EmptyResponse(""), errors.New("timeout")) - req := &csi.CreateSnapshotRequest{ - SourceVolumeId: validBlockVolumeID, - Name: validSnapName, - } + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, firstValidID, "scsi", validRemoteVolID, secondValidID) + req := &csi.DeleteVolumeRequest{VolumeId: volumeID} - res, err := ctrlSvc.CreateSnapshot(context.Background(), req) + res, err := ctrlSvc.DeleteVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res.Snapshot.SnapshotId).To(Equal("new-snap-id/globalvolid1/scsi")) - Expect(res.Snapshot.SizeBytes).To(Equal(int64(validVolSize))) - Expect(res.Snapshot.SourceVolumeId).To(Equal(validBaseVolID)) + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(res).To(gomega.BeNil()) }) + }) - It("should successfully create new snapshot [NFS]", func() { + ginkgo.When("MetroFractured: when local array is down", func() { + ginkgo.It("DeleteVolume should fail ", func() { + // local info + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", Name: "vol1", MetroReplicationSessionID: replicationSessionID, ApplianceID: validApplianceID}, nil) + clientMock.On("GetReplicationSessionByID", mock.Anything, replicationSessionID). + Return(gopowerstore.ReplicationSession{ + ID: replicationSessionID, + State: "Fractured", + LocalResourceState: "Promoted", + }, nil) + clientMock.On("GetSnapshotsByVolumeID", mock.Anything, mock.Anything).Return([]gopowerstore.Volume{}, nil) + clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, mock.Anything).Return(gopowerstore.VolumeGroups{}, nil) + clientMock.On("DeleteVolume", + mock.Anything, + mock.AnythingOfType("*gopowerstore.VolumeDelete"), + validBaseVolID). + Return(gopowerstore.EmptyResponse(""), errors.New("timeout")) + clientMock.On("EndMetroVolume", mock.Anything, mock.Anything, mock.Anything).Return(gopowerstore.EmptyResponse(""), nil) + + // remote info + clientMock.On("GetVolume", mock.Anything, validRemoteVolID). + Return(gopowerstore.Volume{ID: validRemoteVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", Name: "vol1", ApplianceID: validRemoteApplianceID}, errors.New("timeout")) + + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, firstValidID, "scsi", validRemoteVolID, secondValidID) + req := &csi.DeleteVolumeRequest{VolumeId: volumeID} + + res, err := ctrlSvc.DeleteVolume(context.Background(), req) + + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(res).To(gomega.BeNil()) + }) + }) + + ginkgo.When("MetroFractured: no array is down", func() { + ginkgo.It("should succeed", func() { + // local info + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", Name: "vol1", ApplianceID: validApplianceID, MetroReplicationSessionID: replicationSessionID}, nil) + clientMock.On("GetReplicationSessionByID", mock.Anything, replicationSessionID). + Return(gopowerstore.ReplicationSession{ + ID: replicationSessionID, + State: "Fractured", + LocalResourceState: "Promoted", + }, nil) + clientMock.On("GetSnapshotsByVolumeID", mock.Anything, validBaseVolID).Return([]gopowerstore.Volume{}, nil) + clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID).Return(gopowerstore.VolumeGroups{}, nil) + clientMock.On("DeleteVolume", + mock.Anything, + mock.AnythingOfType("*gopowerstore.VolumeDelete"), + validBaseVolID). + Return(gopowerstore.EmptyResponse(""), nil) + clientMock.On("EndMetroVolume", mock.Anything, mock.Anything, mock.Anything).Return(gopowerstore.EmptyResponse(""), nil) + + // remote info + clientMock.On("GetVolume", mock.Anything, validRemoteVolID). + Return(gopowerstore.Volume{ID: validRemoteVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", Name: "vol1", ApplianceID: validRemoteApplianceID, MetroReplicationSessionID: replicationSessionID}, nil) + clientMock.On("GetSnapshotsByVolumeID", mock.Anything, validRemoteVolID).Return([]gopowerstore.Volume{}, nil) + clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validRemoteVolID).Return(gopowerstore.VolumeGroups{}, nil) + clientMock.On("DeleteVolume", + mock.Anything, + mock.AnythingOfType("*gopowerstore.VolumeDelete"), + validRemoteVolID). + Return(gopowerstore.EmptyResponse(""), nil) + + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, firstValidID, "scsi", validRemoteVolID, secondValidID) + req := &csi.DeleteVolumeRequest{VolumeId: volumeID} + + res, err := ctrlSvc.DeleteVolume(context.Background(), req) + + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.DeleteVolumeResponse{})) + }) + + ginkgo.It("succeed: metro only contains promote/demote snapshots", func() { + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", Name: "vol1", ApplianceID: validApplianceID, MetroReplicationSessionID: replicationSessionID}, nil) + clientMock.On("GetReplicationSessionByID", mock.Anything, replicationSessionID). + Return(gopowerstore.ReplicationSession{ + ID: replicationSessionID, + State: "Ok", + LocalResourceState: "Promoted", + }, nil) + clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID).Return(gopowerstore.VolumeGroups{}, nil) + clientMock.On("GetSnapshotsByVolumeID", mock.Anything, validBaseVolID).Return([]gopowerstore.Volume{{ID: "snap-id-1", Name: "Metro_Promote_Backup_Snapshot." + "vol1"}}, nil) + clientMock.On("EndMetroVolume", mock.Anything, mock.Anything, mock.Anything).Return(gopowerstore.EmptyResponse(""), nil) + clientMock.On("DeleteVolume", + mock.Anything, + mock.AnythingOfType("*gopowerstore.VolumeDelete"), + validBaseVolID). + Return(gopowerstore.EmptyResponse(""), nil) + req := &csi.DeleteVolumeRequest{VolumeId: validBaseVolID} + res, err := ctrlSvc.DeleteVolume(context.Background(), req) + + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.DeleteVolumeResponse{})) + }) + + ginkgo.It("failure: metro volume contains snapshot not promote/demote", func() { + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", Name: "vol1", ApplianceID: validApplianceID, MetroReplicationSessionID: replicationSessionID}, nil) + clientMock.On("GetReplicationSessionByID", mock.Anything, replicationSessionID). + Return(gopowerstore.ReplicationSession{ + ID: replicationSessionID, + State: "Ok", + LocalResourceState: "Promoted", + }, nil) + clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID).Return(gopowerstore.VolumeGroups{}, nil) + clientMock.On("GetSnapshotsByVolumeID", mock.Anything, validBaseVolID).Return([]gopowerstore.Volume{{ID: "snap-id-1", Name: "myLocalSnapshot"}}, nil) + clientMock.On("EndMetroVolume", mock.Anything, mock.Anything, mock.Anything).Return(gopowerstore.EmptyResponse(""), nil) + req := &csi.DeleteVolumeRequest{VolumeId: validBaseVolID} + res, err := ctrlSvc.DeleteVolume(context.Background(), req) + + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("snapshots based on this volume still exist")) + }) + }) + + ginkgo.When("MetroFractured: both array down", func() { + ginkgo.It("should fail", func() { + // local info + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", Name: "vol1", ApplianceID: validApplianceID, MetroReplicationSessionID: replicationSessionID}, errors.New("timeout")) + clientMock.On("GetReplicationSessionByID", mock.Anything, replicationSessionID). + Return(gopowerstore.ReplicationSession{ + ID: replicationSessionID, + State: "Fractured", + LocalResourceState: "Promoted", + }, nil) + clientMock.On("GetSnapshotsByVolumeID", mock.Anything, validBaseVolID).Return([]gopowerstore.Volume{}, errors.New("timeout")) + clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID).Return(gopowerstore.VolumeGroups{}, errors.New("timeout")) + clientMock.On("DeleteVolume", + mock.Anything, + mock.AnythingOfType("*gopowerstore.VolumeDelete"), + validBaseVolID). + Return(gopowerstore.EmptyResponse(""), errors.New("timeout")) + clientMock.On("EndMetroVolume", mock.Anything, mock.Anything, mock.Anything).Return(gopowerstore.EmptyResponse(""), errors.New("timeout")) + + // remote info + clientMock.On("GetVolume", mock.Anything, validRemoteVolID). + Return(gopowerstore.Volume{ID: validRemoteVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", Name: "vol1", ApplianceID: validRemoteApplianceID, MetroReplicationSessionID: replicationSessionID}, errors.New("timeout")) + clientMock.On("GetSnapshotsByVolumeID", mock.Anything, validRemoteVolID).Return([]gopowerstore.Volume{}, errors.New("timeout")) + clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validRemoteVolID).Return(gopowerstore.VolumeGroups{}, errors.New("timeout")) + clientMock.On("DeleteVolume", + mock.Anything, + mock.AnythingOfType("*gopowerstore.VolumeDelete"), + validRemoteVolID). + Return(gopowerstore.EmptyResponse(""), errors.New("timeout")) + + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, firstValidID, "scsi", validRemoteVolID, secondValidID) + req := &csi.DeleteVolumeRequest{VolumeId: volumeID} + + res, err := ctrlSvc.DeleteVolume(context.Background(), req) + + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(res).To(gomega.BeNil()) + }) + }) + }) + + ginkgo.Describe("calling CreateSnapshot()", func() { + ginkgo.When("parameters are correct", func() { + ginkgo.It("should successfully create new snapshot [Block]", func() { + snapName := validSnapName + clientMock.On("GetVolume", mock.Anything, validBaseVolID).Return( + gopowerstore.Volume{ + Name: "name", + Size: validVolSize, + }, nil) + + clientMock.On("GetVolumeByName", mock.Anything, validSnapName).Return( + gopowerstore.Volume{}, errors.New("not nil")) + + clientMock.On("CreateSnapshot", mock.Anything, &gopowerstore.SnapshotCreate{ + Name: &snapName, + Description: nil, + }, validBaseVolID).Return(gopowerstore.CreateResponse{ID: "new-snap-id"}, nil) + + req := &csi.CreateSnapshotRequest{ + SourceVolumeId: validBlockVolumeID, + Name: validSnapName, + } + + res, err := ctrlSvc.CreateSnapshot(context.Background(), req) + + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res.Snapshot.SnapshotId).To(gomega.Equal("new-snap-id/globalvolid1/scsi")) + gomega.Expect(res.Snapshot.SizeBytes).To(gomega.Equal(int64(validVolSize))) + gomega.Expect(res.Snapshot.SourceVolumeId).To(gomega.Equal(validBaseVolID)) + }) + + ginkgo.It("should successfully create new snapshot [NFS]", func() { snapName := validSnapName clientMock.On("GetFS", mock.Anything, validBaseVolID).Return( gopowerstore.FileSystem{ @@ -1528,15 +2864,58 @@ var _ = Describe("CSIControllerService", func() { res, err := ctrlSvc.CreateSnapshot(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res.Snapshot.SnapshotId).To(Equal("new-snap-id/globalvolid2/nfs")) - Expect(res.Snapshot.SizeBytes).To(Equal(int64(validVolSize - controller.ReservedSize))) - Expect(res.Snapshot.SourceVolumeId).To(Equal(validBaseVolID)) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res.Snapshot.SnapshotId).To(gomega.Equal("new-snap-id/globalvolid2/nfs")) + gomega.Expect(res.Snapshot.SizeBytes).To(gomega.Equal(int64(validVolSize - ReservedSize))) + gomega.Expect(res.Snapshot.SourceVolumeId).To(gomega.Equal(validBaseVolID)) + }) + }) + + ginkgo.When("snapshot name is empty", func() { + ginkgo.It("should fail", func() { + req := &csi.CreateSnapshotRequest{ + SourceVolumeId: validBlockVolumeID, + Name: "", + } + + res, err := ctrlSvc.CreateSnapshot(context.Background(), req) + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("name cannot be empty")) + }) + }) + + ginkgo.When("snapshot volume sourceVolID is empty", func() { + ginkgo.It("should fail [sourceVolumeId is empty]", func() { + req := &csi.CreateSnapshotRequest{ + SourceVolumeId: "", + Name: validSnapName, + } + + res, err := ctrlSvc.CreateSnapshot(context.Background(), req) + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("volume ID to be snapped is required")) + }) + }) + + ginkgo.When("the array ID could not found", func() { + ginkgo.It("should return error", func() { + req := &csi.CreateSnapshotRequest{ + SourceVolumeId: invalidBlockVolumeID, + Name: validSnapName, + } + + res, err := ctrlSvc.CreateSnapshot(context.Background(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("failed to find array with given ID")) }) }) - When("snapshot name already taken", func() { - It("should fail [sourceVolumeId != snap.sourceVolumeId]", func() { + ginkgo.When("snapshot name already taken", func() { + ginkgo.It("should fail [sourceVolumeId != snap.sourceVolumeId]", func() { clientMock.On("GetVolume", mock.Anything, validBaseVolID).Return( gopowerstore.Volume{ Name: "name", @@ -1560,12 +2939,12 @@ var _ = Describe("CSIControllerService", func() { res, err := ctrlSvc.CreateSnapshot(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).NotTo(BeNil()) - Expect(err.Error()).To(ContainSubstring(fmt.Sprintf("snapshot with name '%s' exists, but SourceVolumeId %s doesn't match", "my-snap", validBaseVolID))) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring(fmt.Sprintf("snapshot with name '%s' exists, but SourceVolumeId %s doesn't match", "my-snap", validBaseVolID))) }) - It("should succeed [same sourceVolumeId]", func() { + ginkgo.It("should succeed [same sourceVolumeId]", func() { clientMock.On("GetVolume", mock.Anything, validBaseVolID).Return( gopowerstore.Volume{ Name: "name", @@ -1589,15 +2968,49 @@ var _ = Describe("CSIControllerService", func() { res, err := ctrlSvc.CreateSnapshot(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res.Snapshot.SnapshotId).To(Equal("old-snap-id/globalvolid1/scsi")) - Expect(res.Snapshot.SizeBytes).To(Equal(int64(validVolSize))) - Expect(res.Snapshot.SourceVolumeId).To(Equal(validBaseVolID)) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res.Snapshot.SnapshotId).To(gomega.Equal("old-snap-id/globalvolid1/scsi")) + gomega.Expect(res.Snapshot.SizeBytes).To(gomega.Equal(int64(validVolSize))) + gomega.Expect(res.Snapshot.SourceVolumeId).To(gomega.Equal(validBaseVolID)) + }) + }) + + ginkgo.When("there is an API error when retrieving the source", func() { + ginkgo.It("should fail [Block]", func() { + clientMock.On("GetVolume", mock.Anything, validBaseVolID).Return( + gopowerstore.Volume{}, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{StatusCode: http.StatusGatewayTimeout}, + }) + + req := &csi.CreateSnapshotRequest{ + SourceVolumeId: validBlockVolumeID, + Name: validSnapName, + } + _, err := ctrlSvc.CreateSnapshot(context.Background(), req) + + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("can't find source volume")) + }) + + ginkgo.It("should fail [NFS]", func() { + clientMock.On("GetFS", mock.Anything, validBaseVolID).Return( + gopowerstore.FileSystem{}, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{StatusCode: http.StatusGatewayTimeout}, + }) + + req := &csi.CreateSnapshotRequest{ + SourceVolumeId: validNfsVolumeID, + Name: validSnapName, + } + _, err := ctrlSvc.CreateSnapshot(context.Background(), req) + + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("can't find source volume")) }) }) - When("there is an API error when creating snapshot", func() { - It("should return that error", func() { + ginkgo.When("there is an API error when creating snapshot", func() { + ginkgo.It("should return that error", func() { snapName := validSnapName clientMock.On("GetVolume", mock.Anything, validBaseVolID).Return( gopowerstore.Volume{ @@ -1625,16 +3038,16 @@ var _ = Describe("CSIControllerService", func() { res, err := ctrlSvc.CreateSnapshot(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring("something went wrong")) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("something went wrong")) }) }) }) - Describe("calling DeleteSnapshot()", func() { - When("parameters are correct", func() { - It("should successfully delete snapshot [Block]", func() { + ginkgo.Describe("calling DeleteSnapshot()", func() { + ginkgo.When("parameters are correct", func() { + ginkgo.It("should successfully delete snapshot [Block]", func() { clientMock.On("GetSnapshot", mock.Anything, validBaseVolID).Return(gopowerstore.Volume{ID: validBaseVolID}, nil) clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID). Return(gopowerstore.VolumeGroups{VolumeGroup: []gopowerstore.VolumeGroup{{ID: validGroupID, Name: validVolumeGroupName}}}, nil) @@ -1648,11 +3061,11 @@ var _ = Describe("CSIControllerService", func() { res, err := ctrlSvc.DeleteSnapshot(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.DeleteSnapshotResponse{})) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.DeleteSnapshotResponse{})) }) - It("should successfully delete snapshot [NFS]", func() { + ginkgo.It("should successfully delete snapshot [NFS]", func() { clientMock.On("GetFsSnapshot", mock.Anything, validBaseVolID).Return(gopowerstore.FileSystem{}, nil) clientMock.On("DeleteFsSnapshot", mock.Anything, validBaseVolID).Return(gopowerstore.EmptyResponse(""), nil) @@ -1663,13 +3076,13 @@ var _ = Describe("CSIControllerService", func() { res, err := ctrlSvc.DeleteSnapshot(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.DeleteSnapshotResponse{})) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.DeleteSnapshotResponse{})) }) }) - When("there is no snapshot", func() { - It("should return no error [Block]", func() { + ginkgo.When("there is no snapshot", func() { + ginkgo.It("should return no error [Block]", func() { clientMock.On("GetSnapshot", mock.Anything, validBaseVolID).Return(gopowerstore.Volume{}, nil) clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, ""). Return(gopowerstore.VolumeGroups{VolumeGroup: []gopowerstore.VolumeGroup{}}, nil) @@ -1687,11 +3100,11 @@ var _ = Describe("CSIControllerService", func() { res, err := ctrlSvc.DeleteSnapshot(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.DeleteSnapshotResponse{})) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.DeleteSnapshotResponse{})) }) - It("should return no error [NFS]", func() { + ginkgo.It("should return no error [NFS]", func() { clientMock.On("GetFsSnapshot", mock.Anything, validBaseVolID).Return(gopowerstore.FileSystem{}, nil) clientMock.On("DeleteFsSnapshot", mock.Anything, validBaseVolID).Return(gopowerstore.EmptyResponse(""), gopowerstore.APIError{ @@ -1706,13 +3119,13 @@ var _ = Describe("CSIControllerService", func() { res, err := ctrlSvc.DeleteSnapshot(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.DeleteSnapshotResponse{})) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.DeleteSnapshotResponse{})) }) }) - When("there is no such source volume", func() { - It("should return no error", func() { + ginkgo.When("there is no such source volume", func() { + ginkgo.It("should return no error", func() { clientMock.On("GetSnapshot", mock.Anything, validBaseVolID).Return(gopowerstore.Volume{}, gopowerstore.APIError{ ErrorMsg: &api.ErrorMsg{ StatusCode: http.StatusNotFound, @@ -1724,14 +3137,40 @@ var _ = Describe("CSIControllerService", func() { _, err := ctrlSvc.DeleteSnapshot(context.Background(), req) - Expect(err).To(BeNil()) + gomega.Expect(err).To(gomega.BeNil()) + }) + }) + + ginkgo.When("the request is not valid", func() { + ginkgo.It("should return error", func() { + req := &csi.DeleteSnapshotRequest{ + SnapshotId: "", + } + + _, err := ctrlSvc.DeleteSnapshot(context.Background(), req) + + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("snapshot ID to be deleted is required")) + }) + }) + + ginkgo.When("the array ID could not found", func() { + ginkgo.It("should return error", func() { + req := &csi.DeleteSnapshotRequest{ + SnapshotId: invalidBlockVolumeID, + } + + _, err := ctrlSvc.DeleteSnapshot(context.Background(), req) + + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("failed to find array with given ID")) }) }) }) - Describe("calling ControllerExpandVolume()", func() { - When("expanding scsi volume", func() { - It("should successfully expand scsi volume", func() { + ginkgo.Describe("calling ControllerExpandVolume()", func() { + ginkgo.When("expanding scsi volume", func() { + ginkgo.It("should successfully expand scsi volume", func() { clientMock.On("GetVolume", mock.Anything, validBaseVolID).Return(gopowerstore.Volume{ Size: validVolSize, }, nil) @@ -1745,50 +3184,136 @@ var _ = Describe("CSIControllerService", func() { res, err := ctrlSvc.ControllerExpandVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.ControllerExpandVolumeResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerExpandVolumeResponse{ CapacityBytes: validVolSize * 2, NodeExpansionRequired: true, })) }) - When("not able to get volume info", func() { - It("should fail", func() { - e := errors.New("some-api-error") - clientMock.On("GetVolume", mock.Anything, validBaseVolID).Return(gopowerstore.Volume{}, e) + ginkgo.It("should successfully expand scsi volume when metro is enabled", func() { + clientMock.On("GetVolume", mock.Anything, validBaseVolID).Return(gopowerstore.Volume{ + MetroReplicationSessionID: validSessionID, + Size: validVolSize, + }, nil) + clientMock.On("ModifyVolume", + mock.Anything, + mock.AnythingOfType("*gopowerstore.VolumeModify"), + validBaseVolID). + Return(gopowerstore.EmptyResponse(""), nil) + // Return metro session status as paused + clientMock.On("GetReplicationSessionByID", mock.Anything, validSessionID).Return(gopowerstore.ReplicationSession{ + ID: validSessionID, + State: gopowerstore.RsStatePaused, + }, nil).Times(1) - req := getTypicalControllerExpandRequest(validBlockVolumeID, validVolSize*2) - _, err := ctrlSvc.ControllerExpandVolume(context.Background(), req) + req := getTypicalControllerExpandRequest(validMetroBlockVolumeID, validVolSize*2) + res, err := ctrlSvc.ControllerExpandVolume(context.Background(), req) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring("detected SCSI protocol but wasn't able to fetch the volume info")) - }) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerExpandVolumeResponse{ + CapacityBytes: validVolSize * 2, + NodeExpansionRequired: true, + })) }) - When("not able to modify volume", func() { - It("should fail", func() { - e := errors.New("some-api-error") - clientMock.On("GetVolume", mock.Anything, validBaseVolID).Return(gopowerstore.Volume{ - Size: validVolSize, - }, nil) - clientMock.On("ModifyVolume", - mock.Anything, - mock.AnythingOfType("*gopowerstore.VolumeModify"), - validBaseVolID). - Return(gopowerstore.EmptyResponse(""), e) + ginkgo.It("should return empty response when current size is already larger than requested size", func() { + clientMock.On("GetVolume", mock.Anything, validBaseVolID).Return(gopowerstore.Volume{ + Size: validVolSize * 3, + }, nil) - req := getTypicalControllerExpandRequest(validBlockVolumeID, validVolSize*2) + req := getTypicalControllerExpandRequest(validBlockVolumeID, validVolSize*2) + res, err := ctrlSvc.ControllerExpandVolume(context.Background(), req) - _, err := ctrlSvc.ControllerExpandVolume(context.Background(), req) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerExpandVolumeResponse{})) + }) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring(e.Error())) - }) + ginkgo.It("should fail to find array ID", func() { + req := getTypicalControllerExpandRequest(invalidBlockVolumeID, validVolSize*2) + _, err := ctrlSvc.ControllerExpandVolume(context.Background(), req) + + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("unable to find array with ID")) + }) + + ginkgo.It("should fail to get volume info", func() { + e := errors.New("some-api-error") + clientMock.On("GetVolume", mock.Anything, validBaseVolID).Return(gopowerstore.Volume{}, e) + + req := getTypicalControllerExpandRequest(validBlockVolumeID, validVolSize*2) + _, err := ctrlSvc.ControllerExpandVolume(context.Background(), req) + + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("detected SCSI protocol but wasn't able to fetch the volume info")) + }) + + ginkgo.It("should fail to modify volume", func() { + e := errors.New("some-api-error") + clientMock.On("GetVolume", mock.Anything, validBaseVolID).Return(gopowerstore.Volume{ + Size: validVolSize, + }, nil) + clientMock.On("ModifyVolume", + mock.Anything, + mock.AnythingOfType("*gopowerstore.VolumeModify"), + validBaseVolID). + Return(gopowerstore.EmptyResponse(""), e) + + req := getTypicalControllerExpandRequest(validBlockVolumeID, validVolSize*2) + _, err := ctrlSvc.ControllerExpandVolume(context.Background(), req) + + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("unable to modify volume size")) + }) + + ginkgo.It("should fail to identify metro volume", func() { + clientMock.On("GetVolume", mock.Anything, validBaseVolID).Return(gopowerstore.Volume{ + Size: validVolSize, + }, nil) + + req := getTypicalControllerExpandRequest(validMetroBlockVolumeID, validVolSize*2) + _, err := ctrlSvc.ControllerExpandVolume(context.Background(), req) + + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("metro replication session ID is empty for metro volume")) + }) + + ginkgo.It("should fail to get metro session", func() { + e := errors.New("some-api-error") + clientMock.On("GetVolume", mock.Anything, validBaseVolID).Return(gopowerstore.Volume{ + MetroReplicationSessionID: validSessionID, + Size: validVolSize, + }, nil) + clientMock.On("GetReplicationSessionByID", mock.Anything, validSessionID).Return(gopowerstore.ReplicationSession{}, e).Times(1) + + req := getTypicalControllerExpandRequest(validMetroBlockVolumeID, validVolSize*2) + _, err := ctrlSvc.ControllerExpandVolume(context.Background(), req) + + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("could not get metro replication session")) + }) + + ginkgo.It("should fail if metro session is not paused", func() { + clientMock.On("GetVolume", mock.Anything, validBaseVolID).Return(gopowerstore.Volume{ + MetroReplicationSessionID: validSessionID, + Size: validVolSize, + }, nil) + // Return error state for pause failure + clientMock.On("GetReplicationSessionByID", mock.Anything, validSessionID).Return(gopowerstore.ReplicationSession{ + ID: validSessionID, + State: gopowerstore.RsStateOk, + }, nil).Times(1) + + req := getTypicalControllerExpandRequest(validMetroBlockVolumeID, validVolSize*2) + _, err := ctrlSvc.ControllerExpandVolume(context.Background(), req) + + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("Please pause the metro replication session manually")) }) }) - When("expanding nfs volume", func() { - It("should successfully expand nfs volume", func() { + ginkgo.When("expanding nfs volume", func() { + ginkgo.It("should successfully expand nfs volume", func() { clientMock.On("GetFS", mock.Anything, validBaseVolID).Return(gopowerstore.FileSystem{ SizeTotal: validVolSize, }, nil) @@ -1802,15 +3327,15 @@ var _ = Describe("CSIControllerService", func() { res, err := ctrlSvc.ControllerExpandVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.ControllerExpandVolumeResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerExpandVolumeResponse{ CapacityBytes: validVolSize * 2, NodeExpansionRequired: false, })) }) - When("not able to modify filesystem", func() { - It("should fail", func() { + ginkgo.When("not able to modify filesystem", func() { + ginkgo.It("should fail", func() { e := errors.New("some-api-error") clientMock.On("GetFS", mock.Anything, validBaseVolID).Return(gopowerstore.FileSystem{ SizeTotal: validVolSize, @@ -1825,14 +3350,14 @@ var _ = Describe("CSIControllerService", func() { _, err := ctrlSvc.ControllerExpandVolume(context.Background(), req) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring(e.Error())) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring(e.Error())) }) }) }) - When("volume id is incorrect", func() { - It("should fail", func() { + ginkgo.When("volume id is incorrect", func() { + ginkgo.It("should fail", func() { e := errors.New("api-error") clientMock.On("GetVolume", mock.Anything, validBaseVolID).Return(gopowerstore.Volume{}, e) clientMock.On("GetFS", mock.Anything, validBaseVolID).Return(gopowerstore.FileSystem{}, e) @@ -1840,30 +3365,30 @@ var _ = Describe("CSIControllerService", func() { req := getTypicalControllerExpandRequest(validBaseVolID, validVolSize*2) _, err := ctrlSvc.ControllerExpandVolume(context.Background(), req) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring("unable to parse the volume id")) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("unable to parse the volume id")) }) }) - When("requested size exceeds limit", func() { - It("should fail", func() { - req := getTypicalControllerExpandRequest(validBlockVolumeID, controller.MaxVolumeSizeBytes+1) + ginkgo.When("requested size exceeds limit", func() { + ginkgo.It("should fail", func() { + req := getTypicalControllerExpandRequest(validBlockVolumeID, MaxVolumeSizeBytes+1) _, err := ctrlSvc.ControllerExpandVolume(context.Background(), req) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring("volume exceeds allowed limit")) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("volume exceeds allowed limit")) }) }) }) - Describe("calling ControllerPublishVolume()", func() { + ginkgo.Describe("calling ControllerPublishVolume()", func() { fsName := "testFS" nfsID := "1ae5edac1-a796-886a-47dc-c72a3j8clw031" nasID := "some-nas-id" interfaceID := "215as1223-d124-ss1h-njh4-c72a3j8clw031" - When("parameters are correct", func() { - It("should succeed [Block]", func() { + ginkgo.When("parameters are correct", func() { + ginkgo.It("should succeed [Block]", func() { clientMock.On("GetVolume", mock.Anything, validBaseVolID). Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9"}, nil) @@ -1885,7 +3410,13 @@ var _ = Describe("CSIControllerService", func() { IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn"}, }, }, nil) - + clientMock.On("GetStorageNVMETCPTargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn"}, + }, + }, nil) clientMock.On("GetFCPorts", mock.Anything). Return([]gopowerstore.FcPort{ { @@ -1900,25 +3431,20 @@ var _ = Describe("CSIControllerService", func() { Return(gopowerstore.Cluster{Name: validClusterName, NVMeNQN: "nqn"}, nil) req := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, validBlockVolumeID) - req.VolumeContext = map[string]string{controller.KeyFsType: "xfs"} + req.VolumeContext = map[string]string{KeyFsType: "xfs"} res, err := ctrlSvc.ControllerPublishVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.ControllerPublishVolumeResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerPublishVolumeResponse{ PublishContext: map[string]string{ - "PORTAL0": "192.168.1.1:3260", - "TARGET0": "iqn", - "NVMEFCPORTAL0": "nn-0x58ccf090c9200c22:pn-0x58ccf091492b0c22", - "NVMEFCTARGET0": "nqn", - "DEVICE_WWN": "68ccf098003ceb5e4577a20be6d11bf9", - "LUN_ADDRESS": "1", - "FCWWPN0": "58ccf09348a003a3", + "DEVICE_WWN": "68ccf098003ceb5e4577a20be6d11bf9", + "LUN_ADDRESS": "1", }, })) }) - It("should succeed [NFS]", func() { + ginkgo.It("should succeed [NFS]", func() { clientMock.On("GetFS", mock.Anything, validBaseVolID). Return(gopowerstore.FileSystem{ ID: validBaseVolID, @@ -1950,20 +3476,20 @@ var _ = Describe("CSIControllerService", func() { clientMock.On("GetNAS", mock.Anything, nasID). Return(gopowerstore.NAS{ Name: validNasName, - CurrentPreferredIPv4InterfaceId: interfaceID, + CurrentPreferredIPv4InterfaceID: interfaceID, }, nil) clientMock.On("GetFileInterface", mock.Anything, interfaceID). - Return(gopowerstore.FileInterface{IpAddress: secondValidID}, nil) + Return(gopowerstore.FileInterface{IPAddress: secondValidID}, nil) req := getTypicalControllerPublishVolumeRequest("multiple-writer", validNodeID, validNfsVolumeID) req.VolumeCapability = getVolumeCapabilityNFS() - req.VolumeContext = map[string]string{controller.KeyFsType: "nfs"} + req.VolumeContext = map[string]string{KeyFsType: "nfs"} res, err := ctrlSvc.ControllerPublishVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.ControllerPublishVolumeResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerPublishVolumeResponse{ PublishContext: map[string]string{ "nasName": validNasName, "NfsExportPath": secondValidID + ":/", @@ -1974,10 +3500,10 @@ var _ = Describe("CSIControllerService", func() { }, })) }) - It("should succeed [NFS] with externalAccess", func() { - //setting externalAccess environment variable - err := csictx.Setenv(context.Background(), common.EnvExternalAccess, "10.0.0.0/24") - Expect(err).To(BeNil()) + ginkgo.It("should succeed [NFS] with externalAccess", func() { + // setting externalAccess environment variable + err := csictx.Setenv(context.Background(), identifiers.EnvExternalAccess, "10.0.0.0/24") + gomega.Expect(err).To(gomega.BeNil()) _ = ctrlSvc.Init() clientMock.On("GetFS", mock.Anything, validBaseVolID). @@ -2014,47 +3540,119 @@ var _ = Describe("CSIControllerService", func() { clientMock.On("GetNAS", mock.Anything, nasID). Return(gopowerstore.NAS{ Name: validNasName, - CurrentPreferredIPv4InterfaceId: interfaceID, + CurrentPreferredIPv4InterfaceID: interfaceID, }, nil) clientMock.On("GetFileInterface", mock.Anything, interfaceID). - Return(gopowerstore.FileInterface{IpAddress: secondValidID}, nil) + Return(gopowerstore.FileInterface{IPAddress: secondValidID}, nil) req := getTypicalControllerPublishVolumeRequest("multiple-writer", validNodeID, validNfsVolumeID) req.VolumeCapability = getVolumeCapabilityNFS() - req.VolumeContext = map[string]string{controller.KeyFsType: "nfs"} + req.VolumeContext = map[string]string{KeyFsType: "nfs"} res, err := ctrlSvc.ControllerPublishVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.ControllerPublishVolumeResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerPublishVolumeResponse{ PublishContext: map[string]string{ - common.KeyNasName: validNasName, - common.KeyNfsExportPath: secondValidID + ":/", - common.KeyExportID: nfsID, - common.KeyAllowRoot: "", - common.KeyHostIP: "127.0.0.1", - common.KeyNfsACL: "", - common.KeyNatIP: "10.0.0.0/255.255.255.0", + identifiers.KeyNasName: validNasName, + identifiers.KeyNfsExportPath: secondValidID + ":/", + identifiers.KeyExportID: nfsID, + identifiers.KeyAllowRoot: "", + identifiers.KeyHostIP: "127.0.0.1", + identifiers.KeyNfsACL: "", + identifiers.KeyNatIP: "10.0.0.0/255.255.255.0", }, })) // Removing externalAccess environment variable after our tests are completed - err = csictx.Setenv(context.Background(), common.EnvExternalAccess, "") - Expect(err).To(BeNil()) + err = csictx.Setenv(context.Background(), identifiers.EnvExternalAccess, "") + gomega.Expect(err).To(gomega.BeNil()) _ = ctrlSvc.Init() }) - }) - When("host name does not contain ip", func() { - It("should truncate ip from kubeID and succeed [Block]", func() { - clientMock.On("GetVolume", mock.Anything, validBaseVolID). - Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9"}, nil) + ginkgo.It("should succeed [NFS] with externalAccess and exclusiveAccess enabled", func() { + // setting externalAccess and exclusiveAccess environment variable + err := csictx.Setenv(context.Background(), identifiers.EnvExternalAccess, "10.0.0.0/24") + gomega.Expect(err).To(gomega.BeNil()) + err = csictx.Setenv(context.Background(), identifiers.EnvExclusiveAccess, "true") + gomega.Expect(err).To(gomega.BeNil()) + _ = ctrlSvc.Init() - By("truncating ip", func() { - clientMock.On("GetHostByName", mock.Anything, validNodeID). - Return(gopowerstore.Host{}, gopowerstore.APIError{ - ErrorMsg: &api.ErrorMsg{ - StatusCode: http.StatusNotFound, + clientMock.On("GetFS", mock.Anything, validBaseVolID). + Return(gopowerstore.FileSystem{ + ID: validBaseVolID, + Name: fsName, + NasServerID: nasID, + }, nil) + + apiError := gopowerstore.NewAPIError() + apiError.StatusCode = http.StatusNotFound + + clientMock.On("GetNFSExportByFileSystemID", mock.Anything, mock.Anything). + Return(gopowerstore.NFSExport{}, *apiError).Once() + + nfsExportCreate := &gopowerstore.NFSExportCreate{ + Name: fsName, + FileSystemID: validBaseVolID, + Path: "/" + fsName, + } + clientMock.On("CreateNFSExport", mock.Anything, nfsExportCreate). + Return(gopowerstore.CreateResponse{ID: nfsID}, nil) + + clientMock.On("GetNFSExportByFileSystemID", mock.Anything, mock.Anything). + Return(gopowerstore.NFSExport{ID: nfsID}, nil).Once() + + clientMock.On("ModifyNFSExport", mock.Anything, &gopowerstore.NFSExportModify{ + AddRWRootHosts: []string{ + "10.0.0.0/255.255.255.0", + }, + }, nfsID).Return(gopowerstore.CreateResponse{}, nil) + + clientMock.On("GetNAS", mock.Anything, nasID). + Return(gopowerstore.NAS{ + Name: validNasName, + CurrentPreferredIPv4InterfaceID: interfaceID, + }, nil) + + clientMock.On("GetFileInterface", mock.Anything, interfaceID). + Return(gopowerstore.FileInterface{IPAddress: secondValidID}, nil) + + req := getTypicalControllerPublishVolumeRequest("multiple-writer", validNodeID, validNfsVolumeID) + req.VolumeCapability = getVolumeCapabilityNFS() + req.VolumeContext = map[string]string{KeyFsType: "nfs"} + + res, err := ctrlSvc.ControllerPublishVolume(context.Background(), req) + + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerPublishVolumeResponse{ + PublishContext: map[string]string{ + identifiers.KeyNasName: validNasName, + identifiers.KeyNfsExportPath: secondValidID + ":/", + identifiers.KeyExportID: nfsID, + identifiers.KeyAllowRoot: "", + identifiers.KeyNfsACL: "", + identifiers.KeyNatIP: "10.0.0.0/255.255.255.0", + }, + })) + // Removing externalAccess and exclusiveAccess environment variable after our tests are completed + err = csictx.Setenv(context.Background(), identifiers.EnvExternalAccess, "") + gomega.Expect(err).To(gomega.BeNil()) + err = csictx.Setenv(context.Background(), identifiers.EnvExclusiveAccess, "") + gomega.Expect(err).To(gomega.BeNil()) + _ = ctrlSvc.Init() + }) + }) + + ginkgo.When("host name does not contain ip", func() { + ginkgo.It("should truncate ip from kubeID and succeed [Block]", func() { + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9"}, nil) + + ginkgo.By("truncating ip", func() { + clientMock.On("GetHostByName", mock.Anything, validNodeID). + Return(gopowerstore.Host{}, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{ + StatusCode: http.StatusNotFound, }, }).Once() @@ -2078,7 +3676,13 @@ var _ = Describe("CSIControllerService", func() { IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn"}, }, }, nil) - + clientMock.On("GetStorageNVMETCPTargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn"}, + }, + }, nil) clientMock.On("GetFCPorts", mock.Anything). Return([]gopowerstore.FcPort{ { @@ -2090,15 +3694,13 @@ var _ = Describe("CSIControllerService", func() { Return(gopowerstore.Cluster{Name: validClusterName}, nil) req := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, validBlockVolumeID) - req.VolumeContext = map[string]string{controller.KeyFsType: "xfs"} + req.VolumeContext = map[string]string{KeyFsType: "xfs"} res, err := ctrlSvc.ControllerPublishVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.ControllerPublishVolumeResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerPublishVolumeResponse{ PublishContext: map[string]string{ - "PORTAL0": "192.168.1.1:3260", - "TARGET0": "iqn", "DEVICE_WWN": "68ccf098003ceb5e4577a20be6d11bf9", "LUN_ADDRESS": "1", }, @@ -2106,10 +3708,10 @@ var _ = Describe("CSIControllerService", func() { }) }) - When("using nfs nat feature", func() { - It("should succeed", func() { + ginkgo.When("using nfs nat feature", func() { + ginkgo.It("should succeed", func() { externalAccess := "10.0.0.1" - _ = csictx.Setenv(context.Background(), common.EnvExternalAccess, externalAccess) + _ = csictx.Setenv(context.Background(), identifiers.EnvExternalAccess, externalAccess) _ = ctrlSvc.Init() clientMock.On("GetFS", mock.Anything, validBaseVolID). @@ -2143,20 +3745,20 @@ var _ = Describe("CSIControllerService", func() { clientMock.On("GetNAS", mock.Anything, nasID). Return(gopowerstore.NAS{ Name: validNasName, - CurrentPreferredIPv4InterfaceId: interfaceID, + CurrentPreferredIPv4InterfaceID: interfaceID, }, nil) clientMock.On("GetFileInterface", mock.Anything, interfaceID). - Return(gopowerstore.FileInterface{IpAddress: secondValidID}, nil) + Return(gopowerstore.FileInterface{IPAddress: secondValidID}, nil) req := getTypicalControllerPublishVolumeRequest("multi-writer", validNodeID, validNfsVolumeID) req.VolumeCapability = getVolumeCapabilityNFS() - req.VolumeContext = map[string]string{controller.KeyFsType: "nfs"} + req.VolumeContext = map[string]string{KeyFsType: "nfs"} res, err := ctrlSvc.ControllerPublishVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.ControllerPublishVolumeResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerPublishVolumeResponse{ PublishContext: map[string]string{ "nasName": validNasName, "NfsExportPath": secondValidID + ":/", @@ -2170,9 +3772,9 @@ var _ = Describe("CSIControllerService", func() { }) }) - When("volume is already attached to some host", func() { - When("mapping has same hostID", func() { - It("should succeed", func() { + ginkgo.When("volume is already attached to some host", func() { + ginkgo.When("mapping has same hostID", func() { + ginkgo.It("should succeed", func() { clientMock.On("GetVolume", mock.Anything, validBaseVolID). Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9"}, nil) @@ -2188,7 +3790,13 @@ var _ = Describe("CSIControllerService", func() { IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn"}, }, }, nil) - + clientMock.On("GetStorageNVMETCPTargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn"}, + }, + }, nil) clientMock.On("GetFCPorts", mock.Anything). Return([]gopowerstore.FcPort{ { @@ -2200,15 +3808,13 @@ var _ = Describe("CSIControllerService", func() { Return(gopowerstore.Cluster{Name: validClusterName}, nil) req := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, validBlockVolumeID) - req.VolumeContext = map[string]string{controller.KeyFsType: "xfs"} + req.VolumeContext = map[string]string{KeyFsType: "xfs"} res, err := ctrlSvc.ControllerPublishVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.ControllerPublishVolumeResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerPublishVolumeResponse{ PublishContext: map[string]string{ - "PORTAL0": "192.168.1.1:3260", - "TARGET0": "iqn", "DEVICE_WWN": "68ccf098003ceb5e4577a20be6d11bf9", "LUN_ADDRESS": "1", }, @@ -2216,9 +3822,9 @@ var _ = Describe("CSIControllerService", func() { }) }) - When("mapping hostID is different", func() { + ginkgo.When("mapping hostID is different", func() { prevNodeID := "prev-id" - It("should fail [single-writer]", func() { + ginkgo.It("should fail [single-writer]", func() { clientMock.On("GetVolume", mock.Anything, validBaseVolID). Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9"}, nil) @@ -2234,7 +3840,13 @@ var _ = Describe("CSIControllerService", func() { IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn"}, }, }, nil) - + clientMock.On("GetStorageNVMETCPTargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn"}, + }, + }, nil) clientMock.On("GetFCPorts", mock.Anything). Return([]gopowerstore.FcPort{ { @@ -2246,17 +3858,17 @@ var _ = Describe("CSIControllerService", func() { Return(gopowerstore.Cluster{Name: validClusterName}, nil) req := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, validBlockVolumeID) - req.VolumeContext = map[string]string{controller.KeyFsType: "xfs"} + req.VolumeContext = map[string]string{KeyFsType: "xfs"} res, err := ctrlSvc.ControllerPublishVolume(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring( + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring( fmt.Sprintf("volume already present in a different lun mapping on node '%s", prevNodeID))) }) - It("should succeed [multi-writer]", func() { + ginkgo.It("should succeed [multi-writer]", func() { clientMock.On("GetVolume", mock.Anything, validBaseVolID). Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9"}, nil) @@ -2281,7 +3893,13 @@ var _ = Describe("CSIControllerService", func() { IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn"}, }, }, nil) - + clientMock.On("GetStorageNVMETCPTargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn"}, + }, + }, nil) clientMock.On("GetFCPorts", mock.Anything). Return([]gopowerstore.FcPort{ { @@ -2293,15 +3911,13 @@ var _ = Describe("CSIControllerService", func() { Return(gopowerstore.Cluster{Name: validClusterName}, nil) req := getTypicalControllerPublishVolumeRequest("multiple-writer", validNodeID, validBlockVolumeID) - req.VolumeContext = map[string]string{controller.KeyFsType: "xfs"} + req.VolumeContext = map[string]string{KeyFsType: "xfs"} res, err := ctrlSvc.ControllerPublishVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.ControllerPublishVolumeResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerPublishVolumeResponse{ PublishContext: map[string]string{ - "PORTAL0": "192.168.1.1:3260", - "TARGET0": "iqn", "DEVICE_WWN": "68ccf098003ceb5e4577a20be6d11bf9", "LUN_ADDRESS": "2", }, @@ -2310,164 +3926,921 @@ var _ = Describe("CSIControllerService", func() { }) }) - When("volume id is empty", func() { - It("should fail", func() { - req := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, validBlockVolumeID) - - req.VolumeId = "" + ginkgo.When("publishing metro volume", func() { + ginkgo.It("should succeed [Block]", func() { + // local info + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", ApplianceID: validApplianceID}, nil) + clientMock.On("GetHostByName", mock.Anything, validNodeID).Return(gopowerstore.Host{ID: validHostID}, nil) + clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, validBaseVolID). + Return([]gopowerstore.HostVolumeMapping{}, nil).Once() + clientMock.On("AttachVolumeToHost", mock.Anything, validHostID, mock.Anything). + Return(gopowerstore.EmptyResponse(""), nil).Times(2) + clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, validBaseVolID). + Return([]gopowerstore.HostVolumeMapping{{HostID: validHostID, LogicalUnitNumber: 1}}, nil).Once() + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn"}, + ApplianceID: validApplianceID, + }, + }, nil).Once() + clientMock.On("GetStorageNVMETCPTargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn"}, + ApplianceID: validApplianceID, + }, + }, nil).Times(2) + clientMock.On("GetCluster", mock.Anything). + Return(gopowerstore.Cluster{Name: validClusterName, NVMeNQN: "nqn"}, nil).Times(2) + clientMock.On("GetFCPorts", mock.Anything). + Return([]gopowerstore.FcPort{ + { + IsLinkUp: true, + Wwn: "58:cc:f0:93:48:a0:03:a3", + WwnNVMe: "58ccf091492b0c22", + WwnNode: "58ccf090c9200c22", + ApplianceID: validApplianceID, + }, + }, nil).Times(2) - _, err := ctrlSvc.ControllerPublishVolume(context.Background(), req) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring("volume ID is required")) - }) - }) + // remote info + clientMock.On("GetVolume", mock.Anything, validRemoteVolID). + Return(gopowerstore.Volume{ID: validRemoteVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", ApplianceID: validRemoteApplianceID}, nil) + clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, validRemoteVolID). + Return([]gopowerstore.HostVolumeMapping{}, nil).Once() + clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, validRemoteVolID). + Return([]gopowerstore.HostVolumeMapping{{HostID: validHostID, LogicalUnitNumber: 1}}, nil).Once() + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.2", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn.2015-10.com.dell:dellemc-powerstore-apm00223"}, + ApplianceID: validRemoteApplianceID, + }, + }, nil).Once() + clientMock.On("GetStorageNVMETCPTargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.2", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn.2015-10.com.dell:dellemc-powerstore-apm00223"}, + ApplianceID: validRemoteApplianceID, + }, + }, nil).Times(2) + clientMock.On("GetCluster", mock.Anything). + Return(gopowerstore.Cluster{Name: validClusterName, NVMeNQN: "nqn.1988-11.com.dell:powerstore:00:303030303030ABCDEFGH"}, nil).Times(2) + clientMock.On("GetFCPorts", mock.Anything). + Return([]gopowerstore.FcPort{ + { + IsLinkUp: true, + Wwn: "58:cc:f0:93:48:a0:03:33", + WwnNVMe: "58ccf091492b0c33", + WwnNode: "58ccf090c9200c33", + ApplianceID: validRemoteApplianceID, + }, + }, nil).Times(2) - When("volume capability is missing", func() { - It("should fail", func() { - req := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, validBlockVolumeID) + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, firstValidID, "scsi", validRemoteVolID, secondValidID) + req := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, volumeID) + req.VolumeContext = map[string]string{KeyFsType: "xfs"} - req.VolumeCapability = nil + res, err := ctrlSvc.ControllerPublishVolume(context.Background(), req) - _, err := ctrlSvc.ControllerPublishVolume(context.Background(), req) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring("volume capability is required")) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerPublishVolumeResponse{ + PublishContext: map[string]string{ + "DEVICE_WWN": "68ccf098003ceb5e4577a20be6d11bf9", + "LUN_ADDRESS": "1", + "REMOTE_DEVICE_WWN": "68ccf098003ceb5e4577a20be6d11bf9", + "REMOTE_LUN_ADDRESS": "1", + }, + })) }) - }) - - When("access mode is missing", func() { - It("should fail", func() { - req := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, validBlockVolumeID) - req.VolumeCapability.AccessMode = nil + ginkgo.It("should fail", func() { + ip := "127.0.0.1" // we don't have array with this IP + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, firstValidID, "scsi", validRemoteVolID, ip) + req := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, volumeID) + req.VolumeContext = map[string]string{KeyFsType: "xfs"} + req.VolumeCapability = nil _, err := ctrlSvc.ControllerPublishVolume(context.Background(), req) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring("access mode is required")) - }) - }) - - When("access mode is unknown", func() { - It("should fail", func() { - req := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, validBlockVolumeID) - - req.VolumeCapability.AccessMode.Mode = csi.VolumeCapability_AccessMode_UNKNOWN - _, err := ctrlSvc.ControllerPublishVolume(context.Background(), req) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring(controller.ErrUnknownAccessMode)) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("failed to find remote array with ID")) }) }) - When("kube node id is empty", func() { - It("should fail", func() { - req := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, validBlockVolumeID) + ginkgo.When("When attach fails for a non-metro volume ", func() { + ginkgo.It("should fail", func() { + // local info + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", ApplianceID: validApplianceID}, nil) + clientMock.On("GetHostByName", mock.Anything, validNodeID).Return(gopowerstore.Host{ID: validHostID}, nil) + clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, validBaseVolID). + Return([]gopowerstore.HostVolumeMapping{}, nil).Once() + clientMock.On("AttachVolumeToHost", mock.Anything, validHostID, mock.Anything). + Return(gopowerstore.EmptyResponse(""), errors.New("some error")).Times(2) - req.NodeId = "" + volumeID := fmt.Sprintf("%s/%s/%s", validBaseVolID, firstValidID, "scsi") + req := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, volumeID) + req.VolumeContext = map[string]string{KeyFsType: "xfs"} - _, err := ctrlSvc.ControllerPublishVolume(context.Background(), req) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring("node ID is required")) + res, err := ctrlSvc.ControllerPublishVolume(context.Background(), req) + + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(res).To(gomega.BeNil()) }) }) - When("volume does not exist", func() { - It("should fail [Block]", func() { + ginkgo.When("MetroFractured: publishing metro volume when one array is down", func() { + ginkgo.It("should succeed [Block] when remote array is down and local was promoted", func() { + // local info clientMock.On("GetVolume", mock.Anything, validBaseVolID). - Return(gopowerstore.Volume{}, gopowerstore.APIError{ - ErrorMsg: &api.ErrorMsg{ - StatusCode: http.StatusNotFound, + Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", Name: "vol1", MetroReplicationSessionID: replicationSessionID, ApplianceID: validApplianceID}, nil) + clientMock.On("GetReplicationSessionByID", mock.Anything, replicationSessionID). + Return(gopowerstore.ReplicationSession{ + ID: replicationSessionID, + State: "Fractured", + LocalResourceState: "Promoted", + }, nil) + clientMock.On("GetHostByName", mock.Anything, validNodeID).Return(gopowerstore.Host{ID: validHostID}, nil) + clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, validBaseVolID). + Return([]gopowerstore.HostVolumeMapping{}, nil).Once() + clientMock.On("AttachVolumeToHost", mock.Anything, validHostID, mock.Anything). + Return(gopowerstore.EmptyResponse(""), nil).Times(2) + clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, validBaseVolID). + Return([]gopowerstore.HostVolumeMapping{{HostID: validHostID, LogicalUnitNumber: 1}}, nil).Once() + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn"}, + ApplianceID: validApplianceID, }, - }) - - req := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, validBlockVolumeID) - req.VolumeContext = map[string]string{controller.KeyFsType: "xfs"} + }, nil).Once() + clientMock.On("GetStorageNVMETCPTargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn"}, + ApplianceID: validApplianceID, + }, + }, nil).Times(2) + clientMock.On("GetCluster", mock.Anything). + Return(gopowerstore.Cluster{Name: validClusterName, NVMeNQN: "nqn"}, nil).Times(2) + clientMock.On("GetFCPorts", mock.Anything). + Return([]gopowerstore.FcPort{ + { + IsLinkUp: true, + Wwn: "58:cc:f0:93:48:a0:03:a3", + WwnNVMe: "58ccf091492b0c22", + WwnNode: "58ccf090c9200c22", + ApplianceID: validApplianceID, + }, + }, nil).Times(2) - _, err := ctrlSvc.ControllerPublishVolume(context.Background(), req) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring(fmt.Sprintf("volume with ID '%s' not found", validBaseVolID))) - }) + // remote info + clientMock.On("GetVolume", mock.Anything, validRemoteVolID). + Return(gopowerstore.Volume{ID: validRemoteVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", Name: "vol1", ApplianceID: validRemoteApplianceID}, errors.New("timeout")) - It("should fail [NFS]", func() { - clientMock.On("GetFS", mock.Anything, validBaseVolID). - Return(gopowerstore.FileSystem{}, gopowerstore.APIError{ - ErrorMsg: &api.ErrorMsg{ - StatusCode: http.StatusNotFound, - }, - }) + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, firstValidID, "scsi", validRemoteVolID, secondValidID) + req := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, volumeID) + req.VolumeContext = map[string]string{KeyFsType: "xfs"} - req := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, validNfsVolumeID) - req.VolumeContext = map[string]string{controller.KeyFsType: "xfs"} + res, err := ctrlSvc.ControllerPublishVolume(context.Background(), req) - _, err := ctrlSvc.ControllerPublishVolume(context.Background(), req) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring(fmt.Sprintf("volume with ID '%s' not found", validBaseVolID))) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerPublishVolumeResponse{ + PublishContext: map[string]string{ + "DEVICE_WWN": "68ccf098003ceb5e4577a20be6d11bf9", + "LUN_ADDRESS": "1", + }, + })) }) + }) - When("using v1.2 volume id", func() { - It("should fail", func() { - e := errors.New("api-error") - clientMock.On("GetVolume", mock.Anything, validBaseVolID).Return(gopowerstore.Volume{}, e) - clientMock.On("GetFS", mock.Anything, validBaseVolID).Return(gopowerstore.FileSystem{}, e) + ginkgo.When("MetroFractured: when local volume was promoted, but local publish failed", func() { + ginkgo.It("publishVolume should fail ", func() { + // local info + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", Name: "vol1", MetroReplicationSessionID: replicationSessionID, ApplianceID: validApplianceID}, nil) + clientMock.On("GetReplicationSessionByID", mock.Anything, replicationSessionID). + Return(gopowerstore.ReplicationSession{ + ID: replicationSessionID, + State: "Fractured", + LocalResourceState: "Promoted", + }, nil) + clientMock.On("GetHostByName", mock.Anything, validNodeID).Return(gopowerstore.Host{ID: validHostID}, nil) + clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, validBaseVolID). + Return([]gopowerstore.HostVolumeMapping{}, nil).Once() + clientMock.On("AttachVolumeToHost", mock.Anything, validHostID, mock.Anything). + Return(gopowerstore.EmptyResponse(""), errors.New("some error")).Times(2) - req := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, validBaseVolID) - req.VolumeContext = map[string]string{controller.KeyFsType: "xfs"} - req.VolumeCapability = nil + // remote info + clientMock.On("GetVolume", mock.Anything, validRemoteVolID). + Return(nil, errors.New("timeout")) - _, err := ctrlSvc.ControllerPublishVolume(context.Background(), req) + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, firstValidID, "scsi", validRemoteVolID, secondValidID) + req := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, volumeID) + req.VolumeContext = map[string]string{KeyFsType: "xfs"} - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring("failure checking volume status")) - }) + res, err := ctrlSvc.ControllerPublishVolume(context.Background(), req) + + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(res).To(gomega.BeNil()) }) }) - When("node id is not valid", func() { - It("should fail [Block]", func() { + ginkgo.When("MetroFractured: remote volume was promoted and local array was down", func() { + ginkgo.It("should succeed", func() { + // local info clientMock.On("GetVolume", mock.Anything, validBaseVolID). - Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9"}, nil) + Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", Name: "vol1", MetroReplicationSessionID: replicationSessionID, ApplianceID: validApplianceID}, errors.New("timeout")) - clientMock.On("GetHostByName", mock.Anything, validNodeID). - Return(gopowerstore.Host{}, gopowerstore.APIError{ - ErrorMsg: &api.ErrorMsg{ - StatusCode: http.StatusNotFound, + // remote info + clientMock.On("GetVolume", mock.Anything, validRemoteVolID). + Return(gopowerstore.Volume{ID: validRemoteVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", Name: "vol1", MetroReplicationSessionID: replicationSessionID, ApplianceID: validRemoteApplianceID}, nil) + clientMock.On("GetReplicationSessionByID", mock.Anything, replicationSessionID). + Return(gopowerstore.ReplicationSession{ + ID: replicationSessionID, + State: "Fractured", + LocalResourceState: "Promoted", + }, nil) + clientMock.On("GetHostByName", mock.Anything, validNodeID).Return(gopowerstore.Host{ID: validHostID}, nil) + clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, validRemoteVolID). + Return([]gopowerstore.HostVolumeMapping{}, nil).Once() + clientMock.On("AttachVolumeToHost", mock.Anything, validHostID, mock.Anything). + Return(gopowerstore.EmptyResponse(""), nil) + clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, validRemoteVolID). + Return([]gopowerstore.HostVolumeMapping{{HostID: validHostID, LogicalUnitNumber: 1}}, nil).Once() + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.2", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn.2015-10.com.dell:dellemc-powerstore-apm00223"}, + ApplianceID: validRemoteApplianceID, }, - }).Once() - - clientMock.On("GetHostByName", mock.Anything, validHostName). - Return(gopowerstore.Host{}, gopowerstore.APIError{ - ErrorMsg: &api.ErrorMsg{ - StatusCode: http.StatusNotFound, + }, nil).Once() + clientMock.On("GetStorageNVMETCPTargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.2", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn.2015-10.com.dell:dellemc-powerstore-apm00223"}, + ApplianceID: validRemoteApplianceID, }, - }).Once() + }, nil).Times(2) + clientMock.On("GetCluster", mock.Anything). + Return(gopowerstore.Cluster{Name: validClusterName, NVMeNQN: "nqn.1988-11.com.dell:powerstore:00:303030303030ABCDEFGH"}, nil).Times(2) + clientMock.On("GetFCPorts", mock.Anything). + Return([]gopowerstore.FcPort{ + { + IsLinkUp: true, + Wwn: "58:cc:f0:93:48:a0:03:33", + WwnNVMe: "58ccf091492b0c33", + WwnNode: "58ccf090c9200c33", + ApplianceID: validRemoteApplianceID, + }, + }, nil).Times(2) - req := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, validBlockVolumeID) - req.VolumeContext = map[string]string{controller.KeyFsType: "xfs"} + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, firstValidID, "scsi", validRemoteVolID, secondValidID) + req := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, volumeID) + req.VolumeContext = map[string]string{KeyFsType: "xfs"} - _, err := ctrlSvc.ControllerPublishVolume(context.Background(), req) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring("host with k8s node ID '" + validNodeID + "' not found")) + res, err := ctrlSvc.ControllerPublishVolume(context.Background(), req) + + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerPublishVolumeResponse{ + PublishContext: map[string]string{ + "REMOTE_DEVICE_WWN": "68ccf098003ceb5e4577a20be6d11bf9", + "REMOTE_LUN_ADDRESS": "1", + }, + })) }) }) - When("ip is incorrect", func() { - It("should fail", func() { - ip := "127.0.0.1" // we don't have array with this ip - req := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, - validBaseVolID+"/"+ip+"/scsi") - req.VolumeContext = map[string]string{controller.KeyFsType: "xfs"} - req.VolumeCapability = nil - - _, err := ctrlSvc.ControllerPublishVolume(context.Background(), req) + ginkgo.When("MetroFractured: when remote volume was promoted, but remote publish failed", func() { + ginkgo.It("publishVolume should fail ", func() { + // local info + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", Name: "vol1", MetroReplicationSessionID: replicationSessionID, ApplianceID: validApplianceID}, nil) + clientMock.On("GetReplicationSessionByID", mock.Anything, replicationSessionID). + Return(gopowerstore.ReplicationSession{ + ID: replicationSessionID, + State: "Fractured", + LocalResourceState: "Demoted", + }, nil) + clientMock.On("GetHostByName", mock.Anything, validNodeID).Return(gopowerstore.Host{ID: validHostID}, nil) + clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, validBaseVolID). + Return([]gopowerstore.HostVolumeMapping{}, nil) + clientMock.On("AttachVolumeToHost", mock.Anything, validHostID, mock.Anything). + Return(gopowerstore.EmptyResponse(""), errors.New("some error")).Times(2) + + // remote info + clientMock.On("GetVolume", mock.Anything, validRemoteVolID). + Return(gopowerstore.Volume{ID: validRemoteVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", Name: "vol1", MetroReplicationSessionID: replicationSessionID, ApplianceID: validRemoteApplianceID}, nil) + clientMock.On("GetReplicationSessionByID", mock.Anything, replicationSessionID). + Return(gopowerstore.ReplicationSession{ + ID: replicationSessionID, + State: "Fractured", + LocalResourceState: "Promoted", + }, nil) + clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, validRemoteVolID). + Return([]gopowerstore.HostVolumeMapping{}, nil) + + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, firstValidID, "scsi", validRemoteVolID, secondValidID) + req := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, volumeID) + req.VolumeContext = map[string]string{KeyFsType: "xfs"} + + res, err := ctrlSvc.ControllerPublishVolume(context.Background(), req) + + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(res).To(gomega.BeNil()) + }) + }) + + ginkgo.When("publishing metro volume with locator", func() { + ginkgo.BeforeEach(func() { + // local info + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", ApplianceID: validApplianceID}, nil) + clientMock.On("GetHostByName", mock.Anything, validNodeID).Return(gopowerstore.Host{ID: validHostID}, nil) + clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, validBaseVolID). + Return([]gopowerstore.HostVolumeMapping{}, nil).Once() + clientMock.On("AttachVolumeToHost", mock.Anything, validHostID, mock.Anything). + Return(gopowerstore.EmptyResponse(""), nil).Times(2) + clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, validBaseVolID). + Return([]gopowerstore.HostVolumeMapping{{HostID: validHostID, LogicalUnitNumber: 1}}, nil).Once() + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn"}, + ApplianceID: validApplianceID, + }, + }, nil).Once() + clientMock.On("GetStorageNVMETCPTargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn"}, + ApplianceID: validApplianceID, + }, + }, nil).Times(2) + clientMock.On("GetCluster", mock.Anything). + Return(gopowerstore.Cluster{Name: validClusterName, NVMeNQN: "nqn"}, nil).Times(2) + clientMock.On("GetFCPorts", mock.Anything). + Return([]gopowerstore.FcPort{ + { + IsLinkUp: true, + Wwn: "58:cc:f0:93:48:a0:03:a3", + WwnNVMe: "58ccf091492b0c22", + WwnNode: "58ccf090c9200c22", + ApplianceID: validApplianceID, + }, + }, nil).Times(2) + + // remote info + clientMock.On("GetVolume", mock.Anything, validRemoteVolID). + Return(gopowerstore.Volume{ID: validRemoteVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", ApplianceID: validRemoteApplianceID}, nil) + clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, validRemoteVolID). + Return([]gopowerstore.HostVolumeMapping{}, nil).Once() + clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, validRemoteVolID). + Return([]gopowerstore.HostVolumeMapping{{HostID: validHostID, LogicalUnitNumber: 1}}, nil).Once() + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.2", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn.2015-10.com.dell:dellemc-powerstore-apm00223"}, + ApplianceID: validRemoteApplianceID, + }, + }, nil).Once() + clientMock.On("GetStorageNVMETCPTargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.2", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn.2015-10.com.dell:dellemc-powerstore-apm00223"}, + ApplianceID: validRemoteApplianceID, + }, + }, nil).Times(2) + clientMock.On("GetCluster", mock.Anything). + Return(gopowerstore.Cluster{Name: validClusterName, NVMeNQN: "nqn.1988-11.com.dell:powerstore:00:303030303030ABCDEFGH"}, nil).Times(2) + clientMock.On("GetFCPorts", mock.Anything). + Return([]gopowerstore.FcPort{ + { + IsLinkUp: true, + Wwn: "58:cc:f0:93:48:a0:03:33", + WwnNVMe: "58ccf091492b0c33", + WwnNode: "58ccf090c9200c33", + ApplianceID: validRemoteApplianceID, + }, + }, nil).Times(2) + + metroArr := &array.PowerStoreArray{ + HostConnectivity: &array.HostConnectivity{ + Local: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"nonu1"}, + }, + }, + }, + }, + }, + + Metro: array.MetroConnectivityOptions{ + ColocatedLocal: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"zone1"}, + }, + }, + }, + }, + }, + ColocatedRemote: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"zone2"}, + }, + }, + }, + }, + }, + ColocatedBoth: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"zone3"}, + }, + }, + }, + }, + }, + }, + }, + GlobalID: "APM0012345", + Endpoint: "https://192.168.0.3/api/rest", + Username: "admin", + Password: "pass", + BlockProtocol: identifiers.ISCSITransport, + Insecure: true, + Client: clientMock, + IP: "192.168.0.3", + NASCooldownTracker: array.NewNASCooldown(time.Minute, 5), + } + metroRemoteArr := &array.PowerStoreArray{ + HostConnectivity: &array.HostConnectivity{ + Local: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"nonu2"}, + }, + }, + }, + }, + }, + + Metro: array.MetroConnectivityOptions{ + ColocatedLocal: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"zone2"}, + }, + }, + }, + }, + }, + ColocatedRemote: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"zone1"}, + }, + }, + }, + }, + }, + ColocatedBoth: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"zone3"}, + }, + }, + }, + }, + }, + }, + }, + GlobalID: "APM0045678", + Endpoint: "https://192.168.1.3/api/rest", + Username: "admin", + Password: "pass", + BlockProtocol: identifiers.ISCSITransport, + Insecure: true, + Client: clientMock, + IP: "192.168.1.3", + NASCooldownTracker: array.NewNASCooldown(time.Minute, 5), + } + arrays := ctrlSvc.Arrays() + arrays["APM0012345"] = metroArr + arrays["APM0045678"] = metroRemoteArr + ctrlSvc.SetArrays(arrays) + }) + + ginkgo.It("should pass with node selector passing [Block]", func() { + // Create a node that matches the node selector + node := &k8score.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: validNodeID, + Labels: map[string]string{ + "topology.kubernetes.io/zone": "zone1", // matches the node selector + }, + Annotations: map[string]string{ + "csi.volume.kubernetes.io/nodeid": `{"csi-powerstore.dellemc.com":"` + validNodeID + `"}`, + }, + }, + } + k8sutils.Kubeclient = &k8sutils.K8sClient{ + Clientset: fake.NewClientset([]runtime.Object{node}...), + } + + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, "APM0012345", "scsi", validRemoteVolID, "APM0045678") + + // Publish the volume + publishReq := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, volumeID) + publishReq.VolumeContext = map[string]string{ + KeyFsType: "xfs", + ctrlSvc.WithRP(KeyReplicationMode): "METRO", + ctrlSvc.WithRP(KeyReplicationEnabled): "true", + ctrlSvc.WithRP(KeyReplicationRPO): zeroRPO, + ctrlSvc.WithRP(KeyReplicationRemoteSystem): validRemoteSystemName, + } + + publishResp, err := ctrlSvc.ControllerPublishVolume(context.Background(), publishReq) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + gomega.Expect(publishResp).To(gomega.Equal(&csi.ControllerPublishVolumeResponse{ + PublishContext: map[string]string{ + "DEVICE_WWN": "68ccf098003ceb5e4577a20be6d11bf9", + "LUN_ADDRESS": "1", + "REMOTE_DEVICE_WWN": "68ccf098003ceb5e4577a20be6d11bf9", + "REMOTE_LUN_ADDRESS": "1", + }, + })) + }) + + ginkgo.It("should pass with local node selector passing [Block]", func() { + // Create a node that matches the node selector + node := &k8score.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: validNodeID, + Labels: map[string]string{ + "topology.kubernetes.io/zone": "nonu1", // matches the node selector + }, + Annotations: map[string]string{ + "csi.volume.kubernetes.io/nodeid": `{"csi-powerstore.dellemc.com":"` + validNodeID + `"}`, + }, + }, + } + k8sutils.Kubeclient.Clientset.CoreV1().Nodes().Create(context.Background(), node, metav1.CreateOptions{}) + + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, "APM0012345", "scsi", validRemoteVolID, "APM0045678") + + // Publish the volume + publishReq := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, volumeID) + publishReq.VolumeContext = map[string]string{ + KeyFsType: "xfs", + ctrlSvc.WithRP(KeyReplicationMode): "METRO", + ctrlSvc.WithRP(KeyReplicationEnabled): "true", + ctrlSvc.WithRP(KeyReplicationRPO): zeroRPO, + ctrlSvc.WithRP(KeyReplicationRemoteSystem): validRemoteSystemName, + } + + publishResp, err := ctrlSvc.ControllerPublishVolume(context.Background(), publishReq) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + gomega.Expect(publishResp).To(gomega.Equal(&csi.ControllerPublishVolumeResponse{ + PublishContext: map[string]string{ + "DEVICE_WWN": "68ccf098003ceb5e4577a20be6d11bf9", + "LUN_ADDRESS": "1", + }, + })) + }) + ginkgo.It("should pass with local node selector of remote array passing [Block]", func() { + // Create a node that matches the node selector + node := &k8score.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: validNodeID, + Labels: map[string]string{ + "topology.kubernetes.io/zone": "nonu2", // matches the node selector + }, + Annotations: map[string]string{ + "csi.volume.kubernetes.io/nodeid": `{"csi-powerstore.dellemc.com":"` + validNodeID + `"}`, + }, + }, + } + + k8sutils.Kubeclient = &k8sutils.K8sClient{ + Clientset: fake.NewClientset([]runtime.Object{node}...), + } + + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, "APM0012345", "scsi", validRemoteVolID, "APM0045678") + + // Publish the volume + publishReq := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, volumeID) + publishReq.VolumeContext = map[string]string{ + KeyFsType: "xfs", + ctrlSvc.WithRP(KeyReplicationMode): "METRO", + ctrlSvc.WithRP(KeyReplicationEnabled): "true", + ctrlSvc.WithRP(KeyReplicationRPO): zeroRPO, + ctrlSvc.WithRP(KeyReplicationRemoteSystem): validRemoteSystemName, + } + + publishResp, err := ctrlSvc.ControllerPublishVolume(context.Background(), publishReq) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + gomega.Expect(publishResp).To(gomega.Equal(&csi.ControllerPublishVolumeResponse{ + PublishContext: map[string]string{ + "REMOTE_DEVICE_WWN": "68ccf098003ceb5e4577a20be6d11bf9", + "REMOTE_LUN_ADDRESS": "1", + }, + })) + }) + + ginkgo.It("should fail due to node selector mismatch [Block]", func() { + // Create a node that does not match the node selector + node := &k8score.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: validNodeID, + Labels: map[string]string{ + "topology.kubernetes.io/zone": "zone5", // does not match the node selector + }, + Annotations: map[string]string{ + "csi.volume.kubernetes.io/nodeid": `{"csi-powerstore.dellemc.com":"` + validNodeID + `"}`, + }, + }, + } + k8sutils.Kubeclient = &k8sutils.K8sClient{ + Clientset: fake.NewClientset([]runtime.Object{node}...), + } + + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, "APM0012345", "scsi", validRemoteVolID, "APM0045678") + + // Publish the volume + publishReq := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, volumeID) + publishReq.VolumeContext = map[string]string{ + KeyFsType: "xfs", + ctrlSvc.WithRP(KeyReplicationMode): "METRO", + ctrlSvc.WithRP(KeyReplicationEnabled): "true", + ctrlSvc.WithRP(KeyReplicationRPO): zeroRPO, + ctrlSvc.WithRP(KeyReplicationRemoteSystem): validRemoteSystemName, + } + + _, err := ctrlSvc.ControllerPublishVolume(context.Background(), publishReq) + gomega.Expect(err).To(gomega.HaveOccurred()) + }) + ginkgo.It("should fail due to no node presence [Block]", func() { + k8sutils.Kubeclient = &k8sutils.K8sClient{} + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, "APM0012345", "scsi", validRemoteVolID, "APM0045678") + + // Publish the volume + publishReq := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, volumeID) + publishReq.VolumeContext = map[string]string{ + KeyFsType: "xfs", + ctrlSvc.WithRP(KeyReplicationMode): "METRO", + ctrlSvc.WithRP(KeyReplicationEnabled): "true", + ctrlSvc.WithRP(KeyReplicationRPO): zeroRPO, + ctrlSvc.WithRP(KeyReplicationRemoteSystem): validRemoteSystemName, + } + + _, err := ctrlSvc.ControllerPublishVolume(context.Background(), publishReq) + gomega.Expect(err).To(gomega.HaveOccurred()) + }) + ginkgo.It("should fail due to no remote system [Block]", func() { + // Create a node that matches the node selector + node := &k8score.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: validNodeID, + Labels: map[string]string{ + "topology.kubernetes.io/zone": "zone1", // matches the node selector + }, + Annotations: map[string]string{ + "csi.volume.kubernetes.io/nodeid": `{"csi-powerstore.dellemc.com":"` + validNodeID + `"}`, + }, + }, + } + k8sutils.Kubeclient = &k8sutils.K8sClient{ + Clientset: fake.NewClientset([]runtime.Object{node}...), + } + clientMock.On("GetVolume", mock.Anything, validRemoteVolID+"1"). + Return(gopowerstore.Volume{}, fmt.Errorf("failed to get volume")) + // Invalid remote system + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, "APM0012345", "scsi", validRemoteVolID+"1", "APM0456789") + + // Publish the volume + publishReq := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, volumeID) + publishReq.VolumeContext = map[string]string{ + KeyFsType: "xfs", + ctrlSvc.WithRP(KeyReplicationMode): "METRO", + ctrlSvc.WithRP(KeyReplicationEnabled): "true", + ctrlSvc.WithRP(KeyReplicationRPO): zeroRPO, + ctrlSvc.WithRP(KeyReplicationRemoteSystem): validRemoteSystemName, + } + + _, err := ctrlSvc.ControllerPublishVolume(context.Background(), publishReq) + gomega.Expect(err).To(gomega.HaveOccurred()) + }) + }) + + ginkgo.When("volume id is empty", func() { + ginkgo.It("should fail", func() { + req := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, validBlockVolumeID) + + req.VolumeId = "" + + _, err := ctrlSvc.ControllerPublishVolume(context.Background(), req) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("volume ID is required")) + }) + }) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring("failed to find array with given ID")) + ginkgo.When("volume capability is missing", func() { + ginkgo.It("should fail", func() { + req := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, validBlockVolumeID) + + req.VolumeCapability = nil + + _, err := ctrlSvc.ControllerPublishVolume(context.Background(), req) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("volume capability is required")) + }) + }) + + ginkgo.When("access mode is missing", func() { + ginkgo.It("should fail", func() { + req := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, validBlockVolumeID) + + req.VolumeCapability.AccessMode = nil + + _, err := ctrlSvc.ControllerPublishVolume(context.Background(), req) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("access mode is required")) + }) + }) + + ginkgo.When("access mode is unknown", func() { + ginkgo.It("should fail", func() { + req := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, validBlockVolumeID) + + req.VolumeCapability.AccessMode.Mode = csi.VolumeCapability_AccessMode_UNKNOWN + + _, err := ctrlSvc.ControllerPublishVolume(context.Background(), req) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring(ErrUnknownAccessMode)) + }) + }) + + ginkgo.When("kube node id is empty", func() { + ginkgo.It("should fail", func() { + req := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, validBlockVolumeID) + + req.NodeId = "" + + _, err := ctrlSvc.ControllerPublishVolume(context.Background(), req) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("node ID is required")) + }) + }) + + ginkgo.When("volume does not exist", func() { + ginkgo.It("should fail [Block]", func() { + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{}, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{ + StatusCode: http.StatusNotFound, + }, + }) + + req := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, validBlockVolumeID) + req.VolumeContext = map[string]string{KeyFsType: "xfs"} + + _, err := ctrlSvc.ControllerPublishVolume(context.Background(), req) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring(fmt.Sprintf("volume with ID '%s' not found", validBaseVolID))) + }) + + ginkgo.It("should fail [NFS]", func() { + clientMock.On("GetFS", mock.Anything, validBaseVolID). + Return(gopowerstore.FileSystem{}, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{ + StatusCode: http.StatusNotFound, + }, + }) + + req := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, validNfsVolumeID) + req.VolumeContext = map[string]string{KeyFsType: "xfs"} + + _, err := ctrlSvc.ControllerPublishVolume(context.Background(), req) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring(fmt.Sprintf("volume with ID '%s' not found", validBaseVolID))) + }) + + ginkgo.When("using v1.2 volume id", func() { + ginkgo.It("should fail", func() { + e := errors.New("api-error") + clientMock.On("GetVolume", mock.Anything, validBaseVolID).Return(gopowerstore.Volume{}, e) + clientMock.On("GetFS", mock.Anything, validBaseVolID).Return(gopowerstore.FileSystem{}, e) + + req := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, validBaseVolID) + req.VolumeContext = map[string]string{KeyFsType: "xfs"} + req.VolumeCapability = nil + + _, err := ctrlSvc.ControllerPublishVolume(context.Background(), req) + + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("failure checking volume status")) + }) + }) + }) + + ginkgo.When("node id is not valid", func() { + ginkgo.It("should fail [Block]", func() { + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9"}, nil) + + clientMock.On("GetHostByName", mock.Anything, validNodeID). + Return(gopowerstore.Host{}, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{ + StatusCode: http.StatusNotFound, + }, + }).Once() + + clientMock.On("GetHostByName", mock.Anything, validHostName). + Return(gopowerstore.Host{}, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{ + StatusCode: http.StatusNotFound, + }, + }).Once() + + req := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, validBlockVolumeID) + req.VolumeContext = map[string]string{KeyFsType: "xfs"} + + _, err := ctrlSvc.ControllerPublishVolume(context.Background(), req) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("host with k8s node ID '" + validNodeID + "' not found")) + }) + }) + + ginkgo.When("ip is incorrect", func() { + ginkgo.It("should fail", func() { + ip := "127.0.0.1" // we don't have array with this ip + req := getTypicalControllerPublishVolumeRequest("single-writer", validNodeID, + validBaseVolID+"/"+ip+"/scsi") + req.VolumeContext = map[string]string{KeyFsType: "xfs"} + req.VolumeCapability = nil + + _, err := ctrlSvc.ControllerPublishVolume(context.Background(), req) + + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("failed to find array with ID")) }) }) }) - Describe("calling ControllerUnpublishVolume()", func() { - When("parameters are correct", func() { - It("should succeed [Block]", func() { + ginkgo.Describe("calling ControllerUnpublishVolume()", func() { + ginkgo.When("parameters are correct", func() { + ginkgo.It("should succeed [Block]", func() { clientMock.On("GetHostByName", mock.Anything, validNodeID). Return(gopowerstore.Host{}, gopowerstore.APIError{ ErrorMsg: &api.ErrorMsg{ @@ -2475,6 +4848,9 @@ var _ = Describe("CSIControllerService", func() { }, }).Once() + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9"}, nil) + clientMock.On("GetHostByName", mock.Anything, validHostName). Return(gopowerstore.Host{ID: validHostID}, nil).Once() @@ -2484,11 +4860,11 @@ var _ = Describe("CSIControllerService", func() { req := &csi.ControllerUnpublishVolumeRequest{VolumeId: validBlockVolumeID, NodeId: validNodeID} res, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.ControllerUnpublishVolumeResponse{})) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerUnpublishVolumeResponse{})) }) - It("should succeed [NFS]", func() { + ginkgo.It("should succeed [NFS]", func() { exportID := "some-export-id" clientMock.On("GetFS", mock.Anything, validBaseVolID). @@ -2505,14 +4881,14 @@ var _ = Describe("CSIControllerService", func() { req := &csi.ControllerUnpublishVolumeRequest{VolumeId: validNfsVolumeID, NodeId: validNodeID} res, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.ControllerUnpublishVolumeResponse{})) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerUnpublishVolumeResponse{})) }) - It("should succeed [NFS] with external access", func() { - //setting externalAccess environment variable - err := csictx.Setenv(context.Background(), common.EnvExternalAccess, "10.0.0.0/24") - Expect(err).To(BeNil()) + ginkgo.It("should succeed [NFS] with external access", func() { + // setting externalAccess environment variable + err := csictx.Setenv(context.Background(), identifiers.EnvExternalAccess, "10.0.0.0/24") + gomega.Expect(err).To(gomega.BeNil()) _ = ctrlSvc.Init() exportID := "some-export-id" @@ -2535,92 +4911,630 @@ var _ = Describe("CSIControllerService", func() { req := &csi.ControllerUnpublishVolumeRequest{VolumeId: validNfsVolumeID, NodeId: validNodeID} res, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.ControllerUnpublishVolumeResponse{})) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerUnpublishVolumeResponse{})) - //setting externalAccess environment variable - err = csictx.Setenv(context.Background(), common.EnvExternalAccess, "") - Expect(err).To(BeNil()) + // setting externalAccess environment variable + err = csictx.Setenv(context.Background(), identifiers.EnvExternalAccess, "") + gomega.Expect(err).To(gomega.BeNil()) _ = ctrlSvc.Init() }) }) - It("should succeed [NFS] by removing external access from the HostAccessList", func() { - //setting externalAccess environment variable - err := csictx.Setenv(context.Background(), common.EnvExternalAccess, "10.0.0.0/16") - Expect(err).To(BeNil()) + ginkgo.It("should succeed [NFS] by removing external access from the HostAccessList", func() { + // setting externalAccess environment variable + err := csictx.Setenv(context.Background(), identifiers.EnvExternalAccess, "10.0.0.0/16") + gomega.Expect(err).To(gomega.BeNil()) _ = ctrlSvc.Init() clientMock.On("GetFsSnapshotsByVolumeID", mock.Anything, validBaseVolID).Return([]gopowerstore.FileSystem{}, nil) exportID := "some-export-id" - clientMock.On("GetFS", mock.Anything, validBaseVolID). - Return(gopowerstore.FileSystem{ - ID: validBaseVolID, - }, nil) + clientMock.On("GetFS", mock.Anything, validBaseVolID). + Return(gopowerstore.FileSystem{ + ID: validBaseVolID, + }, nil) + + clientMock.On("GetNFSExportByFileSystemID", mock.Anything, validBaseVolID). + Return(gopowerstore.NFSExport{ + ID: exportID, + RWRootHosts: []string{"10.0.0.0/255.255.0.0"}, + }, nil) + + clientMock.On("ModifyNFSExport", mock.Anything, + mock.Anything, exportID).Return(gopowerstore.CreateResponse{}, nil) + + clientMock.On("DeleteFS", + mock.Anything, + validBaseVolID). + Return(gopowerstore.EmptyResponse(""), nil) + req := &csi.DeleteVolumeRequest{VolumeId: validNfsVolumeID} + + res, err := ctrlSvc.DeleteVolume(context.Background(), req) + + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.DeleteVolumeResponse{})) + + // setting externalAccess environment variable + err = csictx.Setenv(context.Background(), identifiers.EnvExternalAccess, "") + + gomega.Expect(err).To(gomega.BeNil()) + _ = ctrlSvc.Init() + }) + + ginkgo.It("should return error since HostAccessList contain external as well as Host IP too", func() { + // setting externalAccess environment variable + err := csictx.Setenv(context.Background(), identifiers.EnvExternalAccess, "10.0.0.0/16") + gomega.Expect(err).To(gomega.BeNil()) + _ = ctrlSvc.Init() + clientMock.On("GetFsSnapshotsByVolumeID", mock.Anything, validBaseVolID).Return([]gopowerstore.FileSystem{}, nil) + + exportID := "some-export-id" + + clientMock.On("GetFS", mock.Anything, validBaseVolID). + Return(gopowerstore.FileSystem{ + ID: validBaseVolID, + }, nil) + + clientMock.On("GetNFSExportByFileSystemID", mock.Anything, validBaseVolID). + Return(gopowerstore.NFSExport{ + ID: exportID, + RWRootHosts: []string{"10.0.0.0/255.255.0.0", "10.225.0.0/255.255.255.255"}, + }, nil) + + req := &csi.DeleteVolumeRequest{VolumeId: validNfsVolumeID} + + _, err = ctrlSvc.DeleteVolume(context.Background(), req) + + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("cannot be deleted as it has associated NFS or SMB shares")) + // setting externalAccess environment variable + err = csictx.Setenv(context.Background(), identifiers.EnvExternalAccess, "") + + gomega.Expect(err).To(gomega.BeNil()) + _ = ctrlSvc.Init() + }) + + ginkgo.When("unpublishing metro volume", func() { + ginkgo.It("should succeed [Block]", func() { + clientMock.On("GetHostByName", mock.Anything, mock.Anything). + Return(gopowerstore.Host{ID: validHostID}, nil).Times(2) + clientMock.On("DetachVolumeFromHost", mock.Anything, mock.Anything, mock.Anything). + Return(gopowerstore.EmptyResponse(""), nil).Times(2) + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", Name: "vol1", MetroReplicationSessionID: replicationSessionID, ApplianceID: validApplianceID}, nil) + clientMock.On("GetReplicationSessionByID", mock.Anything, replicationSessionID). + Return(gopowerstore.ReplicationSession{ + ID: replicationSessionID, + State: "Active-Active", + LocalResourceState: "System-Defined", + }, nil) + clientMock.On("GetVolume", mock.Anything, validRemoteVolID). + Return(gopowerstore.Volume{ID: validRemoteVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", Name: "vol1", MetroReplicationSessionID: replicationSessionID, ApplianceID: validApplianceID}, nil) + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, firstValidID, "scsi", validRemoteVolID, secondValidID) + req := &csi.ControllerUnpublishVolumeRequest{VolumeId: volumeID, NodeId: validNodeID} + + res, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerUnpublishVolumeResponse{})) + }) + + ginkgo.It("should fail", func() { + ip := "127.0.0.1" // we don't have array with this IP + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, firstValidID, "scsi", validRemoteVolID, ip) + req := &csi.ControllerUnpublishVolumeRequest{VolumeId: volumeID, NodeId: validNodeID} + + _, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) + + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("cannot find remote array")) + }) + + ginkgo.It("should succeed when volume not found on local or remote array", func() { + clientMock.On("GetHostByName", mock.Anything, mock.Anything).Return(gopowerstore.Host{ID: validHostID}, nil) + originalCheckMetroStateFunc := checkMetroStateFunc + checkMetroStateFunc = func(_ context.Context, _ array.VolumeHandle, _ gopowerstore.Client, _ gopowerstore.Client) (*array.MetroFracturedResponse, bool, error) { + return nil, false, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{ + StatusCode: http.StatusNotFound, + }, + } + } + originalIsNodeConnectedToArrayFunc := isNodeConnectedToArrayFunc + isNodeConnectedToArrayFunc = func(_ context.Context, _ string, _ *array.PowerStoreArray) bool { + return true // mocking uniform metro + } + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{}, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{ + StatusCode: http.StatusNotFound, + }, + }) + clientMock.On("GetVolume", mock.Anything, validRemoteVolID). + Return(gopowerstore.Volume{}, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{ + StatusCode: http.StatusNotFound, + }, + }) + defer func() { + isNodeConnectedToArrayFunc = originalIsNodeConnectedToArrayFunc + checkMetroStateFunc = originalCheckMetroStateFunc + }() + + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, firstValidID, "scsi", validRemoteVolID, secondValidID) + req := &csi.ControllerUnpublishVolumeRequest{VolumeId: volumeID, NodeId: validNodeID} + + res, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerUnpublishVolumeResponse{})) + }) + + ginkgo.It("should fail due to checkMetroStateFunc returning error", func() { + clientMock.On("GetHostByName", mock.Anything, mock.Anything).Return(gopowerstore.Host{ID: validHostID}, nil) + originalCheckMetroStateFunc := checkMetroStateFunc + checkMetroStateFunc = func(_ context.Context, _ array.VolumeHandle, _ gopowerstore.Client, _ gopowerstore.Client) (*array.MetroFracturedResponse, bool, error) { + return nil, false, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{ + StatusCode: http.StatusBadGateway, + }, + } + } + defer func() { + checkMetroStateFunc = originalCheckMetroStateFunc + }() + + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, firstValidID, "scsi", validRemoteVolID, secondValidID) + req := &csi.ControllerUnpublishVolumeRequest{VolumeId: volumeID, NodeId: validNodeID} + + _, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) + gomega.Expect(err).ToNot(gomega.BeNil()) + }) + + ginkgo.It("should succeed when metro is fractured, remote array unpublish fails and local array was promoted", func() { + clientMock.On("GetHostByName", mock.Anything, mock.Anything).Return(gopowerstore.Host{ID: validHostID}, nil) + originalCheckMetroStateFunc := checkMetroStateFunc + checkMetroStateFunc = func(_ context.Context, _ array.VolumeHandle, _ gopowerstore.Client, _ gopowerstore.Client) (*array.MetroFracturedResponse, bool, error) { + return &array.MetroFracturedResponse{IsFractured: true, State: "Promoted", VolumeName: "vol1"}, false, nil + } + originalIsNodeConnectedToArrayFunc := isNodeConnectedToArrayFunc + isNodeConnectedToArrayFunc = func(_ context.Context, _ string, _ *array.PowerStoreArray) bool { + return true // mocking uniform metro + } + originalUnpublishVolumeFunc := unpublishVolumeFunc + unpublishVolumeFunc = func(_ context.Context, _ string, arr *array.PowerStoreArray, _ *array.VolumeHandle, _ *array.PowerStoreArray) (*csi.ControllerUnpublishVolumeResponse, error) { + if arr != nil { + return &csi.ControllerUnpublishVolumeResponse{}, nil + } + return nil, errors.New("some-error") + } + originalCreateOrUpdateJournalEntryFunc := createOrUpdateJournalEntryFunc + createOrUpdateJournalEntryFunc = func(_ context.Context, _ string, _ array.VolumeHandle, _ string, _ string, _ string, _ []byte) error { + return nil + } + defer func() { + isNodeConnectedToArrayFunc = originalIsNodeConnectedToArrayFunc + unpublishVolumeFunc = originalUnpublishVolumeFunc + checkMetroStateFunc = originalCheckMetroStateFunc + createOrUpdateJournalEntryFunc = originalCreateOrUpdateJournalEntryFunc + }() + + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, firstValidID, "scsi", validRemoteVolID, secondValidID) + req := &csi.ControllerUnpublishVolumeRequest{VolumeId: volumeID, NodeId: validNodeID} + + res, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerUnpublishVolumeResponse{})) + }) + + ginkgo.It("should fail when metro is fractured, remote array unpublish fails and local array was demoted", func() { + clientMock.On("GetHostByName", mock.Anything, mock.Anything).Return(gopowerstore.Host{ID: validHostID}, nil) + originalCheckMetroStateFunc := checkMetroStateFunc + checkMetroStateFunc = func(_ context.Context, _ array.VolumeHandle, _ gopowerstore.Client, _ gopowerstore.Client) (*array.MetroFracturedResponse, bool, error) { + return &array.MetroFracturedResponse{IsFractured: true, State: "Demoted", VolumeName: "vol1"}, true, nil + } + originalIsNodeConnectedToArrayFunc := isNodeConnectedToArrayFunc + isNodeConnectedToArrayFunc = func(_ context.Context, _ string, _ *array.PowerStoreArray) bool { + return true // mocking uniform metro + } + originalUnpublishVolumeFunc := unpublishVolumeFunc + unpublishVolumeFunc = func(_ context.Context, _ string, arr *array.PowerStoreArray, _ *array.VolumeHandle, _ *array.PowerStoreArray) (*csi.ControllerUnpublishVolumeResponse, error) { + if arr != nil { + return &csi.ControllerUnpublishVolumeResponse{}, nil + } + return nil, errors.New("some-error") + } + defer func() { + isNodeConnectedToArrayFunc = originalIsNodeConnectedToArrayFunc + unpublishVolumeFunc = originalUnpublishVolumeFunc + checkMetroStateFunc = originalCheckMetroStateFunc + }() + + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, firstValidID, "scsi", validRemoteVolID, secondValidID) + req := &csi.ControllerUnpublishVolumeRequest{VolumeId: volumeID, NodeId: validNodeID} + + res, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(res).To(gomega.BeNil()) + }) + + ginkgo.It("should succeed when metro is fractured, local array unpublish fails and remote array was promoted", func() { + clientMock.On("GetHostByName", mock.Anything, mock.Anything).Return(gopowerstore.Host{ID: validHostID}, nil) + originalCheckMetroStateFunc := checkMetroStateFunc + checkMetroStateFunc = func(_ context.Context, _ array.VolumeHandle, _ gopowerstore.Client, _ gopowerstore.Client) (*array.MetroFracturedResponse, bool, error) { + return &array.MetroFracturedResponse{IsFractured: true, State: "Demoted", VolumeName: "vol1"}, true, nil + } + originalIsNodeConnectedToArrayFunc := isNodeConnectedToArrayFunc + isNodeConnectedToArrayFunc = func(_ context.Context, _ string, _ *array.PowerStoreArray) bool { + return true // mocking uniform metro + } + originalUnpublishVolumeFunc := unpublishVolumeFunc + unpublishVolumeFunc = func(_ context.Context, _ string, _ *array.PowerStoreArray, _ *array.VolumeHandle, remoteArray *array.PowerStoreArray) (*csi.ControllerUnpublishVolumeResponse, error) { + if remoteArray != nil { + return &csi.ControllerUnpublishVolumeResponse{}, nil + } + return nil, errors.New("some-error") + } + originalCreateOrUpdateJournalEntryFunc := createOrUpdateJournalEntryFunc + createOrUpdateJournalEntryFunc = func(_ context.Context, _ string, _ array.VolumeHandle, _ string, _ string, _ string, _ []byte) error { + return nil + } + defer func() { + isNodeConnectedToArrayFunc = originalIsNodeConnectedToArrayFunc + unpublishVolumeFunc = originalUnpublishVolumeFunc + checkMetroStateFunc = originalCheckMetroStateFunc + createOrUpdateJournalEntryFunc = originalCreateOrUpdateJournalEntryFunc + }() + + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, firstValidID, "scsi", validRemoteVolID, secondValidID) + req := &csi.ControllerUnpublishVolumeRequest{VolumeId: volumeID, NodeId: validNodeID} + + res, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerUnpublishVolumeResponse{})) + }) + + ginkgo.It("should fail when metro is fractured, local array unpublish fails and local array was promoted", func() { + clientMock.On("GetHostByName", mock.Anything, mock.Anything).Return(gopowerstore.Host{ID: validHostID}, nil) + originalCheckMetroStateFunc := checkMetroStateFunc + checkMetroStateFunc = func(_ context.Context, _ array.VolumeHandle, _ gopowerstore.Client, _ gopowerstore.Client) (*array.MetroFracturedResponse, bool, error) { + return &array.MetroFracturedResponse{IsFractured: true, State: "Promoted", VolumeName: "vol1"}, false, nil + } + originalIsNodeConnectedToArrayFunc := isNodeConnectedToArrayFunc + isNodeConnectedToArrayFunc = func(_ context.Context, _ string, _ *array.PowerStoreArray) bool { + return true // mocking uniform metro + } + originalUnpublishVolumeFunc := unpublishVolumeFunc + unpublishVolumeFunc = func(_ context.Context, _ string, _ *array.PowerStoreArray, _ *array.VolumeHandle, remoteArray *array.PowerStoreArray) (*csi.ControllerUnpublishVolumeResponse, error) { + if remoteArray != nil { + return &csi.ControllerUnpublishVolumeResponse{}, nil + } + return nil, errors.New("some-error") + } + originalCreateOrUpdateJournalEntryFunc := createOrUpdateJournalEntryFunc + createOrUpdateJournalEntryFunc = func(_ context.Context, _ string, _ array.VolumeHandle, _ string, _ string, _ string, _ []byte) error { + return nil + } + defer func() { + isNodeConnectedToArrayFunc = originalIsNodeConnectedToArrayFunc + unpublishVolumeFunc = originalUnpublishVolumeFunc + checkMetroStateFunc = originalCheckMetroStateFunc + createOrUpdateJournalEntryFunc = originalCreateOrUpdateJournalEntryFunc + }() + + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, firstValidID, "scsi", validRemoteVolID, secondValidID) + req := &csi.ControllerUnpublishVolumeRequest{VolumeId: volumeID, NodeId: validNodeID} + + res, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(res).To(gomega.BeNil()) + }) + }) + + ginkgo.When("unpublishing metro volume with selector", func() { + ginkgo.BeforeEach(func() { + // local info + clientMock.On("GetHostByName", mock.Anything, validNodeID).Return(gopowerstore.Host{ID: validHostID}, nil) + clientMock.On("DetachVolumeFromHost", mock.Anything, mock.Anything, mock.Anything). + Return(gopowerstore.EmptyResponse(""), nil).Times(2) + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", Name: "vol1", MetroReplicationSessionID: replicationSessionID, ApplianceID: validApplianceID}, nil) + clientMock.On("GetVolume", mock.Anything, validRemoteVolID). + Return(gopowerstore.Volume{ID: validRemoteVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9", Name: "vol1", MetroReplicationSessionID: replicationSessionID, ApplianceID: validApplianceID}, nil) + clientMock.On("GetReplicationSessionByID", mock.Anything, replicationSessionID). + Return(gopowerstore.ReplicationSession{ + ID: replicationSessionID, + State: "Active-Active", + LocalResourceState: "System-Defined", + }, nil) + + metroArr := &array.PowerStoreArray{ + HostConnectivity: &array.HostConnectivity{ + Local: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"nonu1"}, + }, + }, + }, + }, + }, + + Metro: array.MetroConnectivityOptions{ + ColocatedLocal: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"zone1"}, + }, + }, + }, + }, + }, + ColocatedRemote: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"zone2"}, + }, + }, + }, + }, + }, + ColocatedBoth: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"zone3"}, + }, + }, + }, + }, + }, + }, + }, + GlobalID: "APM0012345", + Endpoint: "https://192.168.0.3/api/rest", + Username: "admin", + Password: "pass", + BlockProtocol: identifiers.ISCSITransport, + Insecure: true, + Client: clientMock, + IP: "192.168.0.3", + NASCooldownTracker: array.NewNASCooldown(time.Minute, 5), + } + metroRemoteArr := &array.PowerStoreArray{ + HostConnectivity: &array.HostConnectivity{ + Local: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"nonu2"}, + }, + }, + }, + }, + }, + + Metro: array.MetroConnectivityOptions{ + ColocatedLocal: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"zone2"}, + }, + }, + }, + }, + }, + ColocatedRemote: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"zone1"}, + }, + }, + }, + }, + }, + ColocatedBoth: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"zone3"}, + }, + }, + }, + }, + }, + }, + }, + GlobalID: "APM0045678", + Endpoint: "https://192.168.1.3/api/rest", + Username: "admin", + Password: "pass", + BlockProtocol: identifiers.ISCSITransport, + Insecure: true, + Client: clientMock, + IP: "192.168.1.3", + NASCooldownTracker: array.NewNASCooldown(time.Minute, 5), + } + arrays := ctrlSvc.Arrays() + arrays["APM0012345"] = metroArr + arrays["APM0045678"] = metroRemoteArr + ctrlSvc.SetArrays(arrays) + }) + + ginkgo.It("should pass with node selector passing [Block]", func() { + // Create a node that matches the node selector + node := &k8score.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: validNodeID, + Labels: map[string]string{ + "topology.kubernetes.io/zone": "zone1", // matches the node selector + }, + Annotations: map[string]string{ + "csi.volume.kubernetes.io/nodeid": `{"csi-powerstore.dellemc.com":"` + validNodeID + `"}`, + }, + }, + } + k8sutils.GetNodeByCSINodeID = func(_ context.Context, _ string, _ string, _ string) (*k8score.Node, error) { + return node, nil + } + + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, "APM0012345", "scsi", validRemoteVolID, "APM0045678") + + req := &csi.ControllerUnpublishVolumeRequest{VolumeId: volumeID, NodeId: validNodeID} + + res, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerUnpublishVolumeResponse{})) + }) + + ginkgo.It("should pass with local node selector passing [Block]", func() { + // Create a node that matches the node selector + node := &k8score.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: validNodeID, + Labels: map[string]string{ + "topology.kubernetes.io/zone": "nonu1", // matches the node selector + }, + Annotations: map[string]string{ + "csi.volume.kubernetes.io/nodeid": `{"csi-powerstore.dellemc.com":"` + validNodeID + `"}`, + }, + }, + } + k8sutils.GetNodeByCSINodeID = func(_ context.Context, _ string, _ string, _ string) (*k8score.Node, error) { + return node, nil + } + + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, "APM0012345", "scsi", validRemoteVolID, "APM0045678") + + req := &csi.ControllerUnpublishVolumeRequest{VolumeId: volumeID, NodeId: validNodeID} - clientMock.On("GetNFSExportByFileSystemID", mock.Anything, validBaseVolID). - Return(gopowerstore.NFSExport{ - ID: exportID, - RWRootHosts: []string{"10.0.0.0/255.255.0.0"}, - }, nil) + res, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerUnpublishVolumeResponse{})) + }) + ginkgo.It("should pass with local node selector of remote array passing [Block]", func() { + // Create a node that matches the node selector + node := &k8score.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: validNodeID, + Labels: map[string]string{ + "topology.kubernetes.io/zone": "nonu2", // matches the node selector + }, + Annotations: map[string]string{ + "csi.volume.kubernetes.io/nodeid": `{"csi-powerstore.dellemc.com":"` + validNodeID + `"}`, + }, + }, + } + k8sutils.GetNodeByCSINodeID = func(_ context.Context, _ string, _ string, _ string) (*k8score.Node, error) { + return node, nil + } - clientMock.On("ModifyNFSExport", mock.Anything, - mock.Anything, exportID).Return(gopowerstore.CreateResponse{}, nil) + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, "APM0012345", "scsi", validRemoteVolID, "APM0045678") - clientMock.On("DeleteFS", - mock.Anything, - validBaseVolID). - Return(gopowerstore.EmptyResponse(""), nil) - req := &csi.DeleteVolumeRequest{VolumeId: validNfsVolumeID} + req := &csi.ControllerUnpublishVolumeRequest{VolumeId: volumeID, NodeId: validNodeID} - res, err := ctrlSvc.DeleteVolume(context.Background(), req) + res, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerUnpublishVolumeResponse{})) + }) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.DeleteVolumeResponse{})) + ginkgo.It("should fail due to node selector mismatch [Block]", func() { + // Create a node that does not match the node selector + node := &k8score.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: validNodeID, + Labels: map[string]string{ + "topology.kubernetes.io/zone": "zone5", // does not match the node selector + }, + Annotations: map[string]string{ + "csi.volume.kubernetes.io/nodeid": `{"csi-powerstore.dellemc.com":"` + validNodeID + `"}`, + }, + }, + } + k8sutils.GetNodeByCSINodeID = func(_ context.Context, _ string, _ string, _ string) (*k8score.Node, error) { + return node, nil + } + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, "APM0012345", "scsi", validRemoteVolID, "APM0045678") - //setting externalAccess environment variable - err = csictx.Setenv(context.Background(), common.EnvExternalAccess, "") + req := &csi.ControllerUnpublishVolumeRequest{VolumeId: volumeID, NodeId: validNodeID} - Expect(err).To(BeNil()) - _ = ctrlSvc.Init() - }) + _, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) - It("should return error since HostAccessList contain external as well as Host IP too", func() { - //setting externalAccess environment variable - err := csictx.Setenv(context.Background(), common.EnvExternalAccess, "10.0.0.0/16") - Expect(err).To(BeNil()) - _ = ctrlSvc.Init() - clientMock.On("GetFsSnapshotsByVolumeID", mock.Anything, validBaseVolID).Return([]gopowerstore.FileSystem{}, nil) + gomega.Expect(err).To(gomega.HaveOccurred()) + }) - exportID := "some-export-id" + ginkgo.It("should fail due to no node presence [Block]", func() { + k8sutils.GetNodeByCSINodeID = func(_ context.Context, _ string, _ string, _ string) (*k8score.Node, error) { + return nil, fmt.Errorf("failed to list nodes") + } + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, "APM0012345", "scsi", validRemoteVolID, "APM0045678") - clientMock.On("GetFS", mock.Anything, validBaseVolID). - Return(gopowerstore.FileSystem{ - ID: validBaseVolID, - }, nil) + req := &csi.ControllerUnpublishVolumeRequest{VolumeId: volumeID, NodeId: validNodeID} - clientMock.On("GetNFSExportByFileSystemID", mock.Anything, validBaseVolID). - Return(gopowerstore.NFSExport{ - ID: exportID, - RWRootHosts: []string{"10.0.0.0/255.255.0.0", "10.225.0.0/255.255.255.255"}, - }, nil) + _, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) - req := &csi.DeleteVolumeRequest{VolumeId: validNfsVolumeID} + gomega.Expect(err).To(gomega.HaveOccurred()) + }) + ginkgo.It("should fail due to no remote system [Block]", func() { + // Create a node that matches the node selector + node := &k8score.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: validNodeID, + Labels: map[string]string{ + "topology.kubernetes.io/zone": "zone1", // matches the node selector + }, + Annotations: map[string]string{ + "csi.volume.kubernetes.io/nodeid": `{"csi-powerstore.dellemc.com":"` + validNodeID + `"}`, + }, + }, + } + k8sutils.GetNodeByCSINodeID = func(_ context.Context, _ string, _ string, _ string) (*k8score.Node, error) { + return node, nil + } + clientMock.On("GetVolume", mock.Anything, validRemoteVolID+"1"). + Return(gopowerstore.Volume{}, fmt.Errorf("failed to get volume")) + // Invalid remote system + volumeID := fmt.Sprintf("%s/%s/%s:%s/%s", validBaseVolID, "APM0012345", "scsi", validRemoteVolID+"1", "APM0456789") - _, err = ctrlSvc.DeleteVolume(context.Background(), req) + req := &csi.ControllerUnpublishVolumeRequest{VolumeId: volumeID, NodeId: validNodeID} - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring("cannot be deleted as it has associated NFS or SMB shares")) - //setting externalAccess environment variable - err = csictx.Setenv(context.Background(), common.EnvExternalAccess, "") + _, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) - Expect(err).To(BeNil()) - _ = ctrlSvc.Init() + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("cannot find remote array")) + }) }) - When("volume do not exist", func() { - It("should succeed", func() { + ginkgo.When("volume do not exist", func() { + ginkgo.It("should succeed", func() { clientMock.On("GetFS", mock.Anything, validBaseVolID). Return(gopowerstore.FileSystem{}, gopowerstore.APIError{ ErrorMsg: &api.ErrorMsg{ @@ -2631,44 +5545,44 @@ var _ = Describe("CSIControllerService", func() { req := &csi.ControllerUnpublishVolumeRequest{VolumeId: validNfsVolumeID, NodeId: validNodeID} res, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.ControllerUnpublishVolumeResponse{})) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerUnpublishVolumeResponse{})) }) }) - When("volume id is empty", func() { - It("should fail", func() { + ginkgo.When("volume id is empty", func() { + ginkgo.It("should fail", func() { req := &csi.ControllerUnpublishVolumeRequest{VolumeId: "", NodeId: validNodeID} _, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring("volume ID is required")) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("volume ID is required")) }) }) - When("volume id has wrong array id", func() { - It("should fail", func() { + ginkgo.When("volume id has wrong array id", func() { + ginkgo.It("should fail", func() { req := &csi.ControllerUnpublishVolumeRequest{VolumeId: invalidBlockVolumeID, NodeId: validNodeID} _, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring("cannot find array")) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("cannot find array")) }) }) - When("node id is empty", func() { - It("should fail", func() { + ginkgo.When("node id is empty", func() { + ginkgo.It("should fail", func() { req := &csi.ControllerUnpublishVolumeRequest{VolumeId: validBlockVolumeID, NodeId: ""} _, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring("node ID is required")) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("node ID is required")) }) }) - When("using v1.2 volumes", func() { - It("should succeed [Block]", func() { - By("using default array", func() { + ginkgo.When("using v1.2 volumes", func() { + ginkgo.It("should succeed [Block]", func() { + ginkgo.By("using default array", func() { clientMock.On("GetVolume", mock.Anything, validBaseVolID). Return(gopowerstore.Volume{}, nil) }) @@ -2689,12 +5603,12 @@ var _ = Describe("CSIControllerService", func() { req := &csi.ControllerUnpublishVolumeRequest{VolumeId: validBaseVolID, NodeId: validNodeID} res, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.ControllerUnpublishVolumeResponse{})) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerUnpublishVolumeResponse{})) }) - It("should succeed [NFS]", func() { - By("using default array", func() { + ginkgo.It("should succeed [NFS]", func() { + ginkgo.By("using default array", func() { clientMock.On("GetVolume", mock.Anything, validBaseVolID). Return(gopowerstore.Volume{}, gopowerstore.APIError{ ErrorMsg: &api.ErrorMsg{ @@ -2703,7 +5617,7 @@ var _ = Describe("CSIControllerService", func() { }) clientMock.On("GetFS", mock.Anything, validBaseVolID). - Return(gopowerstore.FileSystem{}, nil).Once() + Return(gopowerstore.FileSystem{ID: validBaseVolID}, nil) }) exportID := "some-export-id" @@ -2719,16 +5633,16 @@ var _ = Describe("CSIControllerService", func() { clientMock.On("ModifyNFSExport", mock.Anything, mock.Anything, exportID).Return(gopowerstore.CreateResponse{}, nil) - req := &csi.ControllerUnpublishVolumeRequest{VolumeId: validBaseVolID, NodeId: validNodeID} + req := &csi.ControllerUnpublishVolumeRequest{VolumeId: validNfsVolumeID, NodeId: validNodeID} res, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.ControllerUnpublishVolumeResponse{})) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerUnpublishVolumeResponse{})) }) - When("volume does not exist", func() { - It("should succeed", func() { - By("not finding volume or filesystem", func() { + ginkgo.When("volume does not exist", func() { + ginkgo.It("should succeed", func() { + ginkgo.By("not finding volume or filesystem", func() { clientMock.On("GetVolume", mock.Anything, validBaseVolID). Return(gopowerstore.Volume{}, gopowerstore.APIError{ ErrorMsg: &api.ErrorMsg{ @@ -2744,19 +5658,18 @@ var _ = Describe("CSIControllerService", func() { }).Once() }) - req := &csi.ControllerUnpublishVolumeRequest{VolumeId: validBaseVolID, NodeId: validNodeID} + req := &csi.ControllerUnpublishVolumeRequest{VolumeId: validNfsVolumeID, NodeId: validNodeID} res, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) - - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.ControllerUnpublishVolumeResponse{})) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerUnpublishVolumeResponse{})) }) }) }) - When("kube node id is not correct", func() { - When("no IP found", func() { - It("should fail [Block]", func() { + ginkgo.When("kube node id is not correct", func() { + ginkgo.When("no IP found", func() { + ginkgo.It("should fail [Block]", func() { nodeID := "not-valid-id" clientMock.On("GetHostByName", mock.Anything, nodeID). Return(gopowerstore.Host{}, gopowerstore.APIError{ @@ -2765,14 +5678,17 @@ var _ = Describe("CSIControllerService", func() { }, }).Once() + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{}, nil) + req := &csi.ControllerUnpublishVolumeRequest{VolumeId: validBlockVolumeID, NodeId: nodeID} _, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring("can't find IP in nodeID")) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("can't find IP in nodeID")) }) - It("should fail [NFS]", func() { + ginkgo.It("should fail [NFS]", func() { nodeID := "not-valid-id" clientMock.On("GetFS", mock.Anything, validBaseVolID). Return(gopowerstore.FileSystem{ID: validBaseVolID}, nil) @@ -2780,14 +5696,15 @@ var _ = Describe("CSIControllerService", func() { req := &csi.ControllerUnpublishVolumeRequest{VolumeId: validNfsVolumeID, NodeId: nodeID} _, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring("can't find IP in nodeID")) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("can't find IP in nodeID")) }) - }) - When("host does not exist", func() { - It("should fail", func() { + ginkgo.When("host does not exist", func() { + ginkgo.It("should fail", func() { + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{}, nil) clientMock.On("GetHostByName", mock.Anything, validNodeID). Return(gopowerstore.Host{}, gopowerstore.APIError{ ErrorMsg: &api.ErrorMsg{ @@ -2805,28 +5722,31 @@ var _ = Describe("CSIControllerService", func() { req := &csi.ControllerUnpublishVolumeRequest{VolumeId: validBlockVolumeID, NodeId: validNodeID} _, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring("host with k8s node ID '" + validNodeID + "' not found")) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("host with k8s node ID '" + validNodeID + "' not found")) }) }) - When("fail to check host", func() { - It("should fail", func() { + ginkgo.When("fail to check host", func() { + ginkgo.It("should fail", func() { e := errors.New("some-api-error") + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{}, nil) + clientMock.On("GetHostByName", mock.Anything, validNodeID). Return(gopowerstore.Host{}, e).Once() req := &csi.ControllerUnpublishVolumeRequest{VolumeId: validBlockVolumeID, NodeId: validNodeID} _, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring("failure checking host '" + validNodeID + "' status for volume unpublishing")) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("failure checking host '" + validNodeID + "' status for volume unpublishing")) }) }) }) - When("can not check nfs export status", func() { - It("should fail", func() { + ginkgo.When("can not check nfs export status", func() { + ginkgo.It("should fail", func() { e := errors.New("some-api-error") clientMock.On("GetFS", mock.Anything, validBaseVolID). Return(gopowerstore.FileSystem{ @@ -2838,13 +5758,13 @@ var _ = Describe("CSIControllerService", func() { req := &csi.ControllerUnpublishVolumeRequest{VolumeId: validNfsVolumeID, NodeId: validNodeID} _, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring("failure checking nfs export status for volume unpublishing")) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("failure checking nfs export status for volume unpublishing")) }) }) - When("failed to remove hosts", func() { - It("should fail", func() { + ginkgo.When("failed to remove hosts", func() { + ginkgo.It("should fail", func() { exportID := "some-export-id" e := errors.New("some-api-error") @@ -2861,145 +5781,194 @@ var _ = Describe("CSIControllerService", func() { req := &csi.ControllerUnpublishVolumeRequest{VolumeId: validNfsVolumeID, NodeId: validNodeID} _, err := ctrlSvc.ControllerUnpublishVolume(context.Background(), req) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring("failure when removing new host to nfs export")) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("failure when removing new host to nfs export")) }) }) }) - Describe("calling ListVolumes()", func() { + ginkgo.Describe("calling ListVolumes()", func() { + clientA := &gopowerstoremock.Client{} + clientB := &gopowerstoremock.Client{} + + injectArrays := func() { + arrMap := map[string]*array.PowerStoreArray{ + "globalvolid1": {Client: clientA, GlobalID: "globalvolid1"}, + "globalvolid2": {Client: clientB, GlobalID: "globalvolid2"}, + } + ctrlSvc.SetArrays(arrMap) + } + mockCalls := func() { - clientMock.On("GetVolumes", mock.Anything). + // --- Array globalvolid1 (clientA) --- + clientA.On("GetHostVolumeMappings", mock.Anything). + Return([]gopowerstore.HostVolumeMapping{}, nil).Once() + + clientA.On("GetVolumes", mock.Anything). Return([]gopowerstore.Volume{ - { - ID: "arr1-id1", - Name: "arr1-vol1", - }, - { - ID: "arr1-id2", - Name: "arr1-vol2", - }, + {ID: "arr1-id1", Name: "arr1-vol1"}, + {ID: "arr1-id2", Name: "arr1-vol2"}, + }, nil).Once() + + clientA.On("ListFS", mock.Anything). + Return([]gopowerstore.FileSystem{ + {ID: "arr3-id1", Name: "arr3-fs1"}, + {ID: "arr3-id2", Name: "arr3-fs2"}, }, nil).Once() - clientMock.On("GetVolumes", mock.Anything). + + // --- Array globalvolid2 (clientB) --- + clientB.On("GetHostVolumeMappings", mock.Anything). + Return([]gopowerstore.HostVolumeMapping{}, nil).Once() + + clientB.On("GetVolumes", mock.Anything). Return([]gopowerstore.Volume{ - { - ID: "arr2-id1", - Name: "arr2-vol1", - }, + {ID: "arr2-id1", Name: "arr2-vol1"}, + }, nil).Once() + + clientB.On("ListFS", mock.Anything). + Return([]gopowerstore.FileSystem{ + {ID: "arr4-id1", Name: "arr4-fs1"}, }, nil).Once() } - When("there is no parameters", func() { - It("should return all volumes from both arrays", func() { + // helper to assert the response contains exactly the expected set of volume IDs (order-independent) + expectContainsVolumeIDs := func(res *csi.ListVolumesResponse, expected []string) { + gomega.Expect(res).ToNot(gomega.BeNil()) + got := make(map[string]struct{}, len(res.Entries)) + for _, e := range res.Entries { + if e != nil && e.Volume != nil { + got[e.Volume.VolumeId] = struct{}{} + } + } + for _, id := range expected { + _, ok := got[id] + gomega.Expect(ok).To(gomega.BeTrue(), "missing expected volumeID: %s; got=%v", id, got) + } + // exact-match check + gomega.Expect(len(got)).To(gomega.Equal(len(expected))) + } + + ginkgo.When("there is no parameters", func() { + ginkgo.It("should return all volumes from both arrays (order-independent)", func() { + injectArrays() mockCalls() req := &csi.ListVolumesRequest{} res, err := ctrlSvc.ListVolumes(context.Background(), req) - - Expect(res).To(Equal(&csi.ListVolumesResponse{ - Entries: []*csi.ListVolumesResponse_Entry{ - { - Volume: &csi.Volume{ - VolumeId: "arr1-id1", - }, - }, - { - Volume: &csi.Volume{ - VolumeId: "arr1-id2", - }, - }, - { - Volume: &csi.Volume{ - VolumeId: "arr2-id1", - }, - }, - }, - })) - Expect(err).To(BeNil()) + gomega.Expect(err).To(gomega.BeNil()) + + expected := []string{ + "arr1-id1/globalvolid1/scsi", + "arr1-id2/globalvolid1/scsi", + "arr2-id1/globalvolid2/scsi", + "arr3-id1/globalvolid1/nfs", + "arr3-id2/globalvolid1/nfs", + "arr4-id1/globalvolid2/nfs", + } + expectContainsVolumeIDs(res, expected) }) }) - When("passing max entries", func() { - It("should return 'n' entries and next token", func() { + ginkgo.When("passing max entries", func() { + ginkgo.It("should return 'n' entries and a next token (when applicable)", func() { + injectArrays() mockCalls() - req := &csi.ListVolumesRequest{ - MaxEntries: 1, - } + req := &csi.ListVolumesRequest{MaxEntries: 1} res, err := ctrlSvc.ListVolumes(context.Background(), req) - Expect(res).To(Equal(&csi.ListVolumesResponse{ - Entries: []*csi.ListVolumesResponse_Entry{ - { - Volume: &csi.Volume{ - VolumeId: "arr1-id1", - }, - }, - }, - NextToken: "1", - })) - Expect(err).To(BeNil()) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).ToNot(gomega.BeNil()) + gomega.Expect(len(res.Entries)).To(gomega.Equal(1)) + gomega.Expect(res.NextToken).To(gomega.Satisfy(func(token string) bool { return token != "" })) }) }) - When("using next token", func() { - It("should return volumes starting from token", func() { + ginkgo.When("using starting token", func() { + ginkgo.It("should return volumes starting from token (len check)", func() { + injectArrays() mockCalls() - req := &csi.ListVolumesRequest{ - MaxEntries: 1, - StartingToken: "1", - } + req := &csi.ListVolumesRequest{MaxEntries: 1, StartingToken: "1"} res, err := ctrlSvc.ListVolumes(context.Background(), req) - Expect(res).To(Equal(&csi.ListVolumesResponse{ - Entries: []*csi.ListVolumesResponse_Entry{ - { - Volume: &csi.Volume{ - VolumeId: "arr1-id2", - }, - }, - }, - NextToken: "2", - })) - Expect(err).To(BeNil()) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).ToNot(gomega.BeNil()) + gomega.Expect(len(res.Entries)).To(gomega.Equal(1)) + + expectedAll := []string{ + "arr1-id1/globalvolid1/scsi", + "arr1-id2/globalvolid1/scsi", + "arr2-id1/globalvolid2/scsi", + "arr3-id1/globalvolid1/nfs", + "arr3-id2/globalvolid1/nfs", + "arr4-id1/globalvolid2/nfs", + } + gotID := res.Entries[0].Volume.VolumeId + found := false + for _, e := range expectedAll { + if e == gotID { + found = true + break + } + } + gomega.Expect(found).To(gomega.BeTrue(), "returned unexpected volume id: %s", gotID) }) }) - When("using wrong token", func() { - It("should fail [not parsable]", func() { - token := "as!512$25%!_" - req := &csi.ListVolumesRequest{ - MaxEntries: 1, - StartingToken: token, - } + ginkgo.When("using wrong token", func() { + ginkgo.It("should fail [not parsable]", func() { + injectArrays() + token := "as!512$25%!_" // #nosec G101 + req := &csi.ListVolumesRequest{MaxEntries: 1, StartingToken: token} res, err := ctrlSvc.ListVolumes(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring("unable to parse StartingToken: %v into uint32", token)) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + // check parse error text exists + gomega.Expect(err.Error()).To(gomega.ContainSubstring("unable to parse StartingToken")) }) - It("shoud fail [too high]", func() { - tokenInt := 200 - token := "200" - + ginkgo.It("should fail [too high]", func() { + injectArrays() mockCalls() - req := &csi.ListVolumesRequest{ - MaxEntries: 1, - StartingToken: token, - } + req := &csi.ListVolumesRequest{MaxEntries: 1, StartingToken: "200"} res, err := ctrlSvc.ListVolumes(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring("startingToken=%d > len(volumes)=%d", tokenInt, 3)) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("startingToken=")) + }) + }) + + ginkgo.When("get volumes return error", func() { + ginkgo.It("should fail when backing client returns an error", func() { + injectArrays() + // simulate failures for both arrays' calls + clientA.On("GetVolumes", mock.Anything). + Return([]gopowerstore.Volume{}, gopowerstore.NewNotFoundError()).Once() + clientA.On("ListFS", mock.Anything). + Return([]gopowerstore.FileSystem{}, gopowerstore.NewNotFoundError()).Once() + clientA.On("GetHostVolumeMappings", mock.Anything). + Return([]gopowerstore.HostVolumeMapping{}, gopowerstore.NewNotFoundError()).Once() + + clientB.On("GetVolumes", mock.Anything). + Return([]gopowerstore.Volume{}, gopowerstore.NewNotFoundError()).Once() + clientB.On("ListFS", mock.Anything). + Return([]gopowerstore.FileSystem{}, gopowerstore.NewNotFoundError()).Once() + clientB.On("GetHostVolumeMappings", mock.Anything). + Return([]gopowerstore.HostVolumeMapping{}, gopowerstore.NewNotFoundError()).Once() + + req := &csi.ListVolumesRequest{} + res, err := ctrlSvc.ListVolumes(context.Background(), req) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) }) }) }) - Describe("calling ListSnapshots()", func() { + ginkgo.Describe("calling ListSnapshots()", func() { mockCalls := func() { clientMock.On("GetSnapshots", mock.Anything). Return([]gopowerstore.Volume{ @@ -3051,27 +6020,27 @@ var _ = Describe("CSIControllerService", func() { }).Once() } - When("there is no parameters", func() { - It("should return all volumes from both arrays", func() { + ginkgo.When("there is no parameters", func() { + ginkgo.It("should return all volumes from both arrays", func() { mockCalls() req := &csi.ListSnapshotsRequest{} res, err := ctrlSvc.ListSnapshots(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).ToNot(BeNil()) - Expect(res.Entries).ToNot(BeNil()) - Expect(len(res.Entries)).To(Equal(5)) - Expect(res.Entries[0].Snapshot.SnapshotId).To(Equal("arr1-id1")) - Expect(res.Entries[1].Snapshot.SnapshotId).To(Equal("arr1-id2")) - Expect(res.Entries[2].Snapshot.SnapshotId).To(Equal("arr1-id1-fs")) - Expect(res.Entries[3].Snapshot.SnapshotId).To(Equal("arr2-id1")) - Expect(res.Entries[4].Snapshot.SnapshotId).To(Equal("arr2-id1-fs")) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).ToNot(gomega.BeNil()) + gomega.Expect(res.Entries).ToNot(gomega.BeNil()) + gomega.Expect(len(res.Entries)).To(gomega.Equal(5)) + gomega.Expect(res.Entries[0].Snapshot.SnapshotId).To(gomega.Equal("arr1-id1")) + gomega.Expect(res.Entries[1].Snapshot.SnapshotId).To(gomega.Equal("arr1-id2")) + gomega.Expect(res.Entries[2].Snapshot.SnapshotId).To(gomega.Equal("arr1-id1-fs")) + gomega.Expect(res.Entries[3].Snapshot.SnapshotId).To(gomega.Equal("arr2-id1")) + gomega.Expect(res.Entries[4].Snapshot.SnapshotId).To(gomega.Equal("arr2-id1-fs")) }) }) - When("passing max entries", func() { - It("should return 'n' entries and next token", func() { + ginkgo.When("passing max entries", func() { + ginkgo.It("should return 'n' entries and next token", func() { mockCalls() req := &csi.ListSnapshotsRequest{ @@ -3079,17 +6048,17 @@ var _ = Describe("CSIControllerService", func() { } res, err := ctrlSvc.ListSnapshots(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).ToNot(BeNil()) - Expect(res.Entries).ToNot(BeNil()) - Expect(len(res.Entries)).To(Equal(1)) - Expect(res.Entries[0].Snapshot.SnapshotId).To(Equal("arr1-id1")) - Expect(res.NextToken).To(Equal("1")) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).ToNot(gomega.BeNil()) + gomega.Expect(res.Entries).ToNot(gomega.BeNil()) + gomega.Expect(len(res.Entries)).To(gomega.Equal(1)) + gomega.Expect(res.Entries[0].Snapshot.SnapshotId).To(gomega.Equal("arr1-id1")) + gomega.Expect(res.NextToken).To(gomega.Equal("1")) }) }) - When("using next token", func() { - It("should return volumes starting from token", func() { + ginkgo.When("using next token", func() { + ginkgo.It("should return volumes starting from token", func() { mockCalls() req := &csi.ListSnapshotsRequest{ @@ -3098,30 +6067,30 @@ var _ = Describe("CSIControllerService", func() { } res, err := ctrlSvc.ListSnapshots(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).ToNot(BeNil()) - Expect(res.Entries).ToNot(BeNil()) - Expect(len(res.Entries)).To(Equal(1)) - Expect(res.Entries[0].Snapshot.SnapshotId).To(Equal("arr1-id2")) - Expect(res.NextToken).To(Equal("2")) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).ToNot(gomega.BeNil()) + gomega.Expect(res.Entries).ToNot(gomega.BeNil()) + gomega.Expect(len(res.Entries)).To(gomega.Equal(1)) + gomega.Expect(res.Entries[0].Snapshot.SnapshotId).To(gomega.Equal("arr1-id2")) + gomega.Expect(res.NextToken).To(gomega.Equal("2")) }) }) - When("using wrong token", func() { - It("should fail [not parsable]", func() { - token := "as!512$25%!_" + ginkgo.When("using wrong token", func() { + ginkgo.It("should fail [not parsable]", func() { + token := "as!512$25%!_" // #nosec G101 req := &csi.ListSnapshotsRequest{ MaxEntries: 1, StartingToken: token, } res, err := ctrlSvc.ListSnapshots(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring("unable to parse StartingToken: %v into uint32", token)) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("unable to parse StartingToken: %v into uint32", token)) }) - It("shoud fail [too high]", func() { + ginkgo.It("shoud fail [too high]", func() { tokenInt := 200 token := "200" @@ -3133,43 +6102,107 @@ var _ = Describe("CSIControllerService", func() { } res, err := ctrlSvc.ListSnapshots(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring("startingToken=%d > len(generalSnapshots)=%d", tokenInt, 5)) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("startingToken=%d > len(generalSnapshots)=%d", tokenInt, 5)) }) }) - When("passing snapshot id", func() { - It("should return existing snapshot", func() { + ginkgo.When("passing snapshot id", func() { + ginkgo.It("should return existing snapshot", func() { clientMock.On("GetSnapshot", mock.Anything, validBaseVolID). Return(gopowerstore.Volume{ID: validBaseVolID}, nil) req := &csi.ListSnapshotsRequest{ SnapshotId: validBlockVolumeID, } res, err := ctrlSvc.ListSnapshots(context.Background(), req) - Expect(res).ToNot(BeNil()) - Expect(err).To(BeNil()) - Expect(len(res.Entries)).To(Equal(1)) - Expect(res.Entries[0].Snapshot.SnapshotId).To(Equal(validBlockVolumeID)) + gomega.Expect(res).ToNot(gomega.BeNil()) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(len(res.Entries)).To(gomega.Equal(1)) + gomega.Expect(res.Entries[0].Snapshot.SnapshotId).To(gomega.Equal(validBlockVolumeID)) }) - It("should return existing snapshot [NFS]", func() { + ginkgo.It("should return existing snapshot [NFS]", func() { clientMock.On("GetFsSnapshot", mock.Anything, validBaseVolID). Return(gopowerstore.FileSystem{ID: validBaseVolID}, nil) req := &csi.ListSnapshotsRequest{ SnapshotId: validNfsVolumeID, } res, err := ctrlSvc.ListSnapshots(context.Background(), req) - Expect(res).ToNot(BeNil()) - Expect(err).To(BeNil()) - Expect(len(res.Entries)).To(Equal(1)) - Expect(res.Entries[0].Snapshot.SnapshotId).To(Equal(validNfsVolumeID)) + gomega.Expect(res).ToNot(gomega.BeNil()) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(len(res.Entries)).To(gomega.Equal(1)) + gomega.Expect(res.Entries[0].Snapshot.SnapshotId).To(gomega.Equal(validNfsVolumeID)) + }) + + ginkgo.It("should return empty response", func() { + clientMock.On("GetSnapshot", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{}, gopowerstore.NewNotFoundError()) + + req := &csi.ListSnapshotsRequest{ + SnapshotId: validBlockVolumeID, + } + res, err := ctrlSvc.ListSnapshots(context.Background(), req) + + gomega.Expect(res).ToNot(gomega.BeNil()) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(len(res.Entries)).To(gomega.Equal(0)) + }) + + ginkgo.It("should return error when GetFsSnapshot call fails", func() { + clientMock.On("GetSnapshot", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{}, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{ + StatusCode: http.StatusBadRequest, + }, + }) + + req := &csi.ListSnapshotsRequest{ + SnapshotId: validBlockVolumeID, + } + res, err := ctrlSvc.ListSnapshots(context.Background(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("unable to get block snapshot")) + }) + + ginkgo.It("should return empty response [NFS]", func() { + clientMock.On("GetFsSnapshot", mock.Anything, validBaseVolID). + Return(gopowerstore.FileSystem{}, gopowerstore.NewNotFoundError()) + + req := &csi.ListSnapshotsRequest{ + SnapshotId: validNfsVolumeID, + } + res, err := ctrlSvc.ListSnapshots(context.Background(), req) + + gomega.Expect(res).ToNot(gomega.BeNil()) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(len(res.Entries)).To(gomega.Equal(0)) + }) + + ginkgo.It("should return error when GetFsSnapshot call fails [NFS]", func() { + clientMock.On("GetFsSnapshot", mock.Anything, validBaseVolID). + Return(gopowerstore.FileSystem{}, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{ + StatusCode: http.StatusBadRequest, + }, + }) + + req := &csi.ListSnapshotsRequest{ + SnapshotId: validNfsVolumeID, + } + res, err := ctrlSvc.ListSnapshots(context.Background(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("unable to get filesystem snapshot")) }) - It("should fail [incorrect id]", func() { + ginkgo.It("should fail [incorrect id]", func() { randomID := "something-random" - By("checking with default array", func() { + ginkgo.By("checking with default array", func() { mockCantParseVolumeID(randomID) }) @@ -3177,44 +6210,53 @@ var _ = Describe("CSIControllerService", func() { SnapshotId: randomID, } res, err := ctrlSvc.ListSnapshots(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.ListSnapshotsResponse{})) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ListSnapshotsResponse{})) + }) + + ginkgo.It("should fail [incorrect array id]", func() { + req := &csi.ListSnapshotsRequest{SnapshotId: invalidBlockVolumeID} + res, err := ctrlSvc.ListSnapshots(context.Background(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("unable to get array with arrayID")) }) }) - When("passing source volume id", func() { - It("should return all snapshots of that volume", func() { + ginkgo.When("passing source volume id", func() { + ginkgo.It("should return all snapshots of that volume", func() { clientMock.On("GetSnapshotsByVolumeID", mock.Anything, validBaseVolID). Return([]gopowerstore.Volume{{ID: "snap-id-1"}, {ID: "snap-id-2"}}, nil) req := &csi.ListSnapshotsRequest{ SourceVolumeId: validBlockVolumeID, } res, err := ctrlSvc.ListSnapshots(context.Background(), req) - Expect(res).ToNot(BeNil()) - Expect(err).To(BeNil()) - Expect(len(res.Entries)).To(Equal(2)) - Expect(res.Entries[0].Snapshot.SnapshotId).To(Equal("snap-id-1")) - Expect(res.Entries[1].Snapshot.SnapshotId).To(Equal("snap-id-2")) + gomega.Expect(res).ToNot(gomega.BeNil()) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(len(res.Entries)).To(gomega.Equal(2)) + gomega.Expect(res.Entries[0].Snapshot.SnapshotId).To(gomega.Equal("snap-id-1")) + gomega.Expect(res.Entries[1].Snapshot.SnapshotId).To(gomega.Equal("snap-id-2")) }) - It("should return all snapshots of the filesystem", func() { + ginkgo.It("should return all snapshots of the filesystem", func() { clientMock.On("GetFsSnapshotsByVolumeID", mock.Anything, validBaseVolID). Return([]gopowerstore.FileSystem{{ID: "snap-id-1"}, {ID: "snap-id-2"}}, nil) req := &csi.ListSnapshotsRequest{ SourceVolumeId: validNfsVolumeID, } res, err := ctrlSvc.ListSnapshots(context.Background(), req) - Expect(res).ToNot(BeNil()) - Expect(err).To(BeNil()) - Expect(len(res.Entries)).To(Equal(2)) - Expect(res.Entries[0].Snapshot.SnapshotId).To(Equal("snap-id-1")) - Expect(res.Entries[1].Snapshot.SnapshotId).To(Equal("snap-id-2")) + gomega.Expect(res).ToNot(gomega.BeNil()) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(len(res.Entries)).To(gomega.Equal(2)) + gomega.Expect(res.Entries[0].Snapshot.SnapshotId).To(gomega.Equal("snap-id-1")) + gomega.Expect(res.Entries[1].Snapshot.SnapshotId).To(gomega.Equal("snap-id-2")) }) - It("should fail [incorrect id]", func() { + ginkgo.It("should fail [incorrect id]", func() { randomID := "something-random" - By("checking with default array", func() { + ginkgo.By("checking with default array", func() { mockCantParseVolumeID(randomID) }) @@ -3222,17 +6264,89 @@ var _ = Describe("CSIControllerService", func() { SourceVolumeId: randomID, } res, err := ctrlSvc.ListSnapshots(context.Background(), req) - Expect(res).To(Equal(&csi.ListSnapshotsResponse{})) - Expect(err).To(BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ListSnapshotsResponse{})) + gomega.Expect(err).To(gomega.BeNil()) + }) + + ginkgo.It("should fail [incorrect array id]", func() { + req := &csi.ListSnapshotsRequest{SourceVolumeId: invalidBlockVolumeID} + res, err := ctrlSvc.ListSnapshots(context.Background(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("unable to get array with arrayID")) + }) + + ginkgo.It("should return error when GetFsSnapshotsByVolumeID call fails", func() { + clientMock.On("GetFsSnapshotsByVolumeID", mock.Anything, validBaseVolID). + Return([]gopowerstore.FileSystem{}, gopowerstore.NewNotFoundError()) + + req := &csi.ListSnapshotsRequest{ + SourceVolumeId: validNfsVolumeID, + } + res, err := ctrlSvc.ListSnapshots(context.Background(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("unable to list filesystem snapshots")) + }) + + ginkgo.It("should return error when GetSnapshotsByVolumeID call fails", func() { + clientMock.On("GetSnapshotsByVolumeID", mock.Anything, validBaseVolID). + Return([]gopowerstore.Volume{}, gopowerstore.NewNotFoundError()) + + req := &csi.ListSnapshotsRequest{ + SourceVolumeId: validBlockVolumeID, + } + res, err := ctrlSvc.ListSnapshots(context.Background(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("unable to list block snapshots")) + }) + }) + + ginkgo.When("get snapshots call fails", func() { + ginkgo.It("should fail [block]", func() { + clientMock.On("GetSnapshots", mock.Anything). + Return([]gopowerstore.Volume{}, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{ + StatusCode: http.StatusNotFound, + }, + }) + + req := &csi.ListSnapshotsRequest{} + res, err := ctrlSvc.ListSnapshots(context.Background(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("unable to list block snapshots")) + }) + + ginkgo.It("should fail [NFS]", func() { + clientMock.On("GetSnapshots", mock.Anything).Return([]gopowerstore.Volume{}, nil) + clientMock.On("GetFsSnapshots", mock.Anything). + Return([]gopowerstore.FileSystem{}, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{ + StatusCode: http.StatusNotFound, + }, + }) + + req := &csi.ListSnapshotsRequest{} + res, err := ctrlSvc.ListSnapshots(context.Background(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("unable to list filesystem snapshots")) }) }) }) - Describe("calling GetCapacity()", func() { - When("everything is ok and arrayip is provided", func() { - It("should succeed", func() { + ginkgo.Describe("calling GetCapacity()", func() { + ginkgo.When("everything is ok and arrayip is provided", func() { + ginkgo.It("should succeed", func() { clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) - clientMock.On("GetCustomHTTPHeaders").Return(make(http.Header)) + clientMock.On("GetCustomHTTPHeaders").Return(api.NewSafeHeader().GetHeader()) clientMock.On("GetCapacity", mock.Anything).Return(int64(123123123), nil) clientMock.On("GetMaxVolumeSize", mock.Anything).Return(int64(-1), nil) req := &csi.GetCapacityRequest{ @@ -3241,30 +6355,30 @@ var _ = Describe("CSIControllerService", func() { }, } res, err := ctrlSvc.GetCapacity(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res.AvailableCapacity).To(Equal(int64(123123123))) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res.AvailableCapacity).To(gomega.Equal(int64(123123123))) }) }) - When("everything is ok and array ip is not provided", func() { - It("should succeed", func() { + ginkgo.When("everything is ok and array ip is not provided", func() { + ginkgo.It("should succeed", func() { clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) - clientMock.On("GetCustomHTTPHeaders").Return(make(http.Header)) + clientMock.On("GetCustomHTTPHeaders").Return(api.NewSafeHeader().GetHeader()) clientMock.On("GetCapacity", mock.Anything).Return(int64(123123123), nil) clientMock.On("GetMaxVolumeSize", mock.Anything).Return(int64(-1), nil) req := &csi.GetCapacityRequest{ Parameters: map[string]string{}, } res, err := ctrlSvc.GetCapacity(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res.AvailableCapacity).To(Equal(int64(123123123))) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res.AvailableCapacity).To(gomega.Equal(int64(123123123))) }) }) - When("wrong arrayIP in params", func() { - It("should fail with predefined errmsg", func() { + ginkgo.When("wrong arrayIP in params", func() { + ginkgo.It("should fail with predefined errmsg", func() { clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) - clientMock.On("GetCustomHTTPHeaders").Return(make(http.Header)) + clientMock.On("GetCustomHTTPHeaders").Return(api.NewSafeHeader().GetHeader()) clientMock.On("GetCapacity", mock.Anything).Return(int64(123123123), nil) clientMock.On("GetMaxVolumeSize", mock.Anything).Return(int64(-1), nil) req := &csi.GetCapacityRequest{ @@ -3273,16 +6387,16 @@ var _ = Describe("CSIControllerService", func() { }, } res, err := ctrlSvc.GetCapacity(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring("can't find array with provided id 10.10.10.10")) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("can't find array with provided id 10.10.10.10")) }) }) - When("everything is correct, but API failed", func() { - It("should fail with predefined errmsg", func() { + ginkgo.When("everything is correct, but API failed", func() { + ginkgo.It("should fail with predefined errmsg", func() { clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) - clientMock.On("GetCustomHTTPHeaders").Return(make(http.Header)) + clientMock.On("GetCustomHTTPHeaders").Return(api.NewSafeHeader().GetHeader()) clientMock.On("GetCapacity", mock.Anything).Return(int64(123123123), errors.New("APIErrorUnexpected")) clientMock.On("GetMaxVolumeSize", mock.Anything).Return(int64(-1), nil) req := &csi.GetCapacityRequest{ @@ -3291,16 +6405,16 @@ var _ = Describe("CSIControllerService", func() { }, } res, err := ctrlSvc.GetCapacity(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring("APIErrorUnexpected")) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("APIErrorUnexpected")) }) }) - When("everything is correct, but GetMaxVolumeSize API failed", func() { - It("MaximumVolumeSize should not be set in the response", func() { + ginkgo.When("everything is correct, but GetMaxVolumeSize API failed", func() { + ginkgo.It("MaximumVolumeSize should not be set in the response", func() { clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) - clientMock.On("GetCustomHTTPHeaders").Return(make(http.Header)) + clientMock.On("GetCustomHTTPHeaders").Return(api.NewSafeHeader().GetHeader()) clientMock.On("GetCapacity", mock.Anything).Return(int64(123123123), nil) req := &csi.GetCapacityRequest{ Parameters: map[string]string{}, @@ -3308,15 +6422,15 @@ var _ = Describe("CSIControllerService", func() { clientMock.On("GetMaxVolumeSize", mock.Anything).Return(int64(-1), errors.New("APIErrorUnexpected")) res, err := ctrlSvc.GetCapacity(context.Background(), req) - Expect(res).ToNot(BeNil()) - Expect(err).To(BeNil()) + gomega.Expect(res).ToNot(gomega.BeNil()) + gomega.Expect(err).To(gomega.BeNil()) }) }) - When("negative MaximumVolumeSize", func() { - It("MaximumVolumeSize should not be set in the response", func() { + ginkgo.When("negative MaximumVolumeSize", func() { + ginkgo.It("MaximumVolumeSize should not be set in the response", func() { clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) - clientMock.On("GetCustomHTTPHeaders").Return(make(http.Header)) + clientMock.On("GetCustomHTTPHeaders").Return(api.NewSafeHeader().GetHeader()) clientMock.On("GetCapacity", mock.Anything).Return(int64(123123123), nil) req := &csi.GetCapacityRequest{ Parameters: map[string]string{}, @@ -3324,15 +6438,15 @@ var _ = Describe("CSIControllerService", func() { clientMock.On("GetMaxVolumeSize", mock.Anything).Return(int64(-1), nil) res, err := ctrlSvc.GetCapacity(context.Background(), req) - Expect(res.MaximumVolumeSize).To(BeNil()) - Expect(err).To(BeNil()) + gomega.Expect(res.MaximumVolumeSize).To(gomega.BeNil()) + gomega.Expect(err).To(gomega.BeNil()) }) }) - When("non negative MaximumVolumeSize", func() { - It("MaximumVolumeSize should be set in the response", func() { + ginkgo.When("non negative MaximumVolumeSize", func() { + ginkgo.It("MaximumVolumeSize should be set in the response", func() { clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) - clientMock.On("GetCustomHTTPHeaders").Return(make(http.Header)) + clientMock.On("GetCustomHTTPHeaders").Return(api.NewSafeHeader().GetHeader()) clientMock.On("GetCapacity", mock.Anything).Return(int64(123123123), nil) req := &csi.GetCapacityRequest{ Parameters: map[string]string{}, @@ -3340,18 +6454,17 @@ var _ = Describe("CSIControllerService", func() { clientMock.On("GetMaxVolumeSize", mock.Anything).Return(int64(100000), nil) res, err := ctrlSvc.GetCapacity(context.Background(), req) - Expect(res.MaximumVolumeSize).ToNot(BeNil()) - Expect(err).To(BeNil()) + gomega.Expect(res.MaximumVolumeSize).ToNot(gomega.BeNil()) + gomega.Expect(err).To(gomega.BeNil()) }) }) - }) - Describe("calling ValidateVolumeCapabilities()", func() { - BeforeEach(func() { clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{}, nil) }) + ginkgo.Describe("calling ValidateVolumeCapabilities()", func() { + ginkgo.BeforeEach(func() { clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{}, nil) }) - When("everything is correct. Mode = SNW,block", func() { - It("should succeed", func() { + ginkgo.When("everything is correct. Mode = SNW,block", func() { + ginkgo.It("should succeed", func() { block := new(csi.VolumeCapability_BlockVolume) accessType := new(csi.VolumeCapability_Block) accessType.Block = block @@ -3365,13 +6478,13 @@ var _ = Describe("CSIControllerService", func() { }, }, }) - Expect(err).To(BeNil()) - Expect(res.Confirmed).NotTo(BeNil()) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res.Confirmed).NotTo(gomega.BeNil()) }) }) - When("everything is correct. Mode = SNRO,block", func() { - It("should succeed", func() { + ginkgo.When("everything is correct. Mode = SNRO,block", func() { + ginkgo.It("should succeed", func() { block := new(csi.VolumeCapability_BlockVolume) accessType := new(csi.VolumeCapability_Block) accessType.Block = block @@ -3385,13 +6498,13 @@ var _ = Describe("CSIControllerService", func() { }, }, }) - Expect(err).To(BeNil()) - Expect(res.Confirmed).NotTo(BeNil()) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res.Confirmed).NotTo(gomega.BeNil()) }) }) - When("everything is correct. Mode = MNRO,block", func() { - It("should succeed", func() { + ginkgo.When("everything is correct. Mode = MNRO,block", func() { + ginkgo.It("should succeed", func() { block := new(csi.VolumeCapability_BlockVolume) accessType := new(csi.VolumeCapability_Block) accessType.Block = block @@ -3405,13 +6518,13 @@ var _ = Describe("CSIControllerService", func() { }, }, }) - Expect(err).To(BeNil()) - Expect(res.Confirmed).NotTo(BeNil()) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res.Confirmed).NotTo(gomega.BeNil()) }) }) - When("everything is correct. Mode = MNSW,block", func() { - It("should succeed", func() { + ginkgo.When("everything is correct. Mode = MNSW,block", func() { + ginkgo.It("should succeed", func() { block := new(csi.VolumeCapability_BlockVolume) accessType := new(csi.VolumeCapability_Block) accessType.Block = block @@ -3425,13 +6538,13 @@ var _ = Describe("CSIControllerService", func() { }, }, }) - Expect(err).To(BeNil()) - Expect(res.Confirmed).NotTo(BeNil()) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res.Confirmed).NotTo(gomega.BeNil()) }) }) - When("everything is correct. Mode = MNMW,block", func() { - It("should fail", func() { + ginkgo.When("everything is correct. Mode = MNMW,block", func() { + ginkgo.It("should fail", func() { block := new(csi.VolumeCapability_BlockVolume) accessType := new(csi.VolumeCapability_Block) accessType.Block = block @@ -3445,13 +6558,13 @@ var _ = Describe("CSIControllerService", func() { }, }, }) - Expect(err).To(BeNil()) - Expect(res.Confirmed).NotTo(BeNil()) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res.Confirmed).NotTo(gomega.BeNil()) }) }) - When("wrong pair of AM and AT. Mode = MNMW,mount", func() { - It("should fail", func() { + ginkgo.When("wrong pair of AM and AT. Mode = MNMW,mount", func() { + ginkgo.It("should fail", func() { mount := new(csi.VolumeCapability_MountVolume) accessType := new(csi.VolumeCapability_Mount) accessType.Mount = mount @@ -3465,14 +6578,14 @@ var _ = Describe("CSIControllerService", func() { }, }, }) - Expect(err).ToNot(BeNil()) - Expect(res.Confirmed).To(BeNil()) - Expect(err.Error()).To(ContainSubstring("multi-node with writer(s) only supported for block access type")) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(res.Confirmed).To(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("multi-node with writer(s) only supported for block access type")) }) }) - When("wrong AT is given", func() { - It("should fail", func() { + ginkgo.When("wrong AT is given", func() { + ginkgo.It("should fail", func() { accessType := new(csi.VolumeCapability_Mount) accessType.Mount = nil res, err := ctrlSvc.ValidateVolumeCapabilities(context.Background(), &csi.ValidateVolumeCapabilitiesRequest{ @@ -3485,14 +6598,14 @@ var _ = Describe("CSIControllerService", func() { }, }, }) - Expect(err).ToNot(BeNil()) - Expect(res.Confirmed).To(BeNil()) - Expect(err.Error()).To(ContainSubstring("unknown access type is not Block or Mount")) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(res.Confirmed).To(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("unknown access type is not Block or Mount")) }) }) - When("AM is nil", func() { - It("should fail", func() { + ginkgo.When("AM is nil", func() { + ginkgo.It("should fail", func() { mount := new(csi.VolumeCapability_MountVolume) accessType := new(csi.VolumeCapability_Mount) accessType.Mount = mount @@ -3500,28 +6613,47 @@ var _ = Describe("CSIControllerService", func() { VolumeId: validBlockVolumeID, VolumeContext: nil, VolumeCapabilities: []*csi.VolumeCapability{ + { + AccessMode: &csi.VolumeCapability_AccessMode{Mode: csi.VolumeCapability_AccessMode_UNKNOWN}, + AccessType: accessType, + }, + }, + }) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(res.Confirmed).To(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("access mode cannot be UNKNOWN")) + }) + }) + ginkgo.When("resource ID is null", func() { + ginkgo.It("should fail", func() { + mount := new(csi.VolumeCapability_MountVolume) + accessType := new(csi.VolumeCapability_Mount) + accessType.Mount = mount + _, err := ctrlSvc.ValidateVolumeCapabilities(context.Background(), &csi.ValidateVolumeCapabilitiesRequest{ + VolumeId: "", + VolumeContext: nil, + VolumeCapabilities: []*csi.VolumeCapability{ { AccessMode: &csi.VolumeCapability_AccessMode{Mode: csi.VolumeCapability_AccessMode_UNKNOWN}, AccessType: accessType, }, }, }) - Expect(err).ToNot(BeNil()) - Expect(res.Confirmed).To(BeNil()) - Expect(err.Error()).To(ContainSubstring("access mode cannot be UNKNOWN")) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("No such volume")) }) }) }) - Describe("calling ControllerGetCapabilities()", func() { - When("plugin functions correctly with health monitor capabilities", func() { - It("should return supported capabilities", func() { - csictx.Setenv(context.Background(), common.EnvIsHealthMonitorEnabled, "true") + ginkgo.Describe("calling ControllerGetCapabilities()", func() { + ginkgo.When("plugin functions correctly with health monitor capabilities", func() { + ginkgo.It("should return supported capabilities", func() { + csictx.Setenv(context.Background(), identifiers.EnvIsHealthMonitorEnabled, "true") ctrlSvc.Init() res, err := ctrlSvc.ControllerGetCapabilities(context.Background(), &csi.ControllerGetCapabilitiesRequest{}) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.ControllerGetCapabilitiesResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerGetCapabilitiesResponse{ Capabilities: []*csi.ControllerServiceCapability{ { Type: &csi.ControllerServiceCapability_Rpc{ @@ -3575,14 +6707,21 @@ var _ = Describe("CSIControllerService", func() { { Type: &csi.ControllerServiceCapability_Rpc{ Rpc: &csi.ControllerServiceCapability_RPC{ - Type: csi.ControllerServiceCapability_RPC_SINGLE_NODE_MULTI_WRITER, + Type: csi.ControllerServiceCapability_RPC_SINGLE_NODE_MULTI_WRITER, + }, + }, + }, + { + Type: &csi.ControllerServiceCapability_Rpc{ + Rpc: &csi.ControllerServiceCapability_RPC{ + Type: csi.ControllerServiceCapability_RPC_GET_VOLUME, }, }, }, { Type: &csi.ControllerServiceCapability_Rpc{ Rpc: &csi.ControllerServiceCapability_RPC{ - Type: csi.ControllerServiceCapability_RPC_GET_VOLUME, + Type: csi.ControllerServiceCapability_RPC_LIST_VOLUMES, }, }, }, @@ -3604,13 +6743,13 @@ var _ = Describe("CSIControllerService", func() { })) }) }) - When("plugin functions correctly without health monitor capabilities", func() { - It("should return supported capabilities", func() { - csictx.Setenv(context.Background(), common.EnvIsHealthMonitorEnabled, "false") + ginkgo.When("plugin functions correctly without health monitor capabilities", func() { + ginkgo.It("should return supported capabilities", func() { + csictx.Setenv(context.Background(), identifiers.EnvIsHealthMonitorEnabled, "false") ctrlSvc.Init() res, err := ctrlSvc.ControllerGetCapabilities(context.Background(), &csi.ControllerGetCapabilitiesRequest{}) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.ControllerGetCapabilitiesResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerGetCapabilitiesResponse{ Capabilities: []*csi.ControllerServiceCapability{ { Type: &csi.ControllerServiceCapability_Rpc{ @@ -3674,11 +6813,12 @@ var _ = Describe("CSIControllerService", func() { }) }) - Describe("calling DiscoverStorageProtectionGroup", func() { - When("get info about protection group", func() { + ginkgo.Describe("calling DiscoverStorageProtectionGroup", func() { + ginkgo.When("get info about protection group", func() { getLocalAndRemoteParams := func(localSystemName string, localAddress string, remoteSystemName string, remoteAddress string, - remoteSerialNumber string, volumeGroupName string) (map[string]string, map[string]string) { + remoteSerialNumber string, volumeGroupName string, + ) (map[string]string, map[string]string) { localParams := map[string]string{ "globalID": localAddress, "systemName": localSystemName, @@ -3701,19 +6841,20 @@ var _ = Describe("CSIControllerService", func() { return localParams, remoteParams } - It("should successfully discover protection group if everything is ok", func() { + ginkgo.It("should successfully discover protection group if everything is ok", func() { clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID). Return(gopowerstore.VolumeGroups{VolumeGroup: []gopowerstore.VolumeGroup{{ID: validGroupID, Name: validVolumeGroupName}}}, nil) clientMock.On("GetReplicationSessionByLocalResourceID", mock.Anything, validGroupID). Return(gopowerstore.ReplicationSession{ - RemoteSystemId: validRemoteSystemID, - LocalResourceId: validGroupID, - RemoteResourceId: validRemoteGroupID, + RemoteSystemID: validRemoteSystemID, + LocalResourceID: validGroupID, + RemoteResourceID: validRemoteGroupID, StorageElementPairs: []gopowerstore.StorageElementPair{{ - LocalStorageElementId: validBaseVolID, - RemoteStorageElementId: validRemoteVolId, - }}}, nil) + LocalStorageElementID: validBaseVolID, + RemoteStorageElementID: validRemoteVolID, + }}, + }, nil) clientMock.On("GetCluster", mock.Anything). Return(gopowerstore.Cluster{Name: validClusterName, ManagementAddress: firstValidID}, nil) @@ -3722,7 +6863,8 @@ var _ = Describe("CSIControllerService", func() { Return(gopowerstore.RemoteSystem{ Name: validRemoteSystemName, ManagementAddress: secondValidID, - SerialNumber: validRemoteSystemGlobalID}, nil) + SerialNumber: validRemoteSystemGlobalID, + }, nil) req := &csiext.CreateStorageProtectionGroupRequest{ VolumeHandle: validBaseVolID + "/" + firstValidID + "/" + "iscsi", @@ -3733,9 +6875,8 @@ var _ = Describe("CSIControllerService", func() { localParams, remoteParams := getLocalAndRemoteParams(validClusterName, firstValidID, validRemoteSystemName, secondValidID, validRemoteSystemGlobalID, validVolumeGroupName) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csiext.CreateStorageProtectionGroupResponse{ - + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csiext.CreateStorageProtectionGroupResponse{ LocalProtectionGroupId: validGroupID, RemoteProtectionGroupId: validRemoteGroupID, LocalProtectionGroupAttributes: localParams, @@ -3743,40 +6884,42 @@ var _ = Describe("CSIControllerService", func() { })) }) - It("should fail if volume doesn't exists", func() { + ginkgo.It("should fail if volume doesn't exists", func() { req := &csiext.CreateStorageProtectionGroupRequest{ VolumeHandle: "", } res, err := ctrlSvc.CreateStorageProtectionGroup(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).NotTo(BeNil()) - Expect(err.Error()).To(ContainSubstring("volume ID is required")) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("volume ID is required")) }) - It("should fail if volume is single", func() { - + ginkgo.It("should fail if volume is single", func() { clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID). Return(gopowerstore.VolumeGroups{}, gopowerstore.APIError{}) - + clientMock.On("GetCluster", mock.Anything). + Return(gopowerstore.Cluster{Name: validClusterName}, nil) req := &csiext.CreateStorageProtectionGroupRequest{ VolumeHandle: validBaseVolID + "/" + firstValidID + "/" + "iscsi", } res, err := ctrlSvc.CreateStorageProtectionGroup(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).NotTo(BeNil()) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) }) - It("should fail when volume group not in replication session", func() { + ginkgo.It("should fail when volume group not in replication session", func() { // policy with replication rule not assigned clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID). Return(gopowerstore.VolumeGroups{VolumeGroup: []gopowerstore.VolumeGroup{{ID: validGroupID}}}, nil) clientMock.On("GetReplicationSessionByLocalResourceID", mock.Anything, validGroupID). Return(gopowerstore.ReplicationSession{}, gopowerstore.APIError{}) + clientMock.On("GetCluster", mock.Anything). + Return(gopowerstore.Cluster{Name: validClusterName}, nil) req := &csiext.CreateStorageProtectionGroupRequest{ VolumeHandle: validBaseVolID + "/" + firstValidID + "/" + "iscsi", @@ -3784,27 +6927,27 @@ var _ = Describe("CSIControllerService", func() { res, err := ctrlSvc.CreateStorageProtectionGroup(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).NotTo(BeNil()) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) }) }) }) - Describe("calling CreateRemoteVolume", func() { - When("creating remote volume", func() { - It("should return info if everything is ok", func() { + ginkgo.Describe("calling CreateRemoteVolume", func() { + ginkgo.When("creating remote volume", func() { + ginkgo.It("should return info if everything is ok", func() { clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID). Return(gopowerstore.VolumeGroups{VolumeGroup: []gopowerstore.VolumeGroup{{ID: validGroupID}}}, nil) clientMock.On("GetReplicationSessionByLocalResourceID", mock.Anything, validGroupID). Return(gopowerstore.ReplicationSession{ - LocalResourceId: validGroupID, - RemoteResourceId: validRemoteGroupID, - RemoteSystemId: validRemoteSystemID, + LocalResourceID: validGroupID, + RemoteResourceID: validRemoteGroupID, + RemoteSystemID: validRemoteSystemID, StorageElementPairs: []gopowerstore.StorageElementPair{ { - LocalStorageElementId: validBaseVolID, - RemoteStorageElementId: validRemoteVolId, + LocalStorageElementID: validBaseVolID, + RemoteStorageElementID: validRemoteVolID, }, }, }, nil) @@ -3824,32 +6967,31 @@ var _ = Describe("CSIControllerService", func() { res, err := ctrlSvc.CreateRemoteVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal( + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal( &csiext.CreateRemoteVolumeResponse{RemoteVolume: &csiext.Volume{ CapacityBytes: validVolSize, - VolumeId: validRemoteVolId + "/" + validRemoteSystemGlobalID + "/" + "iscsi", + VolumeId: validRemoteVolID + "/" + validRemoteSystemGlobalID + "/" + "iscsi", VolumeContext: map[string]string{ "remoteSystem": validClusterName, "managementAddress": secondValidID, "arrayID": validRemoteSystemGlobalID, }, }})) - }) - It("should fail if volume id is empty", func() { + ginkgo.It("should fail if volume id is empty", func() { req := &csiext.CreateRemoteVolumeRequest{ VolumeHandle: "", } res, err := ctrlSvc.CreateRemoteVolume(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).NotTo(BeNil()) - Expect(err.Error()).To(ContainSubstring("volume ID is required")) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("volume ID is required")) }) - It("should fail if volume not in volumeGroup", func() { + ginkgo.It("should fail if volume not in volumeGroup", func() { clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID). Return(gopowerstore.VolumeGroups{}, gopowerstore.APIError{}) @@ -3859,11 +7001,11 @@ var _ = Describe("CSIControllerService", func() { res, err := ctrlSvc.CreateRemoteVolume(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).NotTo(BeNil()) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) }) - It("should fail if parent volume group not replicated", func() { + ginkgo.It("should fail if parent volume group not replicated", func() { clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID). Return(gopowerstore.VolumeGroups{}, gopowerstore.APIError{}) @@ -3875,19 +7017,19 @@ var _ = Describe("CSIControllerService", func() { } res, err := ctrlSvc.CreateRemoteVolume(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).NotTo(BeNil()) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) }) - It("should fail if volume group not synced yet", func() { + ginkgo.It("should fail if volume group not synced yet", func() { clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID). Return(gopowerstore.VolumeGroups{VolumeGroup: []gopowerstore.VolumeGroup{{ID: validGroupID}}}, nil) clientMock.On("GetReplicationSessionByLocalResourceID", mock.Anything, validGroupID). Return(gopowerstore.ReplicationSession{ - LocalResourceId: validGroupID, - RemoteResourceId: validRemoteGroupID, - RemoteSystemId: validRemoteSystemID, + LocalResourceID: validGroupID, + RemoteResourceID: validRemoteGroupID, + RemoteSystemID: validRemoteSystemID, StorageElementPairs: []gopowerstore.StorageElementPair{}, }, nil) @@ -3896,29 +7038,27 @@ var _ = Describe("CSIControllerService", func() { } res, err := ctrlSvc.CreateRemoteVolume(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).NotTo(BeNil()) - Expect(err.Error()).To(ContainSubstring( + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring( fmt.Sprintf("couldn't find volume id %s in storage element pairs of replication session", validBaseVolID))) }) - It("should fail if the array id is nil", func() { - + ginkgo.It("should fail if the array id is nil", func() { // create volume handle with nil array ID req := &csiext.CreateRemoteVolumeRequest{ VolumeHandle: validBaseVolID + "/" + "/" + "iscsi", } res, err := ctrlSvc.CreateRemoteVolume(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).NotTo(BeNil()) - Expect(err.Error()).To(ContainSubstring( + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring( "failed to find array with given IP", )) }) - It("should fail if a volume group does not exist for the volume", func() { - + ginkgo.It("should fail if a volume group does not exist for the volume", func() { // return an empty volume group clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID).Return(gopowerstore.VolumeGroups{}, nil) @@ -3927,42 +7067,251 @@ var _ = Describe("CSIControllerService", func() { } res, err := ctrlSvc.CreateRemoteVolume(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring( + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring( "replication of volumes that aren't assigned to group is not implemented yet", )) }) + ginkgo.It("should fail if cluster retrieval fails", func() { + clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID). + Return(gopowerstore.VolumeGroups{VolumeGroup: []gopowerstore.VolumeGroup{{ID: validGroupID}}}, nil) + + clientMock.On("GetReplicationSessionByLocalResourceID", mock.Anything, validGroupID). + Return(gopowerstore.ReplicationSession{ + LocalResourceID: validGroupID, + RemoteResourceID: validRemoteGroupID, + RemoteSystemID: validRemoteSystemID, + StorageElementPairs: []gopowerstore.StorageElementPair{{ + LocalStorageElementID: validBaseVolID, + RemoteStorageElementID: validRemoteVolID, + }}, + }, nil) + + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, Size: validVolSize}, nil) + + clientMock.On("GetCluster", mock.Anything). + Return(gopowerstore.Cluster{}, errors.New("cluster not found")) + + req := &csiext.CreateRemoteVolumeRequest{ + VolumeHandle: validBaseVolID + "/" + firstValidID + "/" + "iscsi", + } + res, err := ctrlSvc.CreateRemoteVolume(context.Background(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("cluster not found")) + }) + ginkgo.It("should return info if everything is ok for NFS", func() { + clientMock.On("GetFS", mock.Anything, validBaseVolID).Return(gopowerstore.FileSystem{ID: validBaseVolID}, nil) + clientMock.On("GetNFSExportByFileSystemID", mock.Anything, validBaseVolID).Return(gopowerstore.NFSExport{ID: validNasID}, nil) + + clientMock.On("GetReplicationSessionByLocalResourceID", mock.Anything, mock.AnythingOfType("string")). + Return(gopowerstore.ReplicationSession{ + LocalResourceID: validNfsVolumeID, + RemoteResourceID: validRemoteGroupID, + RemoteSystemID: validRemoteSystemID, + StorageElementPairs: []gopowerstore.StorageElementPair{ + { + LocalStorageElementID: validBaseVolID, + RemoteStorageElementID: validRemoteVolID, + }, + }, + }, nil) + + clientMock.On("GetCluster", mock.Anything).Return(gopowerstore.Cluster{Name: validClusterName}, nil) + + clientMock.On("GetRemoteSystem", mock.Anything, validRemoteSystemID).Return(gopowerstore.RemoteSystem{ + Name: validRemoteSystemName, + ManagementAddress: secondValidID, + ID: validRemoteSystemID, + SerialNumber: validRemoteSystemGlobalID, + }, nil) + + req := &csiext.CreateRemoteVolumeRequest{ + VolumeHandle: validBaseVolID + "/" + firstValidID + "/" + "nfs", + } + + res, err := ctrlSvc.CreateRemoteVolume(context.Background(), req) + + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csiext.CreateRemoteVolumeResponse{RemoteVolume: &csiext.Volume{ + CapacityBytes: 0, + VolumeId: validBaseVolID + "/" + validRemoteSystemGlobalID + "/" + "nfs", + VolumeContext: map[string]string{ + "remoteSystem": validClusterName, + "managementAddress": secondValidID, + "arrayID": validRemoteSystemGlobalID, + }, + }})) + }) + ginkgo.It("should handle error when NFS export retrieval fails and create a new export", func() { + clientMock.On("GetNFSExportByFileSystemID", mock.Anything, validBaseVolID). + Return(gopowerstore.NFSExport{}, errors.New("NFS export not found")).Once() + + clientMock.On("GetFS", mock.Anything, validBaseVolID). + Return(gopowerstore.FileSystem{ID: validBaseVolID, Name: "fs_name"}, nil) + + clientMock.On("CreateNFSExport", mock.Anything, mock.MatchedBy(func(req *gopowerstore.NFSExportCreate) bool { + return req != nil && + req.FileSystemID == validBaseVolID && + req.Name == "auto_export_"+validBaseVolID[:8] && + req.Path == "/fs_name" + })).Return(gopowerstore.CreateResponse{ID: "new-export-id"}, nil) + + clientMock.On("GetNFSExportByFileSystemID", mock.Anything, validBaseVolID). + Return(gopowerstore.NFSExport{ID: "new-export-id"}, nil).Once() + clientMock.On("GetCluster", mock.Anything).Return(gopowerstore.Cluster{Name: validClusterName}, nil) + + clientMock.On("GetReplicationSessionByLocalResourceID", mock.Anything, validBaseVolID). + Return(gopowerstore.ReplicationSession{ + RemoteSystemID: validRemoteSystemID, + LocalResourceID: validBaseVolID, + RemoteResourceID: validRemoteGroupID, + }, nil) + + clientMock.On("GetRemoteSystem", mock.Anything, validRemoteSystemID). + Return(gopowerstore.RemoteSystem{ + Name: "remote-system", + ManagementAddress: "10.0.0.2", + SerialNumber: "RSN-999", + }, nil) + + arr := &array.PowerStoreArray{Client: clientMock} + // reuse the existing test controller and inject the test array into its Arrays map + arrays := ctrlSvc.Arrays() + arrays["array-id"] = arr + ctrlSvc.SetArrays(arrays) + + req := &csiext.CreateRemoteVolumeRequest{ + VolumeHandle: validBaseVolID + "/array-id/nfs", + } + + res, err := ctrlSvc.CreateRemoteVolume(context.TODO(), req) + + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + gomega.Expect(res).ToNot(gomega.BeNil()) + gomega.Expect(res.GetRemoteVolume().GetVolumeContext()["replication.NasServerID"]).To(gomega.Equal("")) + }) + ginkgo.It("should return error when CreateNFSExport fails", func() { + clientMock.On("GetNFSExportByFileSystemID", mock.Anything, validBaseVolID). + Return(gopowerstore.NFSExport{}, gopowerstore.NewNotFoundError()).Once() + + clientMock.On("GetFS", mock.Anything, validBaseVolID). + Return(gopowerstore.FileSystem{ID: validBaseVolID, Name: "fs_name"}, nil) + + clientMock.On("CreateNFSExport", mock.Anything, mock.AnythingOfType("*gopowerstore.NFSExportCreate")). + Return(gopowerstore.CreateResponse{}, errors.New("mock export creation failure")) + + arr := &array.PowerStoreArray{Client: clientMock} + // reuse the existing test controller and inject the test array into its Arrays map + arrays := ctrlSvc.Arrays() + arrays["array-id"] = arr + ctrlSvc.SetArrays(arrays) + controller := ctrlSvc + + req := &csiext.CreateRemoteVolumeRequest{ + VolumeHandle: validBaseVolID + "/array-id/nfs", + } + + res, err := controller.CreateRemoteVolume(context.TODO(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(status.Code(err)).To(gomega.Equal(codes.Internal)) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("failed to create NFS export")) + }) + ginkgo.It("should return error when replication session is not found", func() { + clientMock.On("GetNFSExportByFileSystemID", mock.Anything, validBaseVolID). + Return(gopowerstore.NFSExport{ID: "export-id"}, nil) + + clientMock.On("GetReplicationSessionByLocalResourceID", mock.Anything, validBaseVolID). + Return(gopowerstore.ReplicationSession{}, errors.New("replication session not found")) + + arr := &array.PowerStoreArray{Client: clientMock} + arrays := ctrlSvc.Arrays() + arrays["array-id"] = arr + ctrlSvc.SetArrays(arrays) + controller := ctrlSvc + + req := &csiext.CreateRemoteVolumeRequest{ + VolumeHandle: validBaseVolID + "/array-id/nfs", + } + + res, err := controller.CreateRemoteVolume(context.TODO(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(status.Code(err)).To(gomega.Equal(codes.Internal)) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("no replication session found")) + }) + ginkgo.It("should create storage protection group for NFS volume", func() { + // Setup mocks + clientMock.On("GetNFSExportByFileSystemID", mock.Anything, validBaseVolID). + Return(gopowerstore.NFSExport{ID: validNasID}, nil) + + clientMock.On("GetReplicationSessionByLocalResourceID", mock.Anything, validBaseVolID). + Return(gopowerstore.ReplicationSession{ + LocalResourceID: validBaseVolID, + RemoteResourceID: validRemoteGroupID, + RemoteSystemID: validRemoteSystemID, + StorageElementPairs: []gopowerstore.StorageElementPair{ + { + LocalStorageElementID: validBaseVolID, + RemoteStorageElementID: validRemoteVolID, + }, + }, + }, nil) + + clientMock.On("GetCluster", mock.Anything). + Return(gopowerstore.Cluster{Name: validClusterName}, nil) + + clientMock.On("GetRemoteSystem", mock.Anything, validRemoteSystemID). + Return(gopowerstore.RemoteSystem{ + Name: validRemoteSystemName, + ManagementAddress: secondValidID, + ID: validRemoteSystemID, + SerialNumber: validRemoteSystemGlobalID, + }, nil) + + req := &csiext.CreateStorageProtectionGroupRequest{ + VolumeHandle: validBaseVolID + "/" + firstValidID + "/nfs", + } + + res, err := ctrlSvc.CreateStorageProtectionGroup(context.Background(), req) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res.LocalProtectionGroupAttributes).To(gomega.HaveKeyWithValue("systemName", "localSystemName")) + gomega.Expect(res.RemoteProtectionGroupAttributes).To(gomega.HaveKeyWithValue("remoteSystemName", "localSystemName")) + }) }) }) - - Describe("calling EnsureProtectionPolicyExists", func() { - When("ensure protection policy exists", func() { - It("should failed if remote system not in list", func() { + ginkgo.Describe("calling EnsureProtectionPolicyExists", func() { + ginkgo.When("ensure protection policy exists", func() { + ginkgo.It("should failed if remote system not in list", func() { clientMock.On("GetRemoteSystemByName", mock.Anything, validRemoteSystemName). Return(gopowerstore.RemoteSystem{}, gopowerstore.NewHostIsNotExistError()) - _, err := controller.EnsureProtectionPolicyExists(context.Background(), ctrlSvc.DefaultArray(), + _, err := EnsureProtectionPolicyExists(context.Background(), ctrlSvc.DefaultArray(), validGroupName, validRemoteSystemName, validRPO) - Expect(err).ToNot(BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) }) - It("should return existing policy", func() { - + ginkgo.It("should return existing policy", func() { clientMock.On("GetRemoteSystemByName", mock.Anything, validRemoteSystemName). Return(gopowerstore.RemoteSystem{ID: validRemoteSystemID, Name: validRemoteSystemName}, nil) clientMock.On("GetProtectionPolicyByName", mock.Anything, validPolicyName). Return(gopowerstore.ProtectionPolicy{ID: validPolicyID, Name: validPolicyName}, nil) - res, err := controller.EnsureProtectionPolicyExists(context.Background(), ctrlSvc.DefaultArray(), + res, err := EnsureProtectionPolicyExists(context.Background(), ctrlSvc.DefaultArray(), validGroupName, validRemoteSystemName, validRPO) - Expect(err).To(BeNil()) - Expect(res).To(Equal(validPolicyID)) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(validPolicyID)) }) - It("should successfully create new policy with existing rule", func() { + ginkgo.It("should successfully create new policy with existing rule", func() { clientMock.On("GetRemoteSystemByName", mock.Anything, validRemoteSystemName). Return(gopowerstore.RemoteSystem{ID: validRemoteSystemID, Name: validRemoteSystemName}, nil) @@ -3975,20 +7324,19 @@ var _ = Describe("CSIControllerService", func() { clientMock.On("CreateProtectionPolicy", mock.Anything, &gopowerstore.ProtectionPolicyCreate{ Name: validPolicyName, - ReplicationRuleIds: []string{validRuleID}, + ReplicationRuleIDs: []string{validRuleID}, }).Return(gopowerstore.CreateResponse{ID: validPolicyID}, nil) - res, err := controller.EnsureProtectionPolicyExists(context.Background(), ctrlSvc.DefaultArray(), + res, err := EnsureProtectionPolicyExists(context.Background(), ctrlSvc.DefaultArray(), validGroupName, validRemoteSystemName, validRPO) - Expect(err).To(BeNil()) - Expect(res).To(Equal(validPolicyID)) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(validPolicyID)) }) - }) }) - Describe("calling EnsureReplicationRuleExists", func() { - When("ensure replication rule exists", func() { - It("should successfully create new rule if it doesn't exists", func() { + ginkgo.Describe("calling EnsureReplicationRuleExists", func() { + ginkgo.When("ensure replication rule exists", func() { + ginkgo.It("should successfully create new rule if it doesn't exists", func() { clientMock.On("GetReplicationRuleByName", mock.Anything, validRuleName). Return(gopowerstore.ReplicationRule{ID: validRuleID}, gopowerstore.APIError{}) @@ -4000,26 +7348,25 @@ var _ = Describe("CSIControllerService", func() { }, ).Return(gopowerstore.CreateResponse{ID: validRuleID}, nil) - res, err := controller.EnsureReplicationRuleExists(context.Background(), ctrlSvc.DefaultArray(), + res, err := EnsureReplicationRuleExists(context.Background(), ctrlSvc.DefaultArray(), validGroupName, validRemoteSystemID, gopowerstore.RpoFiveMinutes) - Expect(err).To(BeNil()) - Expect(res).To(Equal(validRuleID)) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(validRuleID)) }) - It("should return existing rule", func() { + ginkgo.It("should return existing rule", func() { clientMock.On("GetReplicationRuleByName", mock.Anything, validRuleName). Return(gopowerstore.ReplicationRule{ID: validRuleID}, nil) - res, err := controller.EnsureReplicationRuleExists(context.Background(), ctrlSvc.DefaultArray(), + res, err := EnsureReplicationRuleExists(context.Background(), ctrlSvc.DefaultArray(), validGroupName, validRemoteSystemID, validRPO) - Expect(err).To(BeNil()) - Expect(res).To(Equal(validRuleID)) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(validRuleID)) }) - It("should fail to create a replication rule", func() { - + ginkgo.It("should fail to create a replication rule", func() { clientMock.On("GetReplicationRuleByName", mock.Anything, validRuleName).Return( gopowerstore.ReplicationRule{ID: validRuleID}, gopowerstore.NewNotFoundError(), @@ -4037,40 +7384,32 @@ var _ = Describe("CSIControllerService", func() { }, ).Return(gopowerstore.CreateResponse{}, gopowerstore.WrapErr(apiErr)) - res, err := controller.EnsureReplicationRuleExists(context.Background(), ctrlSvc.DefaultArray(), + res, err := EnsureReplicationRuleExists(context.Background(), ctrlSvc.DefaultArray(), validGroupName, validRemoteSystemID, validRPO) - Expect(res).To(BeEmpty()) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring("can't create replication rule")) + gomega.Expect(res).To(gomega.BeEmpty()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("can't create replication rule")) }) }) - }) - Describe("calling ControllerGetVolume", func() { - When("normal block volume exists on array", func() { - It("should successfully get the volume", func() { + ginkgo.Describe("calling ControllerGetVolume", func() { + ginkgo.When("normal block volume exists on array", func() { + ginkgo.It("should successfully get the volume", func() { clientMock.On("GetVolume", mock.Anything, validBaseVolID). Return(gopowerstore.Volume{ID: validBaseVolID, State: gopowerstore.VolumeStateEnumReady}, nil) - clientMock.On("GetHost", mock.Anything, validHostID).Return(gopowerstore.Host{ID: validHostID, Name: validHostName}, nil) - clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, validBaseVolID). Return([]gopowerstore.HostVolumeMapping{{HostID: validHostID}}, nil).Once() - clientMock.On("GetFS", mock.Anything, validBaseVolID). - Return(gopowerstore.FileSystem{ID: validBaseVolID}, nil) - - clientMock.On("GetNFSExportByFileSystemID", mock.Anything, validBaseVolID). - Return(gopowerstore.NFSExport{}, nil) + clientMock.On("GetHost", mock.Anything, validHostID).Return(gopowerstore.Host{ID: validHostID, Name: validHostName}, nil) req := &csi.ControllerGetVolumeRequest{VolumeId: validBlockVolumeID} - res, err := ctrlSvc.ControllerGetVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.ControllerGetVolumeResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerGetVolumeResponse{ Volume: &csi.Volume{ VolumeId: validBaseVolID, }, @@ -4084,8 +7423,38 @@ var _ = Describe("CSIControllerService", func() { })) }) }) - When("normal block volume does not exists on array", func() { - It("should fail", func() { + + ginkgo.When("normal block volume exists on array with different state", func() { + ginkgo.It("should fail", func() { + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, State: gopowerstore.VolumeStateEnumInitializing}, nil) + + clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, validBaseVolID). + Return([]gopowerstore.HostVolumeMapping{{HostID: validHostID}}, nil).Once() + + clientMock.On("GetHost", mock.Anything, validHostID).Return(gopowerstore.Host{ID: validHostID, Name: validHostName}, nil) + + req := &csi.ControllerGetVolumeRequest{VolumeId: validBlockVolumeID} + res, err := ctrlSvc.ControllerGetVolume(context.Background(), req) + + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerGetVolumeResponse{ + Volume: &csi.Volume{ + VolumeId: validBaseVolID, + }, + Status: &csi.ControllerGetVolumeResponse_VolumeStatus{ + PublishedNodeIds: []string{validHostName}, + VolumeCondition: &csi.VolumeCondition{ + Abnormal: true, + Message: fmt.Sprintf("Volume %s is in Initializing state", validBaseVolID), + }, + }, + })) + }) + }) + + ginkgo.When("normal block volume does not exist on array", func() { + ginkgo.It("should fail", func() { var hosts []string clientMock.On("GetVolume", mock.Anything, mock.Anything). Return(gopowerstore.Volume{}, gopowerstore.APIError{ @@ -4094,23 +7463,11 @@ var _ = Describe("CSIControllerService", func() { }, }) - clientMock.On("GetHost", mock.Anything, validHostID).Return(gopowerstore.Host{ID: validHostID, Name: validHostName}, nil) - - clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, validBaseVolID). - Return([]gopowerstore.HostVolumeMapping{{HostID: validHostID}}, nil).Once() - - clientMock.On("GetFS", mock.Anything, validBaseVolID). - Return(gopowerstore.FileSystem{ID: validBaseVolID}, nil) - - clientMock.On("GetNFSExportByFileSystemID", mock.Anything, validBaseVolID). - Return(gopowerstore.NFSExport{}, nil) - req := &csi.ControllerGetVolumeRequest{VolumeId: validBlockVolumeID} - res, err := ctrlSvc.ControllerGetVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.ControllerGetVolumeResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerGetVolumeResponse{ Volume: &csi.Volume{ VolumeId: validBaseVolID, }, @@ -4124,17 +7481,10 @@ var _ = Describe("CSIControllerService", func() { })) }) }) - When("normal filesystem exists on array", func() { - It("should successfully get the filesystem", func() { - var hosts []string - clientMock.On("GetVolume", mock.Anything, validBaseVolID). - Return(gopowerstore.Volume{ID: validBaseVolID, State: gopowerstore.VolumeStateEnumReady}, nil) - - clientMock.On("GetHost", mock.Anything, validHostID).Return(gopowerstore.Host{ID: validHostID, Name: validHostName}, nil) - - clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, validBaseVolID). - Return([]gopowerstore.HostVolumeMapping{{HostID: validHostID}}, nil).Once() + ginkgo.When("normal filesystem exists on array", func() { + ginkgo.It("should successfully get the filesystem", func() { + var hosts []string clientMock.On("GetFS", mock.Anything, validBaseVolID). Return(gopowerstore.FileSystem{ID: validBaseVolID}, nil) @@ -4142,11 +7492,10 @@ var _ = Describe("CSIControllerService", func() { Return(gopowerstore.NFSExport{}, nil) req := &csi.ControllerGetVolumeRequest{VolumeId: validNfsVolumeID} - res, err := ctrlSvc.ControllerGetVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.ControllerGetVolumeResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerGetVolumeResponse{ Volume: &csi.Volume{ VolumeId: validBaseVolID, }, @@ -4160,17 +7509,10 @@ var _ = Describe("CSIControllerService", func() { })) }) }) - When("filesystem does not exists on array", func() { - It("should fail", func() { - var hosts []string - clientMock.On("GetVolume", mock.Anything, validBaseVolID). - Return(gopowerstore.Volume{ID: validBaseVolID, State: gopowerstore.VolumeStateEnumReady}, nil) - - clientMock.On("GetHost", mock.Anything, validHostID).Return(gopowerstore.Host{ID: validHostID, Name: validHostName}, nil) - - clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, validBaseVolID). - Return([]gopowerstore.HostVolumeMapping{{HostID: validHostID}}, nil).Once() + ginkgo.When("filesystem does not exist on array", func() { + ginkgo.It("should fail", func() { + var hosts []string clientMock.On("GetFS", mock.Anything, validBaseVolID). Return(gopowerstore.FileSystem{}, gopowerstore.APIError{ ErrorMsg: &api.ErrorMsg{ @@ -4182,11 +7524,10 @@ var _ = Describe("CSIControllerService", func() { Return(gopowerstore.NFSExport{}, nil) req := &csi.ControllerGetVolumeRequest{VolumeId: validNfsVolumeID} - res, err := ctrlSvc.ControllerGetVolume(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.ControllerGetVolumeResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ControllerGetVolumeResponse{ Volume: &csi.Volume{ VolumeId: validBaseVolID, }, @@ -4200,6 +7541,103 @@ var _ = Describe("CSIControllerService", func() { })) }) }) + + ginkgo.When("volume id is empty", func() { + ginkgo.It("should fail", func() { + req := &csi.ControllerGetVolumeRequest{VolumeId: ""} + res, err := ctrlSvc.ControllerGetVolume(context.Background(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("unable to parse the volume id")) + }) + }) + + ginkgo.When("block API call fails", func() { + ginkgo.It("should fail [GetVolume]", func() { + clientMock.On("GetVolume", mock.Anything, mock.Anything). + Return(gopowerstore.Volume{}, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{StatusCode: http.StatusBadRequest}, + }) + + req := &csi.ControllerGetVolumeRequest{VolumeId: validBlockVolumeID} + res, err := ctrlSvc.ControllerGetVolume(context.Background(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("failed to find volume")) + }) + + ginkgo.It("should fail [GetHostVolumeMappingByVolumeID]", func() { + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, State: gopowerstore.VolumeStateEnumReady}, nil) + + clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, validBaseVolID). + Return([]gopowerstore.HostVolumeMapping{}, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{StatusCode: http.StatusBadRequest}, + }) + + req := &csi.ControllerGetVolumeRequest{VolumeId: validBlockVolumeID} + res, err := ctrlSvc.ControllerGetVolume(context.Background(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("failed to get host volume mapping for volume")) + }) + + ginkgo.It("should fail [GetHost]", func() { + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, State: gopowerstore.VolumeStateEnumReady}, nil) + + clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, validBaseVolID). + Return([]gopowerstore.HostVolumeMapping{{HostID: validHostID}}, nil).Once() + + clientMock.On("GetHost", mock.Anything, validHostID). + Return(gopowerstore.Host{}, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{StatusCode: http.StatusBadRequest}, + }) + + req := &csi.ControllerGetVolumeRequest{VolumeId: validBlockVolumeID} + res, err := ctrlSvc.ControllerGetVolume(context.Background(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("failed to get host")) + }) + }) + + ginkgo.When("filesystem API call fails", func() { + ginkgo.It("should fail [GetFS]", func() { + clientMock.On("GetFS", mock.Anything, validBaseVolID). + Return(gopowerstore.FileSystem{}, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{StatusCode: http.StatusBadRequest}, + }) + + req := &csi.ControllerGetVolumeRequest{VolumeId: validNfsVolumeID} + res, err := ctrlSvc.ControllerGetVolume(context.Background(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("failed to find filesystem")) + }) + + ginkgo.It("should fail [GetNFSExportByFileSystemID]", func() { + clientMock.On("GetFS", mock.Anything, validBaseVolID). + Return(gopowerstore.FileSystem{ID: validBaseVolID}, nil) + + clientMock.On("GetNFSExportByFileSystemID", mock.Anything, validBaseVolID). + Return(gopowerstore.NFSExport{}, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{StatusCode: http.StatusBadRequest}, + }) + + req := &csi.ControllerGetVolumeRequest{VolumeId: validNfsVolumeID} + res, err := ctrlSvc.ControllerGetVolume(context.Background(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("failed to find nfs export for filesystem")) + }) + }) }) }) @@ -4232,7 +7670,7 @@ func getTypicalControllerExpandRequest(volid string, size int64) *csi.Controller VolumeId: volid, CapacityRange: &csi.CapacityRange{ RequiredBytes: size, - LimitBytes: controller.MaxVolumeSizeBytes, + LimitBytes: MaxVolumeSizeBytes, }, } } @@ -4265,7 +7703,7 @@ func getTypicalCreateVolumeNFSRequest(name string, size int64) *csi.CreateVolume capabilities = append(capabilities, getVolumeCapabilityNFS()) req.VolumeCapabilities = capabilities - nfsTopology := &csi.Topology{Segments: map[string]string{common.Name + "/" + ctrlSvc.Arrays()[secondValidID].GetIP() + "-nfs": "true"}} + nfsTopology := &csi.Topology{Segments: map[string]string{identifiers.Name + "/" + ctrlSvc.Arrays()[secondValidID].GetIP() + "-nfs": "true"}} preferred := []*csi.Topology{nfsTopology} accessibilityRequirements := &csi.TopologyRequirement{Preferred: preferred} req.AccessibilityRequirements = accessibilityRequirements @@ -4317,3 +7755,14 @@ func EnsureProtectionPolicyExistsMock() { clientMock.On("GetProtectionPolicyByName", mock.Anything, validPolicyName). Return(gopowerstore.ProtectionPolicy{ID: validPolicyID}, nil) } + +func EnsureProtectionPolicyExistsMockSync() { + // start ensure protection policy exists + clientMock.On("GetRemoteSystemByName", mock.Anything, validRemoteSystemName).Return(gopowerstore.RemoteSystem{ + Name: validRemoteSystemName, + ID: validRemoteSystemID, + }, nil) + + clientMock.On("GetProtectionPolicyByName", mock.Anything, validPolicyNameSync). + Return(gopowerstore.ProtectionPolicy{ID: validPolicyID}, nil) +} diff --git a/pkg/controller/creator.go b/pkg/controller/creator.go index 03df5914..a7c5b9cf 100644 --- a/pkg/controller/creator.go +++ b/pkg/controller/creator.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2021-2023 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2021-2025 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,11 +23,11 @@ import ( "net/http" "strconv" - "github.com/dell/csi-powerstore/v2/pkg/common" + "github.com/dell/csi-powerstore/v2/pkg/identifiers" + "github.com/dell/gopowerstore/api" - "github.com/container-storage-interface/spec/lib/go/csi" "github.com/dell/gopowerstore" - log "github.com/sirupsen/logrus" + "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -82,33 +82,32 @@ func setMetaData(reqParams map[string]string, createParams interface{}) { t.MetaData().Set(HeaderPersistentVolumeClaimName, reqParams[CSIPersistentVolumeClaimName]) t.MetaData().Set(HeaderPersistentVolumeClaimNamespace, reqParams[CSIPersistentVolumeClaimNamespace]) } else { - log.Printf("warning: %T: no MetaData method exists, consider updating gopowerstore library.", createParams) + log.Infof("warning: %T: no MetaData method exists, consider updating gopowerstore library.", createParams) } } func setVolumeCreateAttributes(reqParams map[string]string, createParams *gopowerstore.VolumeCreate) { - if applianceID, ok := reqParams[common.KeyApplianceID]; ok { + if applianceID, ok := reqParams[identifiers.KeyApplianceID]; ok { createParams.ApplianceID = applianceID } - if description, ok := reqParams[common.KeyVolumeDescription]; ok { + if description, ok := reqParams[identifiers.KeyVolumeDescription]; ok { createParams.Description = description } - if protectionPolicyID, ok := reqParams[common.KeyProtectionPolicyID]; ok { + if protectionPolicyID, ok := reqParams[identifiers.KeyProtectionPolicyID]; ok { createParams.ProtectionPolicyID = protectionPolicyID } - if performancePolicyID, ok := reqParams[common.KeyPerformancePolicyID]; ok { + if performancePolicyID, ok := reqParams[identifiers.KeyPerformancePolicyID]; ok { createParams.PerformancePolicyID = performancePolicyID } - if appType, ok := reqParams[common.KeyAppType]; ok { + if appType, ok := reqParams[identifiers.KeyAppType]; ok { createParams.AppType = gopowerstore.AppTypeEnum(appType) - if appTypeOther, ok := reqParams[common.KeyAppTypeOther]; ok { + if appTypeOther, ok := reqParams[identifiers.KeyAppTypeOther]; ok { createParams.AppTypeOther = appTypeOther } } } func validateHostIOSize(hostIOSize string) string { - switch hostIOSize { case gopowerstore.VMware8K, gopowerstore.VMware16K, @@ -121,10 +120,10 @@ func validateHostIOSize(hostIOSize string) string { } func setFLRAttributes(reqParams map[string]string, createParams *gopowerstore.FsCreate) { - flrMode, flrModeFound := reqParams[common.KeyFlrCreateMode] - flrDefaultRetention, flrDefaultRetentionFound := reqParams[common.KeyFlrDefaultRetention] - flrMinimumRetention, flrMinimumRetentionFound := reqParams[common.KeyFlrMinRetention] - flrMaximumRetention, flrMaximumRetentionFound := reqParams[common.KeyFlrMaxRetention] + flrMode, flrModeFound := reqParams[identifiers.KeyFlrCreateMode] + flrDefaultRetention, flrDefaultRetentionFound := reqParams[identifiers.KeyFlrDefaultRetention] + flrMinimumRetention, flrMinimumRetentionFound := reqParams[identifiers.KeyFlrMinRetention] + flrMaximumRetention, flrMaximumRetentionFound := reqParams[identifiers.KeyFlrMaxRetention] if flrModeFound || flrDefaultRetentionFound || @@ -148,40 +147,40 @@ func setFLRAttributes(reqParams map[string]string, createParams *gopowerstore.Fs } func setNFSCreateAttributes(reqParams map[string]string, createParams *gopowerstore.FsCreate) { - if description, ok := reqParams[common.KeyVolumeDescription]; ok { + if description, ok := reqParams[identifiers.KeyVolumeDescription]; ok { createParams.Description = description } - if configType, ok := reqParams[common.KeyConfigType]; ok { + if configType, ok := reqParams[identifiers.KeyConfigType]; ok { createParams.ConfigType = configType } - if accessPolicy, ok := reqParams[common.KeyAccessPolicy]; ok { + if accessPolicy, ok := reqParams[identifiers.KeyAccessPolicy]; ok { createParams.AccessPolicy = accessPolicy } - if lockingPolicy, ok := reqParams[common.KeyLockingPolicy]; ok { + if lockingPolicy, ok := reqParams[identifiers.KeyLockingPolicy]; ok { createParams.LockingPolicy = lockingPolicy } - if folderRenamePolicy, ok := reqParams[common.KeyFolderRenamePolicy]; ok { + if folderRenamePolicy, ok := reqParams[identifiers.KeyFolderRenamePolicy]; ok { createParams.FolderRenamePolicy = folderRenamePolicy } - if isAsyncMTimeEnabled, ok := reqParams[common.KeyIsAsyncMtimeEnabled]; ok { + if isAsyncMTimeEnabled, ok := reqParams[identifiers.KeyIsAsyncMtimeEnabled]; ok { if val, err := strconv.ParseBool(isAsyncMTimeEnabled); err == nil { createParams.IsAsyncMTimeEnabled = val } } - if protectionPolicyID, ok := reqParams[common.KeyProtectionPolicyID]; ok { - createParams.ProtectionPolicyId = protectionPolicyID + if protectionPolicyID, ok := reqParams[identifiers.KeyProtectionPolicyID]; ok { + createParams.ProtectionPolicyID = protectionPolicyID } - if fileEventsPublishingMode, ok := reqParams[common.KeyFileEventsPublishingMode]; ok { + if fileEventsPublishingMode, ok := reqParams[identifiers.KeyFileEventsPublishingMode]; ok { createParams.FileEventsPublishingMode = fileEventsPublishingMode } - if hostIOSize, ok := reqParams[common.KeyHostIoSize]; ok { + if hostIOSize, ok := reqParams[identifiers.KeyHostIoSize]; ok { createParams.HostIOSize = validateHostIOSize(hostIOSize) } setFLRAttributes(reqParams, createParams) } // CheckSize validates that size is correct and returns size in bytes -func (*SCSICreator) CheckSize(ctx context.Context, cr *csi.CapacityRange, isAutoRoundOffFsSizeEnabled bool) (int64, error) { +func (*SCSICreator) CheckSize(_ context.Context, cr *csi.CapacityRange, _ bool) (int64, error) { minSize := cr.GetRequiredBytes() maxSize := cr.GetLimitBytes() @@ -205,7 +204,7 @@ func (*SCSICreator) CheckSize(ctx context.Context, cr *csi.CapacityRange, isAuto } // CheckName validates volume name -func (*SCSICreator) CheckName(ctx context.Context, name string) error { +func (*SCSICreator) CheckName(_ context.Context, name string) error { return volumeNameValidation(name) } @@ -213,7 +212,7 @@ func (*SCSICreator) CheckName(ctx context.Context, name string) error { func (*SCSICreator) CheckIfAlreadyExists(ctx context.Context, name string, sizeInBytes int64, client gopowerstore.Client) (*csi.Volume, error) { alreadyExistVolume, err := client.GetVolumeByName(ctx, name) if err != nil { - return nil, status.Errorf(codes.Internal, "can't find volume '%s': %s", name, err.Error()) + return nil, status.Errorf(status.Code(err), "can't find volume '%s': %s", name, err.Error()) } if alreadyExistVolume.Size < sizeInBytes { @@ -236,10 +235,10 @@ func (sc *SCSICreator) Create(ctx context.Context, req *csi.CreateVolumeRequest, var reqParams *gopowerstore.VolumeCreate defaultHeaders := client.GetCustomHTTPHeaders() if defaultHeaders == nil { - defaultHeaders = make(http.Header) + defaultHeaders = api.NewSafeHeader().GetHeader() } customHeaders := defaultHeaders - k8sMetadataSupported := common.IsK8sMetadataSupported(client) + k8sMetadataSupported := identifiers.IsK8sMetadataSupported(client) if k8sMetadataSupported && metadata["k8s_pvol_name"] != "" && metadata["k8s_claim_name"] != "" && @@ -252,7 +251,7 @@ func (sc *SCSICreator) Create(ctx context.Context, req *csi.CreateVolumeRequest, } if sc.vg != nil { reqParams.VolumeGroupID = sc.vg.ID - } else if vgID, ok := req.Parameters[common.KeyVolumeGroupID]; ok { + } else if vgID, ok := req.Parameters[identifiers.KeyVolumeGroupID]; ok { reqParams.VolumeGroupID = vgID } setMetaData(req.Parameters, reqParams) @@ -268,7 +267,8 @@ func (sc *SCSICreator) Create(ctx context.Context, req *csi.CreateVolumeRequest, // CreateVolumeFromSnapshot create a volume from an existing snapshot. // The snapshotSource gives the SnapshotId which is the volume to be replicated. func (*SCSICreator) CreateVolumeFromSnapshot(ctx context.Context, snapshotSource *csi.VolumeContentSource_SnapshotSource, - volumeName string, sizeInBytes int64, parameters map[string]string, client gopowerstore.Client) (*csi.Volume, error) { + volumeName string, sizeInBytes int64, parameters map[string]string, client gopowerstore.Client, +) (*csi.Volume, error) { var volumeResponse *csi.Volume // Lookup the volume source volume. sourceVol, err := client.GetVolume(ctx, snapshotSource.SnapshotId) @@ -300,7 +300,8 @@ func (*SCSICreator) CreateVolumeFromSnapshot(ctx context.Context, snapshotSource // Clone creates a clone of a Volume func (*SCSICreator) Clone(ctx context.Context, volumeSource *csi.VolumeContentSource_VolumeSource, - volumeName string, sizeInBytes int64, parameters map[string]string, client gopowerstore.Client) (*csi.Volume, error) { + volumeName string, sizeInBytes int64, parameters map[string]string, client gopowerstore.Client, +) (*csi.Volume, error) { var volumeResponse *csi.Volume // Lookup the volume source volume. sourceVol, err := client.GetVolume(ctx, volumeSource.VolumeId) @@ -372,6 +373,7 @@ type NfsCreator struct { // CheckSize validates that size is correct and returns size in bytes func (*NfsCreator) CheckSize(ctx context.Context, cr *csi.CapacityRange, isAutoRoundOffFsSizeEnabled bool) (int64, error) { + log := log.WithContext(ctx) minSize := cr.GetRequiredBytes() maxSize := cr.GetLimitBytes() @@ -387,7 +389,7 @@ func (*NfsCreator) CheckSize(ctx context.Context, cr *csi.CapacityRange, isAutoR minSize = minSize + VolumeSizeMultiple - mod } - //TODO: This roundoff logic to be removed once platform supports minimum filesystem size + // TODO: This roundoff logic to be removed once platform supports minimum filesystem size if isAutoRoundOffFsSizeEnabled && minSize < MinFilesystemSizeBytes { log.Warn("Auto round off Filesystem size has been enabled! Rounding off PVC size to 3Gi.") return MinFilesystemSizeBytes, nil @@ -401,15 +403,16 @@ func (*NfsCreator) CheckSize(ctx context.Context, cr *csi.CapacityRange, isAutoR } // CheckName validates volume name -func (*NfsCreator) CheckName(ctx context.Context, name string) error { +func (*NfsCreator) CheckName(_ context.Context, name string) error { return volumeNameValidation(name) } // CheckIfAlreadyExists queries storage array if FileSystem with given name exists -func (*NfsCreator) CheckIfAlreadyExists(ctx context.Context, name string, sizeInBytes int64, client gopowerstore.Client) (*csi.Volume, error) { +func (c *NfsCreator) CheckIfAlreadyExists(ctx context.Context, name string, sizeInBytes int64, client gopowerstore.Client) (*csi.Volume, error) { + log := log.WithContext(ctx) alreadyExistVolume, err := client.GetFSByName(ctx, name) if err != nil { - return nil, status.Errorf(codes.Internal, "can't find filesystem '%s': %s", name, err.Error()) + return nil, status.Errorf(status.Code(err), "can't find filesystem '%s': %s", name, err.Error()) } if alreadyExistVolume.SizeTotal < sizeInBytes { @@ -417,7 +420,16 @@ func (*NfsCreator) CheckIfAlreadyExists(ctx context.Context, name string, sizeIn "filesystem '%s' already exists but is incompatible volume size: %d < %d", name, alreadyExistVolume.SizeTotal, sizeInBytes) } - volumeResponse := getCSIVolume(alreadyExistVolume.ID, alreadyExistVolume.SizeTotal) + log.Infof("filesystem '%s' already exists", name) + + // update the nas server name for the volume to ensure CreateVolume adds the correct nas to volume context + nasServerID := alreadyExistVolume.NasServerID + nas, err := client.GetNAS(ctx, nasServerID) + if err != nil { + return nil, status.Errorf(codes.Internal, "can't find nas server '%s': %s", nasServerID, err.Error()) + } + c.nasName = nas.Name + volumeResponse := getCSIVolume(alreadyExistVolume.ID, sizeInBytes) return volumeResponse, nil } @@ -440,7 +452,8 @@ func (c *NfsCreator) Create(ctx context.Context, req *csi.CreateVolumeRequest, s // CreateVolumeFromSnapshot create a FileSystem from an existing FileSystem snapshot. func (*NfsCreator) CreateVolumeFromSnapshot(ctx context.Context, snapshotSource *csi.VolumeContentSource_SnapshotSource, - volumeName string, sizeInBytes int64, parameters map[string]string, client gopowerstore.Client) (*csi.Volume, error) { + volumeName string, sizeInBytes int64, parameters map[string]string, client gopowerstore.Client, +) (*csi.Volume, error) { var volumeResponse *csi.Volume // Lookup the volume source volume. @@ -473,7 +486,8 @@ func (*NfsCreator) CreateVolumeFromSnapshot(ctx context.Context, snapshotSource // Clone creates a clone of a FileSystem func (*NfsCreator) Clone(ctx context.Context, volumeSource *csi.VolumeContentSource_VolumeSource, - volumeName string, sizeInBytes int64, parameters map[string]string, client gopowerstore.Client) (*csi.Volume, error) { + volumeName string, sizeInBytes int64, parameters map[string]string, client gopowerstore.Client, +) (*csi.Volume, error) { var volumeResponse *csi.Volume // Lookup the volume source volume. sourceVol, err := client.GetFS(ctx, volumeSource.VolumeId) diff --git a/pkg/controller/creator_test.go b/pkg/controller/creator_test.go index f4a0766a..933d7e6d 100644 --- a/pkg/controller/creator_test.go +++ b/pkg/controller/creator_test.go @@ -16,24 +16,24 @@ * */ -package controller_test +package controller import ( "context" "errors" + "strings" "testing" - "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/dell/csi-powerstore/v2/pkg/controller" "github.com/dell/gopowerstore" "github.com/dell/gopowerstore/mocks" + "github.com/container-storage-interface/spec/lib/go/csi" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" ) func TestVolumeCreator_CheckSize(t *testing.T) { t.Run("scsi creator", func(t *testing.T) { - sc := &controller.SCSICreator{} + sc := &SCSICreator{} t.Run("zeroes", func(t *testing.T) { cr := &csi.CapacityRange{ RequiredBytes: 0, @@ -42,23 +42,23 @@ func TestVolumeCreator_CheckSize(t *testing.T) { res, err := sc.CheckSize(context.Background(), cr, false) assert.NoError(t, err) - assert.Equal(t, res, int64(controller.MinVolumeSizeBytes)) + assert.Equal(t, res, int64(MinVolumeSizeBytes)) }) t.Run("mod != 0", func(t *testing.T) { cr := &csi.CapacityRange{ - RequiredBytes: controller.MinVolumeSizeBytes + 1, + RequiredBytes: MinVolumeSizeBytes + 1, LimitBytes: 0, } res, err := sc.CheckSize(context.Background(), cr, false) assert.NoError(t, err) - assert.Equal(t, res, int64(controller.MinVolumeSizeBytes+controller.VolumeSizeMultiple)) + assert.Equal(t, res, int64(MinVolumeSizeBytes+VolumeSizeMultiple)) }) }) t.Run("nfs creator", func(t *testing.T) { - nc := &controller.NfsCreator{} + nc := &NfsCreator{} t.Run("zeroes", func(t *testing.T) { cr := &csi.CapacityRange{ RequiredBytes: 0, @@ -67,25 +67,25 @@ func TestVolumeCreator_CheckSize(t *testing.T) { res, err := nc.CheckSize(context.Background(), cr, false) assert.NoError(t, err) - assert.Equal(t, res, int64(controller.MinVolumeSizeBytes)) + assert.Equal(t, res, int64(MinVolumeSizeBytes)) }) t.Run("mod != 0", func(t *testing.T) { cr := &csi.CapacityRange{ - RequiredBytes: controller.MinVolumeSizeBytes + 1, + RequiredBytes: MinVolumeSizeBytes + 1, LimitBytes: 0, } res, err := nc.CheckSize(context.Background(), cr, false) assert.NoError(t, err) - assert.Equal(t, res, int64(controller.MinVolumeSizeBytes+controller.VolumeSizeMultiple)) + assert.Equal(t, res, int64(MinVolumeSizeBytes+VolumeSizeMultiple)) }) }) } func TestVolumeCreator_CheckIfAlreadyExists(t *testing.T) { t.Run("can't find volume [block]", func(t *testing.T) { - sc := &controller.SCSICreator{} + sc := &SCSICreator{} name := "test" clientMock := new(mocks.Client) clientMock.On("GetVolumeByName", context.Background(), name). @@ -97,7 +97,7 @@ func TestVolumeCreator_CheckIfAlreadyExists(t *testing.T) { }) t.Run("can't find volume [nfs]", func(t *testing.T) { - nc := &controller.NfsCreator{} + nc := &NfsCreator{} name := "test" clientMock := new(mocks.Client) clientMock.On("GetFSByName", context.Background(), name). @@ -107,11 +107,38 @@ func TestVolumeCreator_CheckIfAlreadyExists(t *testing.T) { assert.Error(t, err) assert.Contains(t, err.Error(), "can't find filesystem") }) + + t.Run("volume already exists [nfs]", func(t *testing.T) { + nc := &NfsCreator{} + name := "test" + sizeInBytes := int64(1610612736) + validNodeID = strings.Join([]string{validHostName, "127.0.0.1"}, "-") + clientMock := new(mocks.Client) + clientMock.On("GetFSByName", context.Background(), name). + Return(gopowerstore.FileSystem{SizeTotal: 3221225472}, nil) + + clientMock.On("GetNAS", context.Background(), mock.Anything).Return(gopowerstore.NAS{CurrentNodeID: validNodeID}, nil) + vol, err := nc.CheckIfAlreadyExists(context.Background(), name, sizeInBytes, clientMock) + assert.NoError(t, err) + assert.Equal(t, sizeInBytes, vol.CapacityBytes) + }) + t.Run("volume with same name exists, but is wrong size [nfs]", func(t *testing.T) { + nc := &NfsCreator{} + name := "test" + sizeInBytes := int64(3221225472) + clientMock := new(mocks.Client) + clientMock.On("GetFSByName", context.Background(), name). + Return(gopowerstore.FileSystem{SizeTotal: 1610612736}, nil) + + _, err := nc.CheckIfAlreadyExists(context.Background(), name, sizeInBytes, clientMock) + assert.Error(t, err) + assert.Contains(t, err.Error(), "already exists but is incompatible volume size") + }) } func TestVolumeCreator_Clone(t *testing.T) { t.Run("scsi creator", func(t *testing.T) { - sc := &controller.SCSICreator{} + sc := &SCSICreator{} name := "test" t.Run("failed to lookup volume", func(t *testing.T) { clientMock := new(mocks.Client) @@ -158,7 +185,7 @@ func TestVolumeCreator_Clone(t *testing.T) { }) t.Run("nfs creator", func(t *testing.T) { - nc := &controller.NfsCreator{} + nc := &NfsCreator{} name := "test" t.Run("failed to lookup filesystem", func(t *testing.T) { clientMock := new(mocks.Client) @@ -191,7 +218,7 @@ func TestVolumeCreator_Clone(t *testing.T) { clientMock.On("GetFS", context.Background(), validBaseVolID). Return(gopowerstore.FileSystem{ Name: name, - SizeTotal: validVolSize + controller.ReservedSize, + SizeTotal: validVolSize + ReservedSize, ID: validBaseVolID, }, nil) clientMock.On("CloneFS", context.Background(), mock.Anything, validBaseVolID). @@ -207,7 +234,7 @@ func TestVolumeCreator_Clone(t *testing.T) { func TestVolumeCreator_CreateFromSnapshot(t *testing.T) { t.Run("scsi creator", func(t *testing.T) { - sc := &controller.SCSICreator{} + sc := &SCSICreator{} name := "test" t.Run("failed to lookup volume", func(t *testing.T) { clientMock := new(mocks.Client) @@ -257,7 +284,7 @@ func TestVolumeCreator_CreateFromSnapshot(t *testing.T) { }) t.Run("nfs creator", func(t *testing.T) { - nc := &controller.NfsCreator{} + nc := &NfsCreator{} name := "test" t.Run("failed to lookup filesystem snapshot", func(t *testing.T) { clientMock := new(mocks.Client) @@ -292,7 +319,7 @@ func TestVolumeCreator_CreateFromSnapshot(t *testing.T) { clientMock.On("GetFS", context.Background(), validBaseVolID). Return(gopowerstore.FileSystem{ Name: name, - SizeTotal: validVolSize + controller.ReservedSize, + SizeTotal: validVolSize + ReservedSize, ID: validBaseVolID, }, nil) clientMock.On("CreateFsFromSnapshot", context.Background(), mock.Anything, validBaseVolID). diff --git a/pkg/controller/csi_extension_server.go b/pkg/controller/csi_extension_server.go index d9e3a3b4..592538c7 100644 --- a/pkg/controller/csi_extension_server.go +++ b/pkg/controller/csi_extension_server.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2022-2023 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2022-2025 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,15 +20,15 @@ import ( "context" "fmt" "strings" + "sync" "time" "github.com/dell/csi-powerstore/v2/pkg/array" - "github.com/dell/csi-powerstore/v2/pkg/common" + "github.com/dell/csi-powerstore/v2/pkg/identifiers" podmon "github.com/dell/dell-csi-extensions/podmon" vgsext "github.com/dell/dell-csi-extensions/volumeGroupSnapshot" "github.com/dell/gopowerstore" "github.com/go-openapi/strfmt" - log "github.com/sirupsen/logrus" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -38,6 +38,7 @@ const StateReady = "Ready" // CreateVolumeGroupSnapshot creates volume group snapshot func (s *Service) CreateVolumeGroupSnapshot(ctx context.Context, request *vgsext.CreateVolumeGroupSnapshotRequest) (*vgsext.CreateVolumeGroupSnapshotResponse, error) { + log := log.WithContext(ctx) log.Infof("CreateVolumeGroupSnapshot called with req: %v", request) err := validateCreateVGSreq(request) @@ -67,7 +68,7 @@ func (s *Service) CreateVolumeGroupSnapshot(ctx context.Context, request *vgsext vgParams := gopowerstore.VolumeGroupCreate{ Name: request.GetName(), Description: request.GetDescription(), - VolumeIds: sourceVols, + VolumeIDs: sourceVols, } gotVg, err := s.Arrays()[arr].GetClient().GetVolumeGroupByName(ctx, request.GetName()) @@ -82,14 +83,14 @@ func (s *Service) CreateVolumeGroupSnapshot(ctx context.Context, request *vgsext // taking the existing volume group to re-create existingVgID = gotVg.ID // add members to existing volume group before taking snapshot - _, err := s.Arrays()[arr].GetClient().AddMembersToVolumeGroup(ctx, &gopowerstore.VolumeGroupMembers{VolumeIds: sourceVols}, existingVgID) + _, err := s.Arrays()[arr].GetClient().AddMembersToVolumeGroup(ctx, &gopowerstore.VolumeGroupMembers{VolumeIDs: sourceVols}, existingVgID) if err != nil { if apiError, ok := err.(gopowerstore.APIError); !(ok && apiError.VolumeNameIsAlreadyUse()) { return nil, status.Errorf(codes.Internal, "Error adding volume group members: %s", err.Error()) } } } else { - r, err := s.Arrays()[arr].GetClient().GetVolumeGroupsByVolumeID(ctx, vgParams.VolumeIds[0]) + r, err := s.Arrays()[arr].GetClient().GetVolumeGroupsByVolumeID(ctx, vgParams.VolumeIDs[0]) if err != nil { if apiError, ok := err.(gopowerstore.APIError); !(ok && apiError.NotFound()) { return nil, status.Errorf(codes.Internal, "Error getting volume group by volume ID: %s", err.Error()) @@ -156,20 +157,17 @@ func (s *Service) CreateVolumeGroupSnapshot(ctx context.Context, request *vgsext func validateCreateVGSreq(request *vgsext.CreateVolumeGroupSnapshotRequest) error { if request.Name == "" { err := status.Error(codes.InvalidArgument, "CreateVolumeGroupSnapshotRequest needs Name to be set") - log.Errorf("Error from validateCreateVGSreq: %v ", err) return err } - //name must be less than 28 chars, because we name snapshots with -, and index can at most be 3 chars + // name must be less than 28 chars, because we name snapshots with -, and index can at most be 3 chars if len(request.Name) > 27 { err := status.Errorf(codes.InvalidArgument, "Requested name %s longer than 27 character max", request.Name) - log.Errorf("Error from validateCreateVGSreq: %v ", err) return err } if len(request.SourceVolumeIDs) == 0 { err := status.Errorf(codes.InvalidArgument, "Source volumes are not present") - log.Errorf("Error from validateCreateVGSreq: %v ", err) return err } @@ -178,7 +176,7 @@ func validateCreateVGSreq(request *vgsext.CreateVolumeGroupSnapshotRequest) erro // ValidateVolumeHostConnectivity menthod will be called by podmon sidecars to check host connectivity with array func (s *Service) ValidateVolumeHostConnectivity(ctx context.Context, req *podmon.ValidateVolumeHostConnectivityRequest) (*podmon.ValidateVolumeHostConnectivityResponse, error) { - // ctx, log, _ := GetRunIDLog(ctx) + log := log.WithContext(ctx) log.Infof("ValidateVolumeHostConnectivity called %+v", req) rep := &podmon.ValidateVolumeHostConnectivityResponse{ Messages: make([]string, 0), @@ -199,16 +197,25 @@ func (s *Service) ValidateVolumeHostConnectivity(ctx context.Context, req *podmo if globalID == "" { if len(req.GetVolumeIds()) == 0 { log.Info("neither globalId nor volumeID is present in request") - globalIDs[s.DefaultArray().GlobalID] = true + // need to put all arrays to check not only default array and not matched ID will be filtered later. + for _, array := range s.Arrays() { + globalIDs[array.GlobalID] = true + } } - // for loop req.GetVolumeIds() + for _, volID := range req.GetVolumeIds() { - _, globalID, _, err := array.ParseVolumeID(ctx, volID, s.DefaultArray(), nil) - if err != nil || globalID == "" { + volumeHandle, err := array.ParseVolumeID(ctx, volID, s.DefaultArray(), nil) + if err != nil || (volumeHandle.LocalArrayGlobalID == "" && volumeHandle.RemoteArrayGlobalID == "") { log.Errorf("unable to retrieve array's globalID after parsing volumeID") globalIDs[s.DefaultArray().GlobalID] = true } else { - globalIDs[globalID] = true + if volumeHandle.LocalArrayGlobalID != "" { + globalIDs[volumeHandle.LocalArrayGlobalID] = true + } + + if volumeHandle.RemoteArrayGlobalID != "" { + globalIDs[volumeHandle.RemoteArrayGlobalID] = true + } } } } else { @@ -217,61 +224,208 @@ func (s *Service) ValidateVolumeHostConnectivity(ctx context.Context, req *podmo // Go through each of the globalIDs for globalID := range globalIDs { + // Check if array is non-uniform and matches the node label + arr, err := s.GetOneArray(globalID) + if err != nil || arr == nil { + log.Errorf("failed to get array %s: %s", globalID, err.Error()) + return nil, err + } + + if !arr.CheckConnectivity(ctx, req.GetNodeId()) { + log.Warnf("Not a match for node %s on array %s, skipping connectivity check", req.GetNodeId(), globalID) + continue + } + // First - check if the array is visible from the node - err := s.checkIfNodeIsConnected(ctx, globalID, req.GetNodeId(), rep) + err = s.checkIfNodeIsConnected(ctx, globalID, req.GetNodeId(), rep) if err != nil { return rep, err } + } + // Check for IOinProgress only when volumes IDs are present in the request as the field is required only in the latter case also to reduce number of calls to the API making it efficient + if len(req.GetVolumeIds()) > 0 { + // Get array config + for _, volID := range req.GetVolumeIds() { + volume, err := array.ParseVolumeID(ctx, volID, s.DefaultArray(), nil) + if err != nil { + log.Errorf("failed to parse volumeID, %s, for querying IO metrics. err: %s", volID, err.Error()) + return nil, err + } - // Check for IOinProgress only when volumes IDs are present in the request as the field is required only in the latter case also to reduce number of calls to the API making it efficient - if len(req.GetVolumeIds()) > 0 { - // Get array config - for _, volID := range req.GetVolumeIds() { - id, globalIDForVol, protocol, _ := array.ParseVolumeID(ctx, volID, s.DefaultArray(), nil) - if globalIDForVol != globalID { - log.Errorf("Recived globalId from podman is %s and retrieved from array is %s ", globalID, globalIDForVol) - return nil, fmt.Errorf("invalid globalId %s is provided", globalID) - } - arraysConfig, err := s.GetOneArray(globalID) - if err != nil || arraysConfig == nil { - log.Error("Failed to get array config with error ", err.Error()) + localArray, err := s.GetOneArray(volume.LocalArrayGlobalID) + if err != nil || localArray == nil { + log.Errorf("failed to get local array configuration for array %s for volume activity validation: %s", + volume.LocalArrayGlobalID, err.Error()) + return nil, err + } + + // set to nil to avoid unnecessary API calls by subsequent iterations + var remoteArray *array.PowerStoreArray + if volume.RemoteArrayGlobalID != "" { + remoteArray, err = s.GetOneArray(volume.RemoteArrayGlobalID) + if err != nil { + log.Errorf("failed to get remote array configuration for array %s for volume activity validation: %s", + volume.RemoteArrayGlobalID, err.Error()) return nil, err } - // check if any IO is inProgress for the current globalID/array - err = s.IsIOInProgress(ctx, id, arraysConfig, protocol) - if err == nil { - rep.IosInProgress = true - return rep, nil - } } + + // This context is for the set of requests for the current volume. + // Used to cancel any pending requests before checking for IO on any + // subsequent volumes. + probeBase := context.WithoutCancel(ctx) // decouple from caller's cancel/deadline + ioCtx, ioCtxCancel := context.WithTimeout(probeBase, identifiers.PodmonArrayConnectivityTimeout) + + // channels for receiving responses from async requests + reqChs := make([]<-chan error, 0) + // check if any IO is inProgress for the current local globalID/array + reqLocalCh := asyncGetIOInProgress(ioCtx, volume.LocalUUID, *localArray, volume.Protocol) + reqChs = append(reqChs, reqLocalCh) + + if remoteArray != nil { + // check if any IO is inProgress for the current remote globalID/array + reqRemoteCh := asyncGetIOInProgress(ioCtx, volume.RemoteUUID, *remoteArray, volume.Protocol) + reqChs = append(reqChs, reqRemoteCh) + } + + if rep.IosInProgress = isIOInProgress(ioCtx, reqChs...); rep.IosInProgress { + // so long as at least one volume has IO in-progress + // we should report it. + // This status is effectively a logical OR of all the volumes + ioCtxCancel() + log.Infof("IO detected for volume %s", volID) + break + } + + // make sure to cancel any pending requests from this iteration + // so no goroutines are left running. + ioCtxCancel() } } + log.Infof("ValidateVolumeHostConnectivity reply %+v", rep) return rep, nil } +// waitAndClose waits for all goroutines to complete by waiting on the WaitGroup, wg, +// then closes the provided channel, ch. +func waitAndClose(wg *sync.WaitGroup, ch chan error) { + log.Debugf("waiting to IO in-progress queries to complete") + wg.Wait() + // close the channel to signal there are no more results + // to be processed and the receiver can move on + log.Debugf("all goroutines complete; closing the channel") + close(ch) +} + +// isIOInProgress listens for responses on channels returned by asyncGetIOInProgress using the +// fan-in concurrency pattern and returns true if at least one response is a nil error, +// denoting IO is in-progress. +func isIOInProgress(ctx context.Context, chs ...<-chan error) bool { + log := log.WithContext(ctx) + // single channel on which the channels in "chs" will write their results + errCh := make(chan error) + wg := &sync.WaitGroup{} + + // writes results from all channels to a single channel so the results can be + // received as they're made available + asyncReceiveWithCtx := func(ctx context.Context, errCh chan<- error, repCh <-chan error, wg *sync.WaitGroup) { + defer wg.Done() + for { + select { + case <-ctx.Done(): + errCh <- ctx.Err() + return + // read the channel until it is closed + case err, isOpen := <-repCh: + if !isOpen { + return + } + errCh <- err + } + } + } + + // Used to exit early if there is no IO in-progress + ioCtx, cancel := context.WithCancel(ctx) + defer cancel() + + wg.Add(len(chs)) + for _, ch := range chs { + go asyncReceiveWithCtx(ioCtx, errCh, ch, wg) + } + + // make sure to close errCh when all asyncReceiveWithCtx goroutines are done + // to signal to the range statement below that there are no more results. + go waitAndClose(wg, errCh) + + // Read results as they're ready. + // If the errCh channel is closed before a nil error is + // received, assume there is no IO in-progress. + for err := range errCh { + if err != nil { + log.Debugf("error received while validating volume connectivity: %s", err.Error()) + continue + } + + // cancel any remaining goroutines so we can report IO in-progress ASAP + // and we don't leave any goroutines blocking, trying to write to the channel. + cancel() + log.Info("IO in-progress detected while validating volume connectivity") + return true + } + + log.Info("no IO in-progress was detected while validating volume connectivity") + return false +} + +// asyncGetIOInProgress starts an async request to getIOInProgress and returns a channel +// on which the result can be received. +// It can be used to dispatch multiple requests in parallel for situations such as metro +// volumes where multiple volumes need to be checked for IO to determine if the volume is active. +func asyncGetIOInProgress(ctx context.Context, volID string, array array.PowerStoreArray, protocol string) <-chan error { + log := log.WithContext(ctx) + errCh := make(chan error) + go func() { + defer close(errCh) + log.Infof("checking if IO is in-progress for volume %s on array %s", volID, array.GlobalID) + + // This blocks until both functions have been evaluated, which can be slow. + // Only then can the select statement determine which case to execute. If context has + // been canceled when the function returns, don't try to write anything to the channel + // because there will likely be no listeners and the select will block forever + // if the channel is not read. + select { + case errCh <- getIOInProgress(ctx, volID, array, protocol): + case <-ctx.Done(): + log.Errorf("context deadline exceeded while querying for IOs in-progress for volume %s on array %s", volID, array.GlobalID) + } + }() + return errCh +} + // checkIfNodeIsConnected looks at the 'nodeId' to determine if there is connectivity to the 'arrayId' array. // The 'rep' object will be filled with the results of the check. func (s *Service) checkIfNodeIsConnected(ctx context.Context, arrayID string, nodeID string, rep *podmon.ValidateVolumeHostConnectivityResponse) error { - + log := log.WithContext(ctx) log.Infof("Checking if array %s is connected to node %s", arrayID, nodeID) var message string rep.Connected = false - nodeIP := common.GetIPListFromString(nodeID) + nodeIP := identifiers.GetIPListFromString(nodeID) if len(nodeIP) == 0 { log.Errorf("failed to parse node ID '%s'", nodeID) return fmt.Errorf("failed to parse node ID") } ip := nodeIP[len(nodeIP)-1] // form url to call array on node - url := "http://" + ip + common.APIPort + common.ArrayStatus + "/" + arrayID + url := "http://" + ip + identifiers.APIPort + identifiers.ArrayStatus + "/" + arrayID connected, err := s.QueryArrayStatus(ctx, url) if err != nil { message = fmt.Sprintf("connectivity unknown for array %s to node %s due to %s", arrayID, nodeID, err) log.Error(message) rep.Messages = append(rep.Messages, message) - log.Errorf(err.Error()) + log.Errorf("%s", err.Error()) } if connected { @@ -285,8 +439,10 @@ func (s *Service) checkIfNodeIsConnected(ctx context.Context, arrayID string, no return nil } -// IsIOInProgress function check the IO operation status on array -func (s *Service) IsIOInProgress(ctx context.Context, volID string, arrayConfig *array.PowerStoreArray, protocol string) (err error) { +// getIOInProgress attempts to determine if IO has recently occurred for a given volume, volID, +// and returns a nil error if IO has occurred. +func getIOInProgress(ctx context.Context, volID string, arrayConfig array.PowerStoreArray, protocol string) (err error) { + log := log.WithContext(ctx) // Call PerformanceMetricsByVolume or PerformanceMetricsByFileSystem in gopowerstore based on the volume type if protocol == "scsi" { resp, err := arrayConfig.Client.PerformanceMetricsByVolume(ctx, volID, gopowerstore.TwentySec) @@ -300,7 +456,7 @@ func (s *Service) IsIOInProgress(ctx context.Context, volID string, arrayConfig return nil } } - return fmt.Errorf("no IOInProgress") + return fmt.Errorf("no IOInProgress for volume %s on array %s", volID, arrayConfig.GlobalID) } // nfs volume type logic resp, err := arrayConfig.Client.PerformanceMetricsByFileSystem(ctx, volID, gopowerstore.TwentySec) @@ -314,7 +470,7 @@ func (s *Service) IsIOInProgress(ctx context.Context, volID string, arrayConfig return nil } } - return fmt.Errorf("no IOInProgress") + return fmt.Errorf("no IOInProgress for volume %s on array %s", volID, arrayConfig.GlobalID) } func checkIfEntryIsLatest(timestamp strfmt.DateTime) bool { diff --git a/pkg/controller/csi_extension_server_test.go b/pkg/controller/csi_extension_server_test.go index 41545ef8..ddb3928b 100644 --- a/pkg/controller/csi_extension_server_test.go +++ b/pkg/controller/csi_extension_server_test.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2022-2023 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2022-2024 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,129 +16,239 @@ * */ -package controller_test +package controller import ( "context" "encoding/json" + "errors" "fmt" "net/http" + "path/filepath" + "sync" + "testing" "time" - "github.com/dell/csi-powerstore/v2/pkg/common" + "github.com/dell/csi-powerstore/v2/pkg/array" + "github.com/dell/csi-powerstore/v2/pkg/identifiers" podmon "github.com/dell/dell-csi-extensions/podmon" vgsext "github.com/dell/dell-csi-extensions/volumeGroupSnapshot" "github.com/dell/gopowerstore" "github.com/dell/gopowerstore/api" + gopowerstoremock "github.com/dell/gopowerstore/mocks" "github.com/go-openapi/strfmt" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" + "github.com/google/uuid" + ginkgo "github.com/onsi/ginkgo" + gomega "github.com/onsi/gomega" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + k8score "k8s.io/api/core/v1" ) -const stateReady = "Ready" +const ( + stateReady = "Ready" +) + +var nodeConnectivityServer = struct { + port string + statusPath string +}{ + port: "9028", + statusPath: "/array-status", +} + +var arrayOneStatusEndpoint = filepath.Join(nodeConnectivityServer.statusPath, firstValidID) + +func getActiveIOVolumeMetrics() []gopowerstore.PerformanceMetricsByVolumeResponse { + volumeMetrics := make([]gopowerstore.PerformanceMetricsByVolumeResponse, 6) + freshTime, _ := strfmt.ParseDateTime(fmt.Sprint(time.Now().UTC().Format("2006-01-02T15:04:05Z"))) + volumeMetrics[0].TotalIops = 0.0 + volumeMetrics[0].WriteIops = 0.0 + volumeMetrics[0].ReadIops = 0.0 + volumeMetrics[1].TotalIops = 0.0 + volumeMetrics[1].WriteIops = 0.0 + volumeMetrics[1].ReadIops = 0.0 + volumeMetrics[2].TotalIops = 4.9 + volumeMetrics[2].WriteIops = 2.6 + volumeMetrics[2].CommonMetricsFields.Timestamp = freshTime + volumeMetrics[2].ReadIops = 2.3 + volumeMetrics[3].TotalIops = 0.0 + volumeMetrics[3].CommonMetricsFields.Timestamp = freshTime + volumeMetrics[4].TotalIops = 4.6 + volumeMetrics[4].CommonMetricsFields.Timestamp = freshTime + volumeMetrics[5].TotalIops = 0.0 + return volumeMetrics +} + +func getInactiveIOVolumeMetrics() []gopowerstore.PerformanceMetricsByVolumeResponse { + volumeMetrics := make([]gopowerstore.PerformanceMetricsByVolumeResponse, 6) + freshTime, _ := strfmt.ParseDateTime(fmt.Sprint(time.Now().UTC().Format("2006-01-02T15:04:05Z"))) + volumeMetrics[0].TotalIops = 0.0 + volumeMetrics[0].WriteIops = 0.0 + volumeMetrics[0].ReadIops = 0.0 + volumeMetrics[1].TotalIops = 0.0 + volumeMetrics[1].WriteIops = 0.0 + volumeMetrics[1].ReadIops = 0.0 + volumeMetrics[2].TotalIops = 0.0 + volumeMetrics[2].WriteIops = 0.0 + volumeMetrics[2].CommonMetricsFields.Timestamp = freshTime + volumeMetrics[2].ReadIops = 0.0 + volumeMetrics[3].TotalIops = 0.0 + volumeMetrics[3].CommonMetricsFields.Timestamp = freshTime + volumeMetrics[4].TotalIops = 0.0 + volumeMetrics[4].CommonMetricsFields.Timestamp = freshTime + volumeMetrics[5].TotalIops = 0.0 + return volumeMetrics +} + +func startNodeConnectivityCheckerServer(port string, endpoints ...string) { + identifiers.APIPort = ":" + port + var status identifiers.ArrayConnectivityStatus + status.LastAttempt = time.Now().Unix() + status.LastSuccess = time.Now().Unix() + input, _ := json.Marshal(status) + // responding with some dummy response that is for the case when array is connected and LastSuccess check was just finished + for _, endpoint := range endpoints { + http.HandleFunc(endpoint, func(w http.ResponseWriter, _ *http.Request) { + _, err := w.Write(input) + if err != nil { + fmt.Printf("error encountered when handling incoming request to mock node connectivity checker server: %s\n", err) + } + }) + } + + fmt.Printf("Starting server at port %s\n", port) -var _ = Describe("csi-extension-server", func() { - BeforeEach(func() { + go func() { + err := http.ListenAndServe(identifiers.APIPort, nil) // #nosec G114 + if err != nil { + fmt.Printf("error encountered serving mock node connectivity checker server: %s\n", err) + } + }() +} + +var _ = ginkgo.Describe("csi-extension-server", func() { + ginkgo.BeforeSuite(func() { + startNodeConnectivityCheckerServer(nodeConnectivityServer.port, arrayOneStatusEndpoint) + }) + + ginkgo.BeforeEach(func() { setVariables() }) - Describe("calling ValidateVolumeHostConnectivity()", func() { - When("checking if ValidateVolumeHostConnectivity is implemented ", func() { - It("should return a message that ValidateVolumeHostConnectivity is implemented", func() { + + ginkgo.Describe("calling ValidateVolumeHostConnectivity()", func() { + ginkgo.When("checking if ValidateVolumeHostConnectivity is implemented ", func() { + ginkgo.It("should return a message that ValidateVolumeHostConnectivity is implemented", func() { req := &podmon.ValidateVolumeHostConnectivityRequest{} res, err := ctrlSvc.ValidateVolumeHostConnectivity(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res.Messages[0]).To(Equal("ValidateVolumeHostConnectivity is implemented")) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res.Messages[0]).To(gomega.Equal("ValidateVolumeHostConnectivity is implemented")) }) }) - When("nodeId is not provided ", func() { - It("should return error", func() { - volId := []string{validBaseVolID} + ginkgo.When("the request contains a bad volumeID", func() { + ginkgo.It("should return an error", func() { req := &podmon.ValidateVolumeHostConnectivityRequest{ ArrayId: "default", - VolumeIds: volId, - NodeId: "", + VolumeIds: []string{"SOMETHING-WRONG"}, + NodeId: validNodeID, } - _, err := ctrlSvc.ValidateVolumeHostConnectivity(context.Background(), req) - Expect(err).ToNot(BeNil()) + clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{}, errors.New("error: bad volume name")) + clientMock.On("GetFS", mock.Anything, mock.Anything).Return(gopowerstore.FileSystem{}, errors.New("error: bad volume name")) + + res, err := ctrlSvc.ValidateVolumeHostConnectivity(context.Background(), req) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) }) }) - When("array status is not fetched so server will not respond ", func() { - It("should return error", func() { - volId := []string{validBaseVolID} + ginkgo.When("the request contains a volumeID with an invalid local arrayID", func() { + ginkgo.It("should return an error", func() { req := &podmon.ValidateVolumeHostConnectivityRequest{ ArrayId: "default", - VolumeIds: volId, - NodeId: "csi-node-003c684ccb0c4ca0a9c99423563dfd2c-127.0.0.1", + VolumeIds: []string{invalidBlockVolumeID}, + NodeId: validNodeID, } - clientMock.On("GetVolume", context.Background(), mock.Anything).Return(gopowerstore.Volume{ApplianceID: validApplianceID}, nil) - _, err := ctrlSvc.ValidateVolumeHostConnectivity(context.Background(), req) - Expect(err).ToNot(BeNil()) + + res, err := ctrlSvc.ValidateVolumeHostConnectivity(context.Background(), req) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) }) }) - When("neither arrayId nor volId is present in the request body ", func() { - It("should not return error", func() { + ginkgo.When("the request contains a metro volumeID with an invalid remote arrayID", func() { + ginkgo.It("should return an error", func() { + req := &podmon.ValidateVolumeHostConnectivityRequest{ + ArrayId: "default", + VolumeIds: []string{invalidMetroBlockVolumeID}, + NodeId: validNodeID, + } + + res, err := ctrlSvc.ValidateVolumeHostConnectivity(context.Background(), req) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + }) + }) + ginkgo.When("nodeId is not provided ", func() { + ginkgo.It("should return error", func() { + volID := []string{validLegacyVolID} + req := &podmon.ValidateVolumeHostConnectivityRequest{ + ArrayId: "default", + VolumeIds: volID, + NodeId: "", + } + _, err := ctrlSvc.ValidateVolumeHostConnectivity(context.Background(), req) + gomega.Expect(err).ToNot(gomega.BeNil()) + }) + }) + + ginkgo.When("neither arrayId nor volId is present in the request body ", func() { + ginkgo.It("should not return error", func() { req := &podmon.ValidateVolumeHostConnectivityRequest{ NodeId: "csi-node-003c684ccb0c4ca0a9c99423563dfd2c-127.0.0.1", } clientMock.On("GetVolume", context.Background(), mock.Anything).Return(gopowerstore.Volume{ApplianceID: validApplianceID}, nil) _, err := ctrlSvc.ValidateVolumeHostConnectivity(context.Background(), req) - Expect(err).To(BeNil()) + gomega.Expect(err).To(gomega.BeNil()) }) }) - When("Invalid nodeID is sent in the request body ", func() { - It("should return error", func() { - + ginkgo.When("Invalid nodeID is sent in the request body ", func() { + ginkgo.It("should return error", func() { req := &podmon.ValidateVolumeHostConnectivityRequest{ NodeId: "csi-node-003c684ccb0c4ca0a9c99423563dfd2c-@@@", } clientMock.On("GetVolume", context.Background(), mock.Anything).Return(gopowerstore.Volume{ApplianceID: validApplianceID}, nil) _, err := ctrlSvc.ValidateVolumeHostConnectivity(context.Background(), req) - Expect(err).ToNot(BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) }) }) - When("not sending arrayId in request body ", func() { - It("should not return error but IO in response should be false", func() { + ginkgo.When("the request has a volume ID but no array ID and no IO is in progress", func() { + ginkgo.It("should return IO in-progress as false", func() { clientMock.On("GetVolume", context.Background(), mock.Anything).Return(gopowerstore.Volume{ApplianceID: validApplianceID}, nil) var resp []gopowerstore.PerformanceMetricsByVolumeResponse - clientMock.On("PerformanceMetricsByVolume", context.Background(), mock.Anything, mock.Anything). + clientMock.On("PerformanceMetricsByVolume", mock.Anything, mock.Anything, mock.Anything). Return(resp, gopowerstore.APIError{ ErrorMsg: &api.ErrorMsg{ StatusCode: http.StatusInternalServerError, }, }) - volId := []string{validBaseVolID} + volID := []string{validLegacyVolID} req := &podmon.ValidateVolumeHostConnectivityRequest{ - VolumeIds: volId, + VolumeIds: volID, NodeId: "csi-node-003c684ccb0c4ca0a9c99423563dfd2c-127.0.0.1", } - common.APIPort = ":9028" - var status common.ArrayConnectivityStatus - status.LastAttempt = time.Now().Unix() - status.LastSuccess = time.Now().Unix() - input, _ := json.Marshal(status) - // responding with some dummy response that is for the case when array is connected and LastSuccess check was just finished - http.HandleFunc("/array-status/globalvolid1", func(w http.ResponseWriter, r *http.Request) { - w.Write(input) - }) - - fmt.Printf("Starting server at port 9028\n") - go http.ListenAndServe(":9028", nil) response, err := ctrlSvc.ValidateVolumeHostConnectivity(context.Background(), req) - Expect(err).To(BeNil()) - Expect(response.IosInProgress).To(BeFalse()) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(response.IosInProgress).To(gomega.BeFalse()) }) }) - When("not sending arrayId in request body and default array is connected well and IO operation is also there ", func() { - It("should not return error", func() { + ginkgo.When("not sending arrayId in request body and default array is connected well and IO operation is also there ", func() { + ginkgo.It("should return IO in-progress", func() { clientMock.On("GetVolume", context.Background(), mock.Anything).Return(gopowerstore.Volume{ApplianceID: validApplianceID}, nil) resp2 := make([]gopowerstore.PerformanceMetricsByVolumeResponse, 6) freshTime, _ := strfmt.ParseDateTime(fmt.Sprint(time.Now().UTC().Format("2006-01-02T15:04:05Z"))) @@ -157,24 +267,164 @@ var _ = Describe("csi-extension-server", func() { resp2[4].TotalIops = 4.6 resp2[4].CommonMetricsFields.Timestamp = freshTime resp2[5].TotalIops = 0.0 - clientMock.On("PerformanceMetricsByVolume", context.Background(), mock.Anything, mock.Anything). + clientMock.On("PerformanceMetricsByVolume", mock.Anything, mock.Anything, mock.Anything). Return(resp2, nil) - volId2 := []string{validBaseVolID} + volID2 := []string{validLegacyVolID} req2 := &podmon.ValidateVolumeHostConnectivityRequest{ - VolumeIds: volId2, + VolumeIds: volID2, NodeId: "csi-node-003c684ccb0c4ca0a9c99423563dfd2c-127.0.0.1", } response, err := ctrlSvc.ValidateVolumeHostConnectivity(context.Background(), req2) - Expect(err).To(BeNil()) - Expect(response.IosInProgress).To(BeTrue()) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(response.IosInProgress).To(gomega.BeTrue()) }) }) + + ginkgo.When("the preferred array of a metro volume is disconnected, but the non-preferred is connected", func() { + ginkgo.It("should report IO is in-progress", func() { + // preferred side will have no IO in-progress + metroMetricsPreferred := getInactiveIOVolumeMetrics() + metroMetricsNonPreferred := getActiveIOVolumeMetrics() + + clientMock.On("PerformanceMetricsByVolume", mock.Anything, validBaseVolID, mock.Anything).Times(1). + Return(metroMetricsPreferred, nil) + clientMock.On("PerformanceMetricsByVolume", mock.Anything, validRemoteVolID, mock.Anything).Times(1). + Return(metroMetricsNonPreferred, nil) + + req := &podmon.ValidateVolumeHostConnectivityRequest{ + VolumeIds: []string{validMetroBlockVolumeID}, + NodeId: validNodeID, + } + + response, err := ctrlSvc.ValidateVolumeHostConnectivity(context.Background(), req) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(response.IosInProgress).To(gomega.BeTrue()) + }) + }) + + ginkgo.When("the both arrays of a metro volume are disconnected", func() { + ginkgo.It("should report IO is not in-progress", func() { + // preferred side will have no IO in-progress + metroMetricsPreferred := getInactiveIOVolumeMetrics() + metroMetricsNonPreferred := getInactiveIOVolumeMetrics() + + clientMock.On("PerformanceMetricsByVolume", mock.Anything, validBaseVolID, mock.Anything).Times(1). + Return(metroMetricsPreferred, nil) + clientMock.On("PerformanceMetricsByVolume", mock.Anything, validRemoteVolID, mock.Anything).Times(1). + Return(metroMetricsNonPreferred, nil) + + req := &podmon.ValidateVolumeHostConnectivityRequest{ + VolumeIds: []string{validMetroBlockVolumeID}, + NodeId: validNodeID, + } + + response, err := ctrlSvc.ValidateVolumeHostConnectivity(context.Background(), req) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(response.IosInProgress).To(gomega.BeFalse()) + }) + }) + + ginkgo.When("context times out for both arrays of a metro volume", func() { + ginkgo.It("should report IO is not in-progress", func() { + clientMock.On("PerformanceMetricsByVolume", mock.Anything, validBaseVolID, mock.Anything).After(time.Second*11).Times(1). + Return(nil, errors.New("a long delay occurred")) + clientMock.On("PerformanceMetricsByVolume", mock.Anything, validRemoteVolID, mock.Anything).Times(1). + Return(nil, errors.New("a long delay occurred")) + + req := &podmon.ValidateVolumeHostConnectivityRequest{ + VolumeIds: []string{validMetroBlockVolumeID}, + NodeId: validNodeID, + } + + // create a context with a deadline that's already expired + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*4)) + defer cancel() + + response, err := ctrlSvc.ValidateVolumeHostConnectivity(ctx, req) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(response.IosInProgress).To(gomega.BeFalse()) + }) + }) + + ginkgo.When("at least one volume has IO in-progress", func() { + ginkgo.It("should report IO is in-progress", func() { + activeVolumeMetrics := getActiveIOVolumeMetrics() + inactiveVolumeMetrics := getInactiveIOVolumeMetrics() + + // Return at least one volume with IO in-progress + clientMock.On("PerformanceMetricsByVolume", mock.Anything, validBaseVolID, mock.Anything).Times(1). + Return(activeVolumeMetrics, nil) + clientMock.On("PerformanceMetricsByVolume", mock.Anything, validRemoteVolID, mock.Anything).Times(1). + Return(inactiveVolumeMetrics, nil) + + testVolUUID := uuid.New() + testVolID := filepath.Join(testVolUUID.String(), firstValidID, "scsi") + clientMock.On("PerformanceMetricsByVolume", mock.Anything, testVolUUID.String(), mock.Anything).Times(1). + Return(inactiveVolumeMetrics, nil) + + req := &podmon.ValidateVolumeHostConnectivityRequest{ + // create a request that checks more than one volume + VolumeIds: []string{testVolID, validMetroBlockVolumeID}, + NodeId: validNodeID, + } + + response, err := ctrlSvc.ValidateVolumeHostConnectivity(context.Background(), req) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(response.IosInProgress).To(gomega.BeTrue()) + }) + }) + + ginkgo.It("unable to parse volume ID - defaults to array - volumeHandle is empty", func() { + req := &podmon.ValidateVolumeHostConnectivityRequest{ + ArrayId: "", + VolumeIds: []string{""}, + NodeId: validNodeID, + } + + res, err := ctrlSvc.ValidateVolumeHostConnectivity(context.Background(), req) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + }) + + ginkgo.It("No matching host connectivity found and no volume ID in request - empty response", func() { + // Set the host connectivity for the first array when the node does not match for labels. + arr := ctrlSvc.Arrays() + gomega.Expect(arr).ToNot(gomega.BeNil()) + + arr[firstValidID].HostConnectivity = &array.HostConnectivity{ + Metro: array.MetroConnectivityOptions{ + ColocatedLocal: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "zone1", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"local"}, + }, + }, + }, + }, + }, + }, + } + ctrlSvc.SetArrays(arr) + + req := &podmon.ValidateVolumeHostConnectivityRequest{ + ArrayId: firstValidID, + NodeId: validNodeID, + } + + res, err := ctrlSvc.ValidateVolumeHostConnectivity(context.Background(), req) + gomega.Expect(res).ToNot(gomega.BeNil()) + gomega.Expect(err).To(gomega.BeNil()) + }) }) - Describe("calling IsIOInProgress and QueryArrayStatus", func() { - When("IOConnectivity for scsi type volume on array", func() { - It("should not fail", func() { + ginkgo.Describe("calling IsIOInProgress and QueryArrayStatus", func() { + ginkgo.When("IOConnectivity for scsi type volume on array", func() { + ginkgo.It("should not fail", func() { var resp []gopowerstore.PerformanceMetricsByVolumeResponse clientMock.On("PerformanceMetricsByVolume", context.Background(), mock.Anything, mock.Anything). Return(resp, gopowerstore.APIError{ @@ -182,13 +432,13 @@ var _ = Describe("csi-extension-server", func() { StatusCode: http.StatusInternalServerError, }, }) - err := ctrlSvc.IsIOInProgress(context.Background(), validBlockVolumeID, ctrlSvc.DefaultArray(), "scsi") - Expect(err).ToNot(BeNil()) + err := getIOInProgress(context.Background(), validBlockVolumeID, *ctrlSvc.DefaultArray(), "scsi") + gomega.Expect(err).ToNot(gomega.BeNil()) }) }) - When("IOConnectivity for nfs type volume on array", func() { - It("should not fail", func() { + ginkgo.When("IOConnectivity for nfs type volume on array", func() { + ginkgo.It("should not fail", func() { var resp []gopowerstore.PerformanceMetricsByFileSystemResponse clientMock.On("PerformanceMetricsByFileSystem", context.Background(), mock.Anything, mock.Anything). Return(resp, gopowerstore.APIError{ @@ -196,13 +446,13 @@ var _ = Describe("csi-extension-server", func() { StatusCode: http.StatusInternalServerError, }, }) - err := ctrlSvc.IsIOInProgress(context.Background(), validBlockVolumeID, ctrlSvc.DefaultArray(), "nfs") - Expect(err).ToNot(BeNil()) + err := getIOInProgress(context.Background(), validBlockVolumeID, *ctrlSvc.DefaultArray(), "nfs") + gomega.Expect(err).ToNot(gomega.BeNil()) }) }) - When("IOConnectivity for scsi type volume on array when IO operation is not there", func() { - It("should not fail", func() { + ginkgo.When("IOConnectivity for scsi type volume on array when IO operation is not there", func() { + ginkgo.It("should not fail", func() { resp := make([]gopowerstore.PerformanceMetricsByVolumeResponse, 6) resp[0].TotalIops = 0.0 resp[1].TotalIops = 0.0 @@ -212,13 +462,13 @@ var _ = Describe("csi-extension-server", func() { resp[5].TotalIops = 0.0 clientMock.On("PerformanceMetricsByVolume", context.Background(), mock.Anything, mock.Anything). Return(resp, nil) - err := ctrlSvc.IsIOInProgress(context.Background(), validBlockVolumeID, ctrlSvc.DefaultArray(), "scsi") - Expect(err).ToNot(BeNil()) + err := getIOInProgress(context.Background(), validBlockVolumeID, *ctrlSvc.DefaultArray(), "scsi") + gomega.Expect(err).ToNot(gomega.BeNil()) }) }) - When("IOConnectivity for scsi type volume on array when IO operation is there", func() { - It("should not fail", func() { + ginkgo.When("IOConnectivity for scsi type volume on array when IO operation is there", func() { + ginkgo.It("should not fail", func() { resp := make([]gopowerstore.PerformanceMetricsByVolumeResponse, 6) freshTime, _ := strfmt.ParseDateTime(fmt.Sprint(time.Now().UTC().Format("2006-01-02T15:04:05Z"))) resp[0].TotalIops = 0.0 @@ -231,13 +481,13 @@ var _ = Describe("csi-extension-server", func() { resp[5].TotalIops = 0.0 clientMock.On("PerformanceMetricsByVolume", context.Background(), mock.Anything, mock.Anything). Return(resp, nil) - err := ctrlSvc.IsIOInProgress(context.Background(), validBlockVolumeID, ctrlSvc.DefaultArray(), "scsi") - Expect(err).To(BeNil()) + err := getIOInProgress(context.Background(), validBlockVolumeID, *ctrlSvc.DefaultArray(), "scsi") + gomega.Expect(err).To(gomega.BeNil()) }) }) - When("IOConnectivity for scsi type volume on array when IO operation is there but entry is not fresh", func() { - It("should fail", func() { + ginkgo.When("IOConnectivity for scsi type volume on array when IO operation is there but entry is not fresh", func() { + ginkgo.It("should fail", func() { resp := make([]gopowerstore.PerformanceMetricsByVolumeResponse, 6) // stale time staleTime, _ := strfmt.ParseDateTime(fmt.Sprint(time.Now().Add(time.Duration(-600) * time.Minute).Format("2006-01-02T15:04:05Z"))) @@ -251,13 +501,13 @@ var _ = Describe("csi-extension-server", func() { resp[5].TotalIops = 0.0 clientMock.On("PerformanceMetricsByVolume", context.Background(), mock.Anything, mock.Anything). Return(resp, nil) - err := ctrlSvc.IsIOInProgress(context.Background(), validBlockVolumeID, ctrlSvc.DefaultArray(), "scsi") - Expect(err).ToNot(BeNil()) + err := getIOInProgress(context.Background(), validBlockVolumeID, *ctrlSvc.DefaultArray(), "scsi") + gomega.Expect(err).ToNot(gomega.BeNil()) }) }) - When("IOConnectivity for nfs type volume on array when IO operation is not there", func() { - It("should not fail", func() { + ginkgo.When("IOConnectivity for nfs type volume on array when IO operation is not there", func() { + ginkgo.It("should not fail", func() { resp := make([]gopowerstore.PerformanceMetricsByFileSystemResponse, 6) resp[0].TotalIops = 0.0 resp[1].TotalIops = 0.0 @@ -267,13 +517,13 @@ var _ = Describe("csi-extension-server", func() { resp[5].TotalIops = 0.0 clientMock.On("PerformanceMetricsByFileSystem", context.Background(), mock.Anything, mock.Anything). Return(resp, nil) - err := ctrlSvc.IsIOInProgress(context.Background(), validBlockVolumeID, ctrlSvc.DefaultArray(), "nfs") - Expect(err).ToNot(BeNil()) + err := getIOInProgress(context.Background(), validBlockVolumeID, *ctrlSvc.DefaultArray(), "nfs") + gomega.Expect(err).ToNot(gomega.BeNil()) }) }) - When("IOConnectivity for nfs type volume on array when IO operation is there", func() { - It("should not fail", func() { + ginkgo.When("IOConnectivity for nfs type volume on array when IO operation is there", func() { + ginkgo.It("should not fail", func() { resp := make([]gopowerstore.PerformanceMetricsByFileSystemResponse, 6) freshTime, _ := strfmt.ParseDateTime(fmt.Sprint(time.Now().UTC().Format("2006-01-02T15:04:05Z"))) resp[0].TotalIops = 0.0 @@ -286,24 +536,24 @@ var _ = Describe("csi-extension-server", func() { resp[5].TotalIops = 0.0 clientMock.On("PerformanceMetricsByFileSystem", context.Background(), mock.Anything, mock.Anything). Return(resp, nil) - err := ctrlSvc.IsIOInProgress(context.Background(), validBlockVolumeID, ctrlSvc.DefaultArray(), "nfs") - Expect(err).To(BeNil()) + err := getIOInProgress(context.Background(), validBlockVolumeID, *ctrlSvc.DefaultArray(), "nfs") + gomega.Expect(err).To(gomega.BeNil()) }) }) - When("API call to the specified url to retrieve connection status for the array that is connected", func() { - It("should not fail", func() { - common.SetAPIPort(context.Background()) - var status common.ArrayConnectivityStatus + ginkgo.When("API call to the specified url to retrieve connection status for the array that is connected", func() { + ginkgo.It("should not fail", func() { + identifiers.SetAPIPort(context.Background()) + var status identifiers.ArrayConnectivityStatus status.LastAttempt = time.Now().Unix() status.LastSuccess = time.Now().Unix() input, _ := json.Marshal(status) // responding with some dummy response that is for the case when array is connected and LastSuccess check was just finished - http.HandleFunc("/array/id1", func(w http.ResponseWriter, r *http.Request) { + http.HandleFunc("/array/id1", func(w http.ResponseWriter, _ *http.Request) { w.Write(input) }) - server := &http.Server{Addr: ":49154"} + server := &http.Server{Addr: ":49154"} // #nosec G112 fmt.Printf("Starting server at port 49154 \n") go func() { err := server.ListenAndServe() @@ -312,25 +562,25 @@ var _ = Describe("csi-extension-server", func() { } }() check, err := ctrlSvc.QueryArrayStatus(context.Background(), "http://localhost:49154/array/id1") - Expect(err).To(BeNil()) - Expect(check).ToNot(BeFalse()) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(check).ToNot(gomega.BeFalse()) server.Shutdown(context.Background()) }) }) - When("API call to the specified url to retrieve connection status for the array that is not connected", func() { - It("should not fail", func() { - common.SetAPIPort(context.Background()) - var status common.ArrayConnectivityStatus + ginkgo.When("API call to the specified url to retrieve connection status for the array that is not connected", func() { + ginkgo.It("should not fail", func() { + identifiers.SetAPIPort(context.Background()) + var status identifiers.ArrayConnectivityStatus status.LastAttempt = time.Now().Unix() status.LastSuccess = time.Now().Unix() - 100 input, _ := json.Marshal(status) // responding with some dummy response that is for the case when array is connected and LastSuccess check was just finished - http.HandleFunc("/array/id2", func(w http.ResponseWriter, r *http.Request) { + http.HandleFunc("/array/id2", func(w http.ResponseWriter, _ *http.Request) { w.Write(input) }) - server := &http.Server{Addr: ":49153"} + server := &http.Server{Addr: ":49153"} // #nosec G112 fmt.Printf("Starting server at port 49153 \n") go func() { err := server.ListenAndServe() @@ -339,28 +589,28 @@ var _ = Describe("csi-extension-server", func() { } }() check, err := ctrlSvc.QueryArrayStatus(context.Background(), "http://localhost:49153/array/id2") - Expect(err).To(BeNil()) - Expect(check).ToNot(BeTrue()) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(check).ToNot(gomega.BeTrue()) server.Shutdown(context.Background()) }) }) - When("API call to the specified url to retrieve connection status for the array with diff diff error conditions", func() { - It("should not fail", func() { - common.SetAPIPort(context.Background()) - var status common.ArrayConnectivityStatus + ginkgo.When("API call to the specified url to retrieve connection status for the array with diff diff error conditions", func() { + ginkgo.It("should not fail", func() { + identifiers.SetAPIPort(context.Background()) + var status identifiers.ArrayConnectivityStatus status.LastAttempt = time.Now().Unix() - 200 status.LastSuccess = time.Now().Unix() - 200 input, _ := json.Marshal(status) // Responding with a dummy response for the case when the array check was done a while ago - http.HandleFunc("/array/id3", func(w http.ResponseWriter, r *http.Request) { + http.HandleFunc("/array/id3", func(w http.ResponseWriter, _ *http.Request) { w.Write(input) }) - http.HandleFunc("/array/id4", func(w http.ResponseWriter, r *http.Request) { + http.HandleFunc("/array/id4", func(w http.ResponseWriter, _ *http.Request) { w.Write([]byte("invalid type response")) }) - server := &http.Server{Addr: ":49152"} + server := &http.Server{Addr: ":49152"} // #nosec G112 fmt.Printf("Starting server at port 49152 \n") go func() { err := server.ListenAndServe() @@ -369,28 +619,24 @@ var _ = Describe("csi-extension-server", func() { } }() check, err := ctrlSvc.QueryArrayStatus(context.Background(), "http://localhost:49152/array/id3") - Expect(err).To(BeNil()) - Expect(check).ToNot(BeTrue()) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(check).ToNot(gomega.BeTrue()) check, err = ctrlSvc.QueryArrayStatus(context.Background(), "http://localhost:49152/array/id4") - Expect(err).ToNot(BeNil()) - Expect(check).ToNot(BeTrue()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(check).ToNot(gomega.BeTrue()) check, err = ctrlSvc.QueryArrayStatus(context.Background(), "http://localhost:49152/array/id5") - Expect(err).ToNot(BeNil()) - Expect(check).ToNot(BeTrue()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(check).ToNot(gomega.BeTrue()) server.Shutdown(context.Background()) }) }) }) - Describe("calling CreateVolumeGroupSnapshot()", func() { - When("valid member volumes are present", func() { - It("should create volume group snapshot successfully", func() { - clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID). - Return(gopowerstore.VolumeGroups{VolumeGroup: []gopowerstore.VolumeGroup{{ID: validGroupID, ProtectionPolicyID: validPolicyID}}}, nil) - clientMock.On("CreateVolumeGroupSnapshot", mock.Anything, validGroupID, mock.Anything). - Return(gopowerstore.CreateResponse{ID: validGroupID}, nil) + ginkgo.Describe("calling CreateVolumeGroupSnapshot()", func() { + ginkgo.When("should create volume group snapshot successfully", func() { + ginkgo.It("valid member volumes are present", func() { clientMock.On("GetVolumeGroupByName", mock.Anything, validGroupName). Return(gopowerstore.VolumeGroup{ID: validGroupID, ProtectionPolicyID: validPolicyID}, nil) clientMock.On("AddMembersToVolumeGroup", @@ -398,6 +644,8 @@ var _ = Describe("csi-extension-server", func() { mock.AnythingOfType("*gopowerstore.VolumeGroupMembers"), validGroupID). Return(gopowerstore.EmptyResponse(""), nil) + clientMock.On("CreateVolumeGroupSnapshot", mock.Anything, validGroupID, mock.Anything). + Return(gopowerstore.CreateResponse{ID: validGroupID}, nil) clientMock.On("GetVolumeGroup", mock.Anything, validGroupID). Return(gopowerstore.VolumeGroup{ ID: validGroupID, @@ -413,24 +661,23 @@ var _ = Describe("csi-extension-server", func() { } res, err := ctrlSvc.CreateVolumeGroupSnapshot(context.Background(), &req) - Expect(err).To(BeNil()) - Expect(res.SnapshotGroupID).To(Equal(validGroupID)) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res.SnapshotGroupID).To(gomega.Equal(validGroupID)) }) - }) - When("there is no existing volume group created", func() { - It("should create volume group and snapshot successfully", func() { + ginkgo.It("there is no existing volume group", func() { + clientMock.On("GetVolumeGroupByName", mock.Anything, validGroupName). + Return(gopowerstore.VolumeGroup{}, nil) clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID). Return(gopowerstore.VolumeGroups{}, nil) - clientMock.On("GetVolumeGroupByName", mock.Anything, validGroupName). - Return(gopowerstore.VolumeGroup{ID: validGroupID, ProtectionPolicyID: validPolicyID}, nil) + createGroupRequest := &gopowerstore.VolumeGroupCreate{ + Name: validGroupName, + VolumeIDs: []string{validBaseVolID}, + } + clientMock.On("CreateVolumeGroup", mock.Anything, createGroupRequest). + Return(gopowerstore.CreateResponse{ID: validGroupID}, nil) clientMock.On("CreateVolumeGroupSnapshot", mock.Anything, validGroupID, mock.Anything). Return(gopowerstore.CreateResponse{ID: validGroupID}, nil) - clientMock.On("AddMembersToVolumeGroup", - mock.Anything, - mock.AnythingOfType("*gopowerstore.VolumeGroupMembers"), - validGroupID). - Return(gopowerstore.EmptyResponse(""), nil) clientMock.On("GetVolumeGroup", mock.Anything, validGroupID). Return(gopowerstore.VolumeGroup{ ID: validGroupID, @@ -438,12 +685,53 @@ var _ = Describe("csi-extension-server", func() { Volumes: []gopowerstore.Volume{{ID: validBaseVolID, State: stateReady}}, }, nil) - createGroupRequest := &gopowerstore.VolumeGroupCreate{ - Name: validGroupName, - VolumeIds: []string{validBaseVolID}, + var sourceVols []string + sourceVols = append(sourceVols, validBaseVolID+"/"+firstValidID+"/scsi") + req := vgsext.CreateVolumeGroupSnapshotRequest{ + Name: validGroupName, + SourceVolumeIDs: sourceVols, } - clientMock.On("CreateVolumeGroup", mock.Anything, createGroupRequest). - Return(gopowerstore.CreateResponse{ID: validGroupID}, nil) + res, err := ctrlSvc.CreateVolumeGroupSnapshot(context.Background(), &req) + + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res.SnapshotGroupID).To(gomega.Equal(validGroupID)) + }) + }) + + ginkgo.When("should not create volume group snapshot with invalid request", func() { + ginkgo.It("volume group name is empty in the request", func() { + res, err := ctrlSvc.CreateVolumeGroupSnapshot(context.Background(), &vgsext.CreateVolumeGroupSnapshotRequest{}) + + gomega.Expect(err).Error() + gomega.Expect(err.Error()).To(gomega.ContainSubstring("Name to be set")) + gomega.Expect(res).To(gomega.BeNil()) + }) + + ginkgo.It("volume group name length is greater than 27 in the request", func() { + res, err := ctrlSvc.CreateVolumeGroupSnapshot(context.Background(), &vgsext.CreateVolumeGroupSnapshotRequest{ + Name: "1234561111111111111111111112", + }) + + gomega.Expect(err).Error() + gomega.Expect(err.Error()).To(gomega.ContainSubstring("longer than 27 character max")) + gomega.Expect(res).To(gomega.BeNil()) + }) + + ginkgo.It("source volumes are not present in the request", func() { + res, err := ctrlSvc.CreateVolumeGroupSnapshot(context.Background(), &vgsext.CreateVolumeGroupSnapshotRequest{ + Name: validGroupName, + }) + + gomega.Expect(err).Error() + gomega.Expect(err.Error()).To(gomega.ContainSubstring("Source volumes are not present")) + gomega.Expect(res).To(gomega.BeNil()) + }) + }) + + ginkgo.When("should not create volume group snapshot", func() { + ginkgo.It("get volume group by name fails", func() { + clientMock.On("GetVolumeGroupByName", mock.Anything, validGroupName). + Return(gopowerstore.VolumeGroup{}, gopowerstore.NewAPIError()) var sourceVols []string sourceVols = append(sourceVols, validBaseVolID+"/"+firstValidID+"/scsi") @@ -453,112 +741,115 @@ var _ = Describe("csi-extension-server", func() { } res, err := ctrlSvc.CreateVolumeGroupSnapshot(context.Background(), &req) - Expect(err).To(BeNil()) - Expect(res.SnapshotGroupID).To(Equal(validGroupID)) + gomega.Expect(err).Error() + gomega.Expect(err.Error()).To(gomega.ContainSubstring("Error getting volume group by name")) + gomega.Expect(res).To(gomega.BeNil()) }) - }) - When("member volumes are not present", func() { - It("should not create volume group snapshot successfully", func() { - clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID). - Return(gopowerstore.VolumeGroups{VolumeGroup: []gopowerstore.VolumeGroup{{ID: validGroupID, ProtectionPolicyID: validPolicyID}}}, nil) - clientMock.On("CreateVolumeGroupSnapshot", mock.Anything, validGroupID, mock.Anything). - Return(gopowerstore.CreateResponse{ID: validGroupID}, nil) + ginkgo.It("add members to volume group fails", func() { clientMock.On("GetVolumeGroupByName", mock.Anything, validGroupName). - Return(gopowerstore.VolumeGroup{ID: validGroupID, ProtectionPolicyID: validPolicyID}, nil) - clientMock.On("GetVolumeGroup", mock.Anything, validGroupID). - Return(gopowerstore.VolumeGroup{ - ID: validGroupID, - ProtectionPolicyID: validPolicyID, - Volumes: []gopowerstore.Volume{{ID: validBaseVolID, State: stateReady}}, - }, nil) + Return(gopowerstore.VolumeGroup{ID: validGroupID}, nil) clientMock.On("AddMembersToVolumeGroup", mock.Anything, mock.AnythingOfType("*gopowerstore.VolumeGroupMembers"), validGroupID). - Return(gopowerstore.EmptyResponse(""), nil) + Return(gopowerstore.EmptyResponse(""), gopowerstore.NewNotFoundError()) var sourceVols []string sourceVols = append(sourceVols, validBaseVolID+"/"+firstValidID+"/scsi") req := vgsext.CreateVolumeGroupSnapshotRequest{ - Name: validGroupName, + Name: validGroupName, + SourceVolumeIDs: sourceVols, } res, err := ctrlSvc.CreateVolumeGroupSnapshot(context.Background(), &req) - Expect(err).Error() - Expect(res).To(BeNil()) + gomega.Expect(err).Error() + gomega.Expect(err.Error()).To(gomega.ContainSubstring("Error adding volume group members")) + gomega.Expect(res).To(gomega.BeNil()) }) - }) - When("volume group name is empty", func() { - It("should not create volume group snapshot successfully", func() { + ginkgo.It("get volume group by ID fails", func() { + clientMock.On("GetVolumeGroupByName", mock.Anything, validGroupName). + Return(gopowerstore.VolumeGroup{}, nil) clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID). - Return(gopowerstore.VolumeGroups{VolumeGroup: []gopowerstore.VolumeGroup{{ID: validGroupID, ProtectionPolicyID: validPolicyID}}}, nil) - clientMock.On("CreateVolumeGroupSnapshot", mock.Anything, validGroupID, mock.Anything). - Return(gopowerstore.CreateResponse{ID: validGroupID}, nil) - clientMock.On("AddMembersToVolumeGroup", - mock.Anything, - mock.AnythingOfType("*gopowerstore.VolumeGroupMembers"), - validGroupID). - Return(gopowerstore.EmptyResponse(""), nil) - clientMock.On("GetVolumeGroup", mock.Anything, validGroupID). - Return(gopowerstore.VolumeGroup{ - ID: validGroupID, - ProtectionPolicyID: validPolicyID, - Volumes: []gopowerstore.Volume{{ID: validBaseVolID, State: stateReady}}, - }, nil) + Return(gopowerstore.VolumeGroups{}, gopowerstore.NewAPIError()) var sourceVols []string sourceVols = append(sourceVols, validBaseVolID+"/"+firstValidID+"/scsi") - res, err := ctrlSvc.CreateVolumeGroupSnapshot(context.Background(), &vgsext.CreateVolumeGroupSnapshotRequest{}) + req := vgsext.CreateVolumeGroupSnapshotRequest{ + Name: validGroupName, + SourceVolumeIDs: sourceVols, + } + res, err := ctrlSvc.CreateVolumeGroupSnapshot(context.Background(), &req) - Expect(err).Error() - Expect(res).To(BeNil()) + gomega.Expect(err).Error() + gomega.Expect(err.Error()).To(gomega.ContainSubstring("Error getting volume group by volume ID")) + gomega.Expect(res).To(gomega.BeNil()) + }) + + ginkgo.It("create volume group fails", func() { + clientMock.On("GetVolumeGroupByName", mock.Anything, validGroupName). + Return(gopowerstore.VolumeGroup{}, nil) + clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID). + Return(gopowerstore.VolumeGroups{}, nil) + createGroupRequest := &gopowerstore.VolumeGroupCreate{ + Name: validGroupName, + VolumeIDs: []string{validBaseVolID}, + } + clientMock.On("CreateVolumeGroup", mock.Anything, createGroupRequest). + Return(gopowerstore.CreateResponse{ID: validGroupID}, gopowerstore.NewNotFoundError()) + + var sourceVols []string + sourceVols = append(sourceVols, validBaseVolID+"/"+firstValidID+"/scsi") + req := vgsext.CreateVolumeGroupSnapshotRequest{ + Name: validGroupName, + SourceVolumeIDs: sourceVols, + } + res, err := ctrlSvc.CreateVolumeGroupSnapshot(context.Background(), &req) + + gomega.Expect(err).Error() + gomega.Expect(err.Error()).To(gomega.ContainSubstring("Error creating volume group")) + gomega.Expect(res).To(gomega.BeNil()) }) - }) - When("volume group name length is greater than 27", func() { - It("should not create volume group snapshot successfully", func() { + ginkgo.It("create volume group snapshot fails", func() { + clientMock.On("GetVolumeGroupByName", mock.Anything, validGroupName). + Return(gopowerstore.VolumeGroup{}, nil) clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID). Return(gopowerstore.VolumeGroups{VolumeGroup: []gopowerstore.VolumeGroup{{ID: validGroupID, ProtectionPolicyID: validPolicyID}}}, nil) - clientMock.On("CreateVolumeGroupSnapshot", mock.Anything, validGroupID, mock.Anything). - Return(gopowerstore.CreateResponse{ID: validGroupID}, nil) clientMock.On("AddMembersToVolumeGroup", mock.Anything, mock.AnythingOfType("*gopowerstore.VolumeGroupMembers"), validGroupID). Return(gopowerstore.EmptyResponse(""), nil) - clientMock.On("GetVolumeGroup", mock.Anything, validGroupID). - Return(gopowerstore.VolumeGroup{ - ID: validGroupID, - ProtectionPolicyID: validPolicyID, - Volumes: []gopowerstore.Volume{{ID: validBaseVolID, State: stateReady}}, - }, nil) + clientMock.On("CreateVolumeGroupSnapshot", mock.Anything, validGroupID, mock.Anything). + Return(gopowerstore.CreateResponse{}, gopowerstore.NewNotFoundError()) var sourceVols []string sourceVols = append(sourceVols, validBaseVolID+"/"+firstValidID+"/scsi") - res, err := ctrlSvc.CreateVolumeGroupSnapshot(context.Background(), &vgsext.CreateVolumeGroupSnapshotRequest{ - Name: "1234561111111111111111111112", - }) + req := vgsext.CreateVolumeGroupSnapshotRequest{ + Name: validGroupName, + SourceVolumeIDs: sourceVols, + } + res, err := ctrlSvc.CreateVolumeGroupSnapshot(context.Background(), &req) - Expect(err).Error() - Expect(res).To(BeNil()) + gomega.Expect(err).Error() + gomega.Expect(err.Error()).To(gomega.ContainSubstring("Error creating volume group snapshot")) + gomega.Expect(res).To(gomega.BeNil()) }) - }) - When("get volume group fails", func() { - It("should not create volume group snapshot successfully", func() { - clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID). - Return(gopowerstore.VolumeGroups{VolumeGroup: []gopowerstore.VolumeGroup{{ID: validGroupID, ProtectionPolicyID: validPolicyID}}}, nil) - clientMock.On("CreateVolumeGroupSnapshot", mock.Anything, validGroupID, mock.Anything). - Return(gopowerstore.CreateResponse{ID: validGroupID}, nil) + ginkgo.It("get volume group fails", func() { clientMock.On("GetVolumeGroupByName", mock.Anything, validGroupName). Return(gopowerstore.VolumeGroup{}, nil) + clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID). + Return(gopowerstore.VolumeGroups{VolumeGroup: []gopowerstore.VolumeGroup{{ID: validGroupID, ProtectionPolicyID: validPolicyID}}}, nil) clientMock.On("AddMembersToVolumeGroup", mock.Anything, mock.AnythingOfType("*gopowerstore.VolumeGroupMembers"), validGroupID). Return(gopowerstore.EmptyResponse(""), nil) + clientMock.On("CreateVolumeGroupSnapshot", mock.Anything, validGroupID, mock.Anything). + Return(gopowerstore.CreateResponse{ID: validGroupID}, nil) clientMock.On("GetVolumeGroup", mock.Anything, validGroupID). Return(gopowerstore.VolumeGroup{}, gopowerstore.NewNotFoundError()) @@ -570,9 +861,195 @@ var _ = Describe("csi-extension-server", func() { } res, err := ctrlSvc.CreateVolumeGroupSnapshot(context.Background(), &req) - Expect(err).Error() - Expect(res).To(BeNil()) + gomega.Expect(err).Error() + gomega.Expect(err.Error()).To(gomega.ContainSubstring("Error getting volume group snapshot")) + gomega.Expect(res).To(gomega.BeNil()) }) }) }) }) + +func Test_waitAndClose(t *testing.T) { + type args struct { + wg *sync.WaitGroup + ch chan error + } + tests := []struct { + name string + args args + }{ + { + name: "success", + args: args{ + wg: &sync.WaitGroup{}, + ch: make(chan error), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + waitAndClose(tt.args.wg, tt.args.ch) + + assert.Panics(t, func() { close(tt.args.ch) }) + }) + } +} + +func Test_isIOInProgress(t *testing.T) { + type args struct { + ctx context.Context + chs []<-chan error + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "remaining goroutines are canceled after receiving a non-nil error", + args: args{ + ctx: context.Background(), + chs: func() []<-chan error { + var chs []<-chan error + + // provide a channel that will immediately write a non-nil error + // as soon as the receiver is ready to receive. + nilErrCh := func() <-chan error { + ch := make(chan error) + go func() { + defer close(ch) + ch <- nil + }() + return ch + }() + chs = append(chs, nilErrCh) + + // add a channel on which nothing will ever be written + // causing one of the goroutines to block until the context + // is cancelled + canceledCh := make(chan error) + chs = append(chs, canceledCh) + + return chs + }(), + }, + want: true, + }, + { + name: "channels are closed after sending two non-nil errors", + args: args{ + ctx: context.Background(), + chs: func() []<-chan error { + var chs []<-chan error + // provide a channel that immediately writes a non-nil error + // and closes the channel, signaling to the goroutine to exit. + nonNilErrors := func() <-chan error { + ch := make(chan error) + go func() { + defer close(ch) + ch <- errors.New("an error occurred") + }() + return ch + } + chs = append(chs, nonNilErrors()) + chs = append(chs, nonNilErrors()) + return chs + }(), + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isIOInProgress(tt.args.ctx, tt.args.chs...); got != tt.want { + t.Errorf("isIOInProgress() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_asyncGetIOInProgress(t *testing.T) { + ctxTimeout := time.Millisecond * 100 + responseDelay := ctxTimeout * 2 + + type args struct { + ctx func() context.Context + volID string + array array.PowerStoreArray + protocol string + } + tests := []struct { + name string + args args + wantResp bool + wantErr bool + }{ + { + name: "context times out while waiting for a response", + args: args{ + ctx: func() context.Context { + ctx, cancel := context.WithTimeout(context.Background(), ctxTimeout) + t.Cleanup(func() { cancel() }) + return ctx + }, + volID: validBlockVolumeID, + array: func() array.PowerStoreArray { + clientMock = new(gopowerstoremock.Client) + // delay the response until after the ctx timeout + clientMock.On("PerformanceMetricsByVolume", mock.Anything, validBlockVolumeID, mock.Anything).After(responseDelay). + Return([]gopowerstore.PerformanceMetricsByVolumeResponse{}, nil).Times(1) + + return array.PowerStoreArray{Client: clientMock, IP: "192.168.0.1", GlobalID: firstValidID} + }(), + protocol: "scsi", + }, + wantResp: false, + }, + { + name: "returns the error", + args: args{ + ctx: func() context.Context { + ctx, cancel := context.WithTimeout(context.Background(), ctxTimeout) + t.Cleanup(func() { cancel() }) + return ctx + }, + volID: validBlockVolumeID, + array: func() array.PowerStoreArray { + clientMock = new(gopowerstoremock.Client) + clientMock.On("PerformanceMetricsByVolume", mock.Anything, validBlockVolumeID, mock.Anything). + Return(nil, errors.New("an error occurred")).Times(1) + + return array.PowerStoreArray{Client: clientMock, IP: "192.168.0.1", GlobalID: firstValidID} + }(), + protocol: "scsi", + }, + wantResp: true, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + now := time.Now() + + ctx := tt.args.ctx() + errCh := asyncGetIOInProgress(ctx, tt.args.volID, tt.args.array, tt.args.protocol) + + gotResp := false + select { + case err := <-errCh: + gotResp = true + if (err != nil) != tt.wantErr { + t.Errorf("asyncGetIOInProgress() = %v, wanted error to be %v", err, tt.wantErr) + } + case <-ctx.Done(): // if ctx times out, we do not want to be listening anymore + // give time for the mock function to return so the select statement can be + // evaluated in asyncGetIOInProgress + time.Sleep(responseDelay - time.Since(now)) + } + + if tt.wantResp != gotResp { + t.Errorf("asyncGetIOInProgress() wrote a response on the channel and was not expecting a response") + } + }) + } +} diff --git a/pkg/controller/publisher.go b/pkg/controller/publisher.go index eb053031..5b544407 100644 --- a/pkg/controller/publisher.go +++ b/pkg/controller/publisher.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2021-2023 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2021-2025 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,15 +20,13 @@ package controller import ( "context" - "errors" "fmt" "strconv" "strings" - "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/dell/csi-powerstore/v2/pkg/common" + "github.com/dell/csi-powerstore/v2/pkg/identifiers" "github.com/dell/gopowerstore" - log "github.com/sirupsen/logrus" + "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -38,17 +36,18 @@ type VolumePublisher interface { // CheckIfVolumeExists queries storage array if given volume already exists CheckIfVolumeExists(ctx context.Context, client gopowerstore.Client, volID string) error // Publish does the steps necessary for volume to be available on the node - Publish(ctx context.Context, req *csi.ControllerPublishVolumeRequest, client gopowerstore.Client, - kubeNodeID string, volumeID string) (*csi.ControllerPublishVolumeResponse, error) + Publish(ctx context.Context, publishContext map[string]string, req *csi.ControllerPublishVolumeRequest, client gopowerstore.Client, + kubeNodeID string, volumeID string, isRemote bool) (*csi.ControllerPublishVolumeResponse, error) } // SCSIPublisher implementation of VolumePublisher for SCSI based (FC, iSCSI) volumes -type SCSIPublisher struct { -} +type SCSIPublisher struct{} // Publish publishes Volume by attaching it to the host -func (s *SCSIPublisher) Publish(ctx context.Context, req *csi.ControllerPublishVolumeRequest, client gopowerstore.Client, - kubeNodeID string, volumeID string) (*csi.ControllerPublishVolumeResponse, error) { +func (s *SCSIPublisher) Publish(ctx context.Context, publishContext map[string]string, req *csi.ControllerPublishVolumeRequest, + client gopowerstore.Client, kubeNodeID string, volumeID string, isRemote bool, +) (*csi.ControllerPublishVolumeResponse, error) { + log := log.WithContext(ctx) volume, err := client.GetVolume(ctx, volumeID) if err != nil { if apiError, ok := err.(gopowerstore.APIError); ok && apiError.NotFound() { @@ -57,14 +56,12 @@ func (s *SCSIPublisher) Publish(ctx context.Context, req *csi.ControllerPublishV return nil, status.Errorf(codes.Internal, "failure checking volume status for volume publishing: %s", err.Error()) } - publishContext := make(map[string]string) - var node gopowerstore.Host node, err = client.GetHostByName(ctx, kubeNodeID) if err != nil { if apiError, ok := err.(gopowerstore.APIError); ok && apiError.HostIsNotExist() { // We need additional check here since we can just have host without ip in it - ipList := common.GetIPListFromString(kubeNodeID) + ipList := identifiers.GetIPListFromString(kubeNodeID) if ipList == nil || len(ipList) == 0 { return nil, status.Errorf(codes.NotFound, "can't find IP in node ID") } @@ -86,7 +83,6 @@ func (s *SCSIPublisher) Publish(ctx context.Context, req *csi.ControllerPublishV "failed to get mapping for volume with ID '%s': %s", volume.ID, err.Error()) } - err = s.addTargetsInfoToPublishContext(publishContext, volume.ApplianceID, client) if err != nil { return nil, status.Errorf(codes.Internal, "could not get scsi Targets: %s", err.Error()) } @@ -97,9 +93,10 @@ func (s *SCSIPublisher) Publish(ctx context.Context, req *csi.ControllerPublishV for _, m := range mapping { if m.HostID == node.ID { log.Debug("Volume already mapped") - s.addLUNIDToPublishContext(publishContext, m, volume) + s.addLUNIDToPublishContext(publishContext, m, volume, isRemote) return &csi.ControllerPublishVolumeResponse{ - PublishContext: publishContext}, nil + PublishContext: publishContext, + }, nil } } @@ -134,7 +131,7 @@ func (s *SCSIPublisher) Publish(ctx context.Context, req *csi.ControllerPublishV } for _, m := range mapping { if m.HostID == node.ID { - s.addLUNIDToPublishContext(publishContext, m, volume) + s.addLUNIDToPublishContext(publishContext, m, volume, isRemote) return &csi.ControllerPublishVolumeResponse{PublishContext: publishContext}, nil } } @@ -158,54 +155,32 @@ func (s *SCSIPublisher) CheckIfVolumeExists(ctx context.Context, client gopowers func (s *SCSIPublisher) addLUNIDToPublishContext( publishContext map[string]string, mapping gopowerstore.HostVolumeMapping, - volume gopowerstore.Volume) { - publishContext[common.PublishContextDeviceWWN] = strings.TrimPrefix(volume.Wwn, common.WWNPrefix) - publishContext[common.PublishContextLUNAddress] = strconv.FormatInt(mapping.LogicalUnitNumber, 10) -} - -func (s *SCSIPublisher) addTargetsInfoToPublishContext( - publishContext map[string]string, volumeApplianceID string, client gopowerstore.Client) error { - iscsiTargetsInfo, err := common.GetISCSITargetsInfoFromStorage(client, volumeApplianceID) - if err != nil { - log.Error("error unable to get iSCSI targets from array", err) - } - for i, t := range iscsiTargetsInfo { - publishContext[fmt.Sprintf("%s%d", common.PublishContextISCSIPortalsPrefix, i)] = t.Portal - publishContext[fmt.Sprintf("%s%d", common.PublishContextISCSITargetsPrefix, i)] = t.Target - } - fcTargetsInfo, err := common.GetFCTargetsInfoFromStorage(client, volumeApplianceID) - if err != nil { - log.Error("error unable to get FC targets from array", err) - } - for i, t := range fcTargetsInfo { - publishContext[fmt.Sprintf("%s%d", common.PublishContextFCWWPNPrefix, i)] = t.WWPN - } - - // There is no API availble for NVMeTCP and hence targets are added in node staging using goNVMe - nvmefcTargetInfo, err := common.GetNVMEFCTargetInfoFromStorage(client, volumeApplianceID) - if err != nil { - log.Error("error unable to get NVMeFC targets from array", err) - } - for i, t := range nvmefcTargetInfo { - publishContext[fmt.Sprintf("%s%d", common.PublishContextNVMEFCPortalsPrefix, i)] = t.Portal - publishContext[fmt.Sprintf("%s%d", common.PublishContextNVMEFCTargetsPrefix, i)] = t.Target + volume gopowerstore.Volume, + isRemote bool, +) { + if !isRemote { + publishContext[identifiers.TargetMapDeviceWWN] = strings.TrimPrefix(volume.Wwn, identifiers.WWNPrefix) + publishContext[identifiers.TargetMapLUNAddress] = strconv.FormatInt(mapping.LogicalUnitNumber, 10) + } else { + publishContext[identifiers.TargetMapRemoteDeviceWWN] = strings.TrimPrefix(volume.Wwn, identifiers.WWNPrefix) + publishContext[identifiers.TargetMapRemoteLUNAddress] = strconv.FormatInt(mapping.LogicalUnitNumber, 10) } - // If the system is not capable of any protocol, then we will through the error - if len(iscsiTargetsInfo) == 0 && len(fcTargetsInfo) == 0 && len(nvmefcTargetInfo) == 0 { - return errors.New("unable to get targets for any protocol") - } - return nil } // NfsPublisher implementation of VolumePublisher for NFS volumes type NfsPublisher struct { // ExternalAccess used to set custom ip to be added to the NFS Export 'hosts' list ExternalAccess string + + // ExclusiveAccess indicates whether only externalAccess entries should be added to the NFS export + ExclusiveAccess bool } // Publish publishes FileSystem by adding host (node) to the NFS Export 'hosts' list -func (n *NfsPublisher) Publish(ctx context.Context, req *csi.ControllerPublishVolumeRequest, client gopowerstore.Client, - kubeNodeID string, volumeID string) (*csi.ControllerPublishVolumeResponse, error) { +func (n *NfsPublisher) Publish(ctx context.Context, publishContext map[string]string, req *csi.ControllerPublishVolumeRequest, client gopowerstore.Client, + kubeNodeID string, volumeID string, _ bool, +) (*csi.ControllerPublishVolumeResponse, error) { + log := log.WithContext(ctx) fs, err := client.GetFS(ctx, volumeID) if err != nil { if apiError, ok := err.(gopowerstore.APIError); ok && apiError.NotFound() { @@ -213,16 +188,14 @@ func (n *NfsPublisher) Publish(ctx context.Context, req *csi.ControllerPublishVo } return nil, status.Errorf(codes.Internal, "failure checking volume status for volume publishing: %s", err.Error()) } - publishContext := make(map[string]string) - ipList := common.GetIPListFromString(kubeNodeID) + ipList := identifiers.GetIPListFromString(kubeNodeID) if ipList == nil || len(ipList) == 0 { - return nil, errors.New("can't find IP in node ID") + return nil, status.Error(codes.NotFound, "can't find IP in node ID") } ip := ipList[0] ipWithNat := make([]string, 0, 2) - ipWithNat = append(ipWithNat, ip) // Create NFS export if it doesn't exist _, err = client.GetNFSExportByFileSystemID(ctx, fs.ID) @@ -247,12 +220,22 @@ func (n *NfsPublisher) Publish(ctx context.Context, req *csi.ControllerPublishVo return nil, status.Errorf(codes.Internal, "failure getting nfs export: %s", err.Error()) } - if n.ExternalAccess != "" && !common.ExternalAccessAlreadyAdded(export, n.ExternalAccess) { - externalAccess, err := common.GetIPListWithMaskFromString(n.ExternalAccess) + // Add host IP to nfs export if not already present + if !identifiers.HostAlreadyPresentInNFSExport(export, ip) { + log.Debug("IPs have not been added") + + if !n.ExclusiveAccess { + log.Infof("Exclusive access is disabled, adding nodeIP %s to the nfs export", ip) + ipWithNat = append(ipWithNat, ip) + } + } + + if n.ExternalAccess != "" && !identifiers.ExternalAccessAlreadyAdded(export, n.ExternalAccess) { + externalAccess, err := identifiers.GetIPListWithMaskFromString(n.ExternalAccess) if err != nil { return nil, status.Errorf(codes.InvalidArgument, "can't find IP in X_CSI_POWERSTORE_EXTERNAL_ACCESS variable") } - log.Debug("externalAccess parsed IP:", externalAccess) + log.Debugf("externalAccess parsed IP: %s", externalAccess) ipWithNat = append(ipWithNat, externalAccess) } // Add host IP to existing nfs export @@ -260,7 +243,7 @@ func (n *NfsPublisher) Publish(ctx context.Context, req *csi.ControllerPublishVo AddRWRootHosts: ipWithNat, }, export.ID) if err != nil { - log.Debug("Error while PublishVolume: ", err.Error()) + log.Debugf("Error while PublishVolume: %s ", err.Error()) if apiError, ok := err.(gopowerstore.APIError); !(ok && (apiError.NotFound() || apiError.HostAlreadyPresentInNFSExport())) { return nil, status.Errorf(codes.Internal, "failure when adding new host to nfs export: %s", err.Error()) } @@ -270,20 +253,29 @@ func (n *NfsPublisher) Publish(ctx context.Context, req *csi.ControllerPublishVo if err != nil { return nil, status.Errorf(codes.Internal, "failure getting nas %s", err.Error()) } - fileInterface, err := client.GetFileInterface(ctx, nas.CurrentPreferredIPv4InterfaceId) + fileInterface, err := client.GetFileInterface(ctx, nas.CurrentPreferredIPv4InterfaceID) if err != nil { return nil, status.Errorf(codes.Internal, "failure getting file interface %s", err.Error()) } publishContext[KeyNasName] = nas.Name // we need to pass that to node part of the driver - publishContext[common.KeyNfsExportPath] = fileInterface.IpAddress + ":/" + export.Name - publishContext[common.KeyHostIP] = ipWithNat[0] + publishContext[identifiers.KeyNfsExportPath] = fileInterface.IPAddress + ":/" + export.Name + + // Add node IP to publish context only if exclusive access is not set + if !n.ExclusiveAccess { + // Ensure we don't index an empty slice; fallback to the node IP when ipWithNat is empty. + if len(ipWithNat) > 0 { + publishContext[identifiers.KeyHostIP] = ipWithNat[0] + } else { + publishContext[identifiers.KeyHostIP] = ip + } + } if n.ExternalAccess != "" { - parsedExternalAccess, _ := common.GetIPListWithMaskFromString(n.ExternalAccess) - publishContext[common.KeyNatIP] = parsedExternalAccess + parsedExternalAccess, _ := identifiers.GetIPListWithMaskFromString(n.ExternalAccess) + publishContext[identifiers.KeyNatIP] = parsedExternalAccess } - publishContext[common.KeyExportID] = export.ID - publishContext[common.KeyAllowRoot] = req.VolumeContext[common.KeyAllowRoot] - publishContext[common.KeyNfsACL] = req.VolumeContext[common.KeyNfsACL] + publishContext[identifiers.KeyExportID] = export.ID + publishContext[identifiers.KeyAllowRoot] = req.VolumeContext[identifiers.KeyAllowRoot] + publishContext[identifiers.KeyNfsACL] = req.VolumeContext[identifiers.KeyNfsACL] return &csi.ControllerPublishVolumeResponse{PublishContext: publishContext}, nil } diff --git a/pkg/controller/publisher_test.go b/pkg/controller/publisher_test.go index f3cbc414..5f75b6b3 100644 --- a/pkg/controller/publisher_test.go +++ b/pkg/controller/publisher_test.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2021-2023 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2021-2024 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ * limitations under the License. * */ -package controller_test +package controller import ( "context" @@ -24,7 +24,6 @@ import ( "net/http" "testing" - "github.com/dell/csi-powerstore/v2/pkg/controller" "github.com/dell/gopowerstore" "github.com/dell/gopowerstore/api" "github.com/dell/gopowerstore/mocks" @@ -34,7 +33,7 @@ import ( func TestVolumePublisher_Publish(t *testing.T) { t.Run("scsi publisher", func(t *testing.T) { - sp := &controller.SCSIPublisher{} + sp := &SCSIPublisher{} getVolumeOK := func(clientMock *mocks.Client) { clientMock.On("GetVolume", mock.Anything, validBaseVolID). Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9"}, nil) @@ -48,7 +47,7 @@ func TestVolumePublisher_Publish(t *testing.T) { clientMock := new(mocks.Client) clientMock.On("GetVolume", context.Background(), validBaseVolID). Return(gopowerstore.Volume{}, errors.New("error")) - _, err := sp.Publish(context.Background(), nil, clientMock, validNodeID, validBaseVolID) + _, err := sp.Publish(context.Background(), nil, nil, clientMock, validNodeID, validBaseVolID, false) assert.Error(t, err) assert.Contains(t, err.Error(), "failure checking volume status for volume publishing") }) @@ -61,7 +60,7 @@ func TestVolumePublisher_Publish(t *testing.T) { StatusCode: http.StatusNotFound, }, }) - _, err := sp.Publish(context.Background(), nil, clientMock, validNodeID, validBaseVolID) + _, err := sp.Publish(context.Background(), nil, nil, clientMock, validNodeID, validBaseVolID, false) assert.Error(t, err) assert.Contains(t, err.Error(), fmt.Sprintf("volume with ID '%s' not found", validBaseVolID)) }) @@ -79,7 +78,7 @@ func TestVolumePublisher_Publish(t *testing.T) { }, }).Once() - _, err := sp.Publish(context.Background(), nil, clientMock, nodeID, validBaseVolID) + _, err := sp.Publish(context.Background(), nil, nil, clientMock, nodeID, validBaseVolID, true) assert.Error(t, err) assert.Contains(t, err.Error(), "can't find IP in node ID") }) @@ -94,7 +93,7 @@ func TestVolumePublisher_Publish(t *testing.T) { clientMock.On("GetHostByName", mock.Anything, validNodeID). Return(gopowerstore.Host{}, e).Once() - _, err := sp.Publish(context.Background(), nil, clientMock, validNodeID, validBaseVolID) + _, err := sp.Publish(context.Background(), nil, nil, clientMock, validNodeID, validBaseVolID, false) assert.Error(t, err) assert.Contains(t, err.Error(), fmt.Sprintf("failure checking host '%s' status for volume publishing: %s", validNodeID, e.Error())) @@ -111,7 +110,7 @@ func TestVolumePublisher_Publish(t *testing.T) { clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, validBaseVolID). Return([]gopowerstore.HostVolumeMapping{}, e).Once() - _, err := sp.Publish(context.Background(), nil, clientMock, validNodeID, validBaseVolID) + _, err := sp.Publish(context.Background(), nil, nil, clientMock, validNodeID, validBaseVolID, false) assert.Error(t, err) assert.Contains(t, err.Error(), fmt.Sprintf("failed to get mapping for volume with ID '%s': %s", validBaseVolID, e.Error())) @@ -129,6 +128,8 @@ func TestVolumePublisher_Publish(t *testing.T) { Return([]gopowerstore.HostVolumeMapping{}, nil).Once() clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). Return([]gopowerstore.IPPoolAddress{}, e) + clientMock.On("GetStorageNVMETCPTargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{}, e) clientMock.On("GetFCPorts", mock.Anything). Return([]gopowerstore.FcPort{}, nil) clientMock.On("GetCluster", mock.Anything). @@ -138,14 +139,14 @@ func TestVolumePublisher_Publish(t *testing.T) { clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, validBaseVolID). Return([]gopowerstore.HostVolumeMapping{{HostID: validHostID, LogicalUnitNumber: 1}}, nil).Once() - _, err := sp.Publish(context.Background(), nil, clientMock, validNodeID, validBaseVolID) - assert.Error(t, err) - assert.Contains(t, err.Error(), e.Error()) + _, err := sp.Publish(context.Background(), make(map[string]string), nil, clientMock, validNodeID, validBaseVolID, false) + // as of 1.16, logic for finding targets is handled in the stager + assert.Nil(t, err) }) }) t.Run("nfs publisher", func(t *testing.T) { - np := &controller.NfsPublisher{} + np := &NfsPublisher{} getFSOK := func(clientMock *mocks.Client) { clientMock.On("GetFS", mock.Anything, validBaseVolID). @@ -161,7 +162,7 @@ func TestVolumePublisher_Publish(t *testing.T) { clientMock := new(mocks.Client) clientMock.On("GetFS", context.Background(), validBaseVolID). Return(gopowerstore.FileSystem{}, errors.New("error")) - _, err := np.Publish(context.Background(), nil, clientMock, validNodeID, validBaseVolID) + _, err := np.Publish(context.Background(), make(map[string]string), nil, clientMock, validNodeID, validBaseVolID, false) assert.Error(t, err) assert.Contains(t, err.Error(), "failure checking volume status for volume publishing") }) @@ -174,7 +175,7 @@ func TestVolumePublisher_Publish(t *testing.T) { StatusCode: http.StatusNotFound, }, }) - _, err := np.Publish(context.Background(), nil, clientMock, validNodeID, validBaseVolID) + _, err := np.Publish(context.Background(), make(map[string]string), nil, clientMock, validNodeID, validBaseVolID, false) assert.Error(t, err) assert.Contains(t, err.Error(), fmt.Sprintf("volume with ID '%s' not found", validBaseVolID)) }) @@ -185,7 +186,7 @@ func TestVolumePublisher_Publish(t *testing.T) { clientMock.On("GetFS", mock.Anything, validBaseVolID). Return(gopowerstore.FileSystem{ID: validBaseVolID}, nil) - _, err := np.Publish(context.Background(), nil, clientMock, nodeID, validBaseVolID) + _, err := np.Publish(context.Background(), make(map[string]string), nil, clientMock, nodeID, validBaseVolID, false) assert.Error(t, err) assert.Contains(t, err.Error(), "can't find IP in node ID") }) @@ -199,7 +200,7 @@ func TestVolumePublisher_Publish(t *testing.T) { clientMock.On("GetNFSExportByFileSystemID", mock.Anything, mock.Anything). Return(gopowerstore.NFSExport{}, e).Once() - _, err := np.Publish(context.Background(), nil, clientMock, validNodeID, validBaseVolID) + _, err := np.Publish(context.Background(), make(map[string]string), nil, clientMock, validNodeID, validBaseVolID, false) assert.Error(t, err) assert.Contains(t, err.Error(), "failure checking nfs export status for volume publishing") }) @@ -214,7 +215,7 @@ func TestVolumePublisher_Publish(t *testing.T) { clientMock.On("GetNFSExportByFileSystemID", mock.Anything, mock.Anything). Return(gopowerstore.NFSExport{}, e).Once() - _, err := np.Publish(context.Background(), nil, clientMock, validNodeID, validBaseVolID) + _, err := np.Publish(context.Background(), make(map[string]string), nil, clientMock, validNodeID, validBaseVolID, false) assert.Error(t, err) assert.Contains(t, err.Error(), "failure getting nfs export") }) @@ -229,7 +230,7 @@ func TestVolumePublisher_Publish(t *testing.T) { clientMock.On("ModifyNFSExport", mock.Anything, mock.Anything, mock.Anything). Return(gopowerstore.CreateResponse{}, e).Once() - _, err := np.Publish(context.Background(), nil, clientMock, validNodeID, validBaseVolID) + _, err := np.Publish(context.Background(), make(map[string]string), nil, clientMock, validNodeID, validBaseVolID, false) assert.Error(t, err) assert.Contains(t, err.Error(), "failure when adding new host to nfs export") }) diff --git a/pkg/controller/replication.go b/pkg/controller/replication.go index d12a6747..36fa3aa6 100644 --- a/pkg/controller/replication.go +++ b/pkg/controller/replication.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2021-2023 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2021-2025 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,24 +24,32 @@ import ( "github.com/dell/csi-powerstore/v2/pkg/array" csiext "github.com/dell/dell-csi-extensions/replication" "github.com/dell/gopowerstore" - log "github.com/sirupsen/logrus" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) // CreateRemoteVolume creates replica of volume in remote cluster func (s *Service) CreateRemoteVolume(ctx context.Context, - req *csiext.CreateRemoteVolumeRequest) (*csiext.CreateRemoteVolumeResponse, error) { + req *csiext.CreateRemoteVolumeRequest, +) (*csiext.CreateRemoteVolumeResponse, error) { + log := log.WithContext(ctx) volID := req.GetVolumeHandle() + log.Infof("CreateRemoteVolume: Full Volumerequest: %+v", req) if volID == "" { return nil, status.Error(codes.InvalidArgument, "volume ID is required") } - id, arrayID, protocol, err := array.ParseVolumeID(ctx, volID, s.DefaultArray(), nil) + volumeHandle, err := array.ParseVolumeID(ctx, volID, s.DefaultArray(), nil) if err != nil { - log.Error(err) + log.Error(err.Error()) return nil, err } + id := volumeHandle.LocalUUID + arrayID := volumeHandle.LocalArrayGlobalID + protocol := strings.ToLower(volumeHandle.Protocol) + log.Infof("CreateRemoteVolume: Parsed volume ID: LocalUUID=%s, ArrayID=%s, Protocol=%s", id, arrayID, protocol) + + volPrefix := "" arr, ok := s.Arrays()[arrayID] if !ok { @@ -49,40 +57,106 @@ func (s *Service) CreateRemoteVolume(ctx context.Context, return nil, status.Error(codes.InvalidArgument, "failed to find array with given IP") } - vgs, err := arr.GetClient().GetVolumeGroupsByVolumeID(ctx, id) - if err != nil { - return nil, err - } - if len(vgs.VolumeGroup) == 0 { - return nil, status.Error(codes.Unimplemented, "replication of volumes that aren't assigned to group is not implemented yet") - } - vg := vgs.VolumeGroup[0] + var remoteVolumeID string + var volSize int64 + var remoteSystemID string - rs, err := arr.Client.GetReplicationSessionByLocalResourceID(ctx, vg.ID) - if err != nil { - return nil, err - } + if protocol == "nfs" { + log.Infof("CreateRemoteVolume: Checking NFS export for file system ID: %s", id) + export, err := arr.Client.GetNFSExportByFileSystemID(ctx, id) + if err != nil { + log.Warnf("CreateRemoteVolume: NFS export not found, attempting to create one for file system ID: %s", id) - var remoteVolumeID string - for _, sp := range rs.StorageElementPairs { - if sp.LocalStorageElementId == id { - remoteVolumeID = sp.RemoteStorageElementId + fs, err := arr.Client.GetFS(ctx, id) + if err != nil { + log.Errorf("Failed to get file system by ID %s: %v", id, err) + return nil, status.Errorf(codes.NotFound, "file system not found") + } + prefix := "auto_export_" + var exportName string + if len(id) >= 8 { + exportName = prefix + id[:8] + } else { + exportName = prefix + id + } + + createReq := gopowerstore.NFSExportCreate{ + FileSystemID: id, + Name: exportName, + Path: "/" + fs.Name, + DefaultAccess: gopowerstore.NFSExportDefaultAccessEnum("No_Access"), + MinSecurity: "Sys", + AnonymousUID: -2, + AnonymousGID: -2, + IsNoSUID: false, + NFSOwnerUsername: "root", + } + exportID, createErr := arr.Client.CreateNFSExport(ctx, &createReq) + if createErr != nil { + log.Errorf("CreateRemoteVolume: Failed to create NFS export: %v", createErr) + return nil, status.Errorf(codes.Internal, "failed to create NFS export for file system ID %s", id) + } + + log.Infof("CreateRemoteVolume: Successfully created NFS export with ID: %s", exportID) + + // Retrieve the newly created export + export, err = arr.Client.GetNFSExportByFileSystemID(ctx, id) + if err != nil { + log.Errorf("CreateRemoteVolume: Failed to retrieve newly created NFS export: %v", err) + return nil, status.Errorf(codes.Internal, "unable to retrieve NFS export after creation for file system ID %s", id) + } + } else { + log.Infof("CreateRemoteVolume: Retrieved existing export: %+v", export) } - } - if remoteVolumeID == "" { - return nil, status.Errorf(codes.Internal, "couldn't find volume id %s in storage element pairs of replication session", id) - } + rs, err := arr.Client.GetReplicationSessionByLocalResourceID(ctx, id) + if err != nil { + log.Errorf("CreateRemoteVolume: No replication session found for file system ID: %s, error: %v", id, err) + return nil, status.Error(codes.Internal, "no replication session found for file system") + } - vol, err := arr.Client.GetVolume(ctx, id) - if err != nil { - return nil, status.Errorf(codes.Internal, "can't query volume: %s", err.Error()) + remoteSystemID = rs.RemoteSystemID + remoteVolumeID = id + volSize = 0 + } else { + // Block volume logic + vgs, err := arr.GetClient().GetVolumeGroupsByVolumeID(ctx, id) + if err != nil { + return nil, err + } + if len(vgs.VolumeGroup) == 0 { + return nil, status.Error(codes.Unimplemented, "replication of volumes that aren't assigned to group is not implemented yet") + } + vg := vgs.VolumeGroup[0] + + rs, err := arr.Client.GetReplicationSessionByLocalResourceID(ctx, vg.ID) + if err != nil { + return nil, err + } + remoteSystemID = rs.RemoteSystemID + + for _, sp := range rs.StorageElementPairs { + if sp.LocalStorageElementID == id { + remoteVolumeID = sp.RemoteStorageElementID + } + } + if remoteVolumeID == "" { + return nil, status.Errorf(codes.Internal, "couldn't find volume id %s in storage element pairs of replication session", id) + } + + vol, err := arr.Client.GetVolume(ctx, id) + if err != nil { + return nil, status.Errorf(codes.Internal, "can't query volume: %s", err.Error()) + } + volSize = vol.Size } + + // Get local and remote system info localSystem, err := arr.Client.GetCluster(ctx) if err != nil { return nil, err } - remoteSystem, err := arr.Client.GetRemoteSystem(ctx, rs.RemoteSystemId) + remoteSystem, err := arr.Client.GetRemoteSystem(ctx, remoteSystemID) if err != nil { return nil, err } @@ -92,8 +166,13 @@ func (s *Service) CreateRemoteVolume(ctx context.Context, s.replicationContextPrefix + "arrayID": remoteSystem.SerialNumber, s.replicationContextPrefix + "managementAddress": remoteSystem.ManagementAddress, } - remoteVolume := getRemoteCSIVolume(remoteVolumeID+"/"+remoteParams[s.replicationContextPrefix+"arrayID"]+"/"+protocol, vol.Size) + + remoteVolume := getRemoteCSIVolume( + volPrefix+remoteVolumeID+"/"+remoteParams[s.replicationContextPrefix+"arrayID"]+"/"+protocol, + volSize, + ) remoteVolume.VolumeContext = remoteParams + return &csiext.CreateRemoteVolumeResponse{ RemoteVolume: remoteVolume, }, nil @@ -101,72 +180,124 @@ func (s *Service) CreateRemoteVolume(ctx context.Context, // CreateStorageProtectionGroup creates storage protection group func (s *Service) CreateStorageProtectionGroup(ctx context.Context, - req *csiext.CreateStorageProtectionGroupRequest) (*csiext.CreateStorageProtectionGroupResponse, error) { + req *csiext.CreateStorageProtectionGroupRequest, +) (*csiext.CreateStorageProtectionGroupResponse, error) { + log := log.WithContext(ctx) volID := req.GetVolumeHandle() if volID == "" { return nil, status.Error(codes.InvalidArgument, "volume ID is required") } - id, arrayID, protocol, err := array.ParseVolumeID(ctx, volID, s.DefaultArray(), nil) + volumeHandle, err := array.ParseVolumeID(ctx, volID, s.DefaultArray(), nil) if err != nil { - log.Error(err) + log.Error(err.Error()) return nil, err } + id := volumeHandle.LocalUUID + arrayID := volumeHandle.LocalArrayGlobalID + protocol := strings.ToLower(volumeHandle.Protocol) + arr, ok := s.Arrays()[arrayID] if !ok { log.Info("id is nil") return nil, status.Error(codes.InvalidArgument, "failed to find array with given ID") } - if protocol == "nfs" { - return nil, status.Error(codes.InvalidArgument, "replication is not supported for NFS volumes") - } + var localGroupID, remoteGroupID string + var localParams, remoteParams map[string]string - vgs, err := arr.GetClient().GetVolumeGroupsByVolumeID(ctx, id) + localSystem, err := arr.Client.GetCluster(ctx) if err != nil { return nil, err } - if len(vgs.VolumeGroup) == 0 { - return nil, status.Error(codes.Unimplemented, "replication of volumes that aren't assigned to group is not implemented yet") - } - vg := vgs.VolumeGroup[0] - rs, err := arr.Client.GetReplicationSessionByLocalResourceID(ctx, vg.ID) - if err != nil { - return nil, err - } + if protocol == "nfs" { + log.Infof("CreateRemoteVolume: Checking NFS export for file system ID: %s", id) + export, err := arr.Client.GetNFSExportByFileSystemID(ctx, id) + if err != nil { + log.Errorf("CreateRemoteVolume: Error retrieving NFS export: %v", err) + } else { + log.Infof("CreateRemoteVolume: Retrieved export: %+v", export) + } + nasServerID := export.ID - localSystem, err := arr.Client.GetCluster(ctx) - if err != nil { - return nil, err - } + rs, err := arr.Client.GetReplicationSessionByLocalResourceID(ctx, id) + if err != nil { + return nil, status.Error(codes.Internal, "no replication session found for NAS server") + } - remoteSystem, err := arr.Client.GetRemoteSystem(ctx, rs.RemoteSystemId) - if err != nil { - return nil, err - } - localParams := map[string]string{ - s.replicationContextPrefix + "systemName": localSystem.Name, - s.replicationContextPrefix + "managementAddress": localSystem.ManagementAddress, - s.replicationContextPrefix + "remoteSystemName": remoteSystem.Name, - s.replicationContextPrefix + "remoteManagementAddress": remoteSystem.ManagementAddress, - s.replicationContextPrefix + "globalID": arrayID, - s.replicationContextPrefix + "remoteGlobalID": remoteSystem.SerialNumber, - s.replicationContextPrefix + "VolumeGroupName": vg.Name, - } - remoteParams := map[string]string{ - s.replicationContextPrefix + "systemName": remoteSystem.Name, - s.replicationContextPrefix + "managementAddress": remoteSystem.ManagementAddress, - s.replicationContextPrefix + "remoteSystemName": localSystem.Name, - s.replicationContextPrefix + "remoteManagementAddress": localSystem.ManagementAddress, - s.replicationContextPrefix + "globalID": remoteSystem.SerialNumber, - s.replicationContextPrefix + "VolumeGroupName": vg.Name, + remoteSystem, err := arr.Client.GetRemoteSystem(ctx, rs.RemoteSystemID) + if err != nil { + return nil, err + } + + localGroupID = rs.LocalResourceID + remoteGroupID = rs.RemoteResourceID + + localParams = map[string]string{ + s.replicationContextPrefix + "systemName": localSystem.Name, + s.replicationContextPrefix + "managementAddress": localSystem.ManagementAddress, + s.replicationContextPrefix + "remoteSystemName": remoteSystem.Name, + s.replicationContextPrefix + "remoteManagementAddress": remoteSystem.ManagementAddress, + s.replicationContextPrefix + "globalID": arrayID, + s.replicationContextPrefix + "remoteGlobalID": remoteSystem.SerialNumber, + s.replicationContextPrefix + "NasServerID": nasServerID, + } + remoteParams = map[string]string{ + s.replicationContextPrefix + "systemName": remoteSystem.Name, + s.replicationContextPrefix + "managementAddress": remoteSystem.ManagementAddress, + s.replicationContextPrefix + "remoteSystemName": localSystem.Name, + s.replicationContextPrefix + "remoteManagementAddress": localSystem.ManagementAddress, + s.replicationContextPrefix + "globalID": remoteSystem.SerialNumber, + s.replicationContextPrefix + "NasServerID": rs.RemoteResourceID, + } + } else { + // Block volume logic + vgs, err := arr.GetClient().GetVolumeGroupsByVolumeID(ctx, id) + if err != nil { + return nil, err + } + if len(vgs.VolumeGroup) == 0 { + return nil, status.Error(codes.Unimplemented, "replication of volumes that aren't assigned to group is not implemented yet") + } + vg := vgs.VolumeGroup[0] + + rs, err := arr.Client.GetReplicationSessionByLocalResourceID(ctx, vg.ID) + if err != nil { + return nil, err + } + + remoteSystem, err := arr.Client.GetRemoteSystem(ctx, rs.RemoteSystemID) + if err != nil { + return nil, err + } + + localGroupID = rs.LocalResourceID + remoteGroupID = rs.RemoteResourceID + + localParams = map[string]string{ + s.replicationContextPrefix + "systemName": localSystem.Name, + s.replicationContextPrefix + "managementAddress": localSystem.ManagementAddress, + s.replicationContextPrefix + "remoteSystemName": remoteSystem.Name, + s.replicationContextPrefix + "remoteManagementAddress": remoteSystem.ManagementAddress, + s.replicationContextPrefix + "globalID": arrayID, + s.replicationContextPrefix + "remoteGlobalID": remoteSystem.SerialNumber, + s.replicationContextPrefix + "VolumeGroupName": vg.Name, + } + remoteParams = map[string]string{ + s.replicationContextPrefix + "systemName": remoteSystem.Name, + s.replicationContextPrefix + "managementAddress": remoteSystem.ManagementAddress, + s.replicationContextPrefix + "remoteSystemName": localSystem.Name, + s.replicationContextPrefix + "remoteManagementAddress": localSystem.ManagementAddress, + s.replicationContextPrefix + "globalID": remoteSystem.SerialNumber, + s.replicationContextPrefix + "VolumeGroupName": vg.Name, + } } return &csiext.CreateStorageProtectionGroupResponse{ - LocalProtectionGroupId: rs.LocalResourceId, - RemoteProtectionGroupId: rs.RemoteResourceId, + LocalProtectionGroupId: localGroupID, + RemoteProtectionGroupId: remoteGroupID, LocalProtectionGroupAttributes: localParams, RemoteProtectionGroupAttributes: remoteParams, }, nil @@ -174,8 +305,8 @@ func (s *Service) CreateStorageProtectionGroup(ctx context.Context, // EnsureProtectionPolicyExists ensures protection policy exists func EnsureProtectionPolicyExists(ctx context.Context, arr *array.PowerStoreArray, - vgName string, remoteSystemName string, rpoEnum gopowerstore.RPOEnum) (string, error) { - + vgName string, remoteSystemName string, rpoEnum gopowerstore.RPOEnum, +) (string, error) { // Get id of specified remote system rs, err := arr.Client.GetRemoteSystemByName(ctx, remoteSystemName) if err != nil { @@ -198,7 +329,7 @@ func EnsureProtectionPolicyExists(ctx context.Context, arr *array.PowerStoreArra newPp, err := arr.Client.CreateProtectionPolicy(ctx, &gopowerstore.ProtectionPolicyCreate{ Name: ppName, - ReplicationRuleIds: []string{rrID}, + ReplicationRuleIDs: []string{rrID}, }) if err != nil { return "", status.Errorf(codes.Internal, "can't create protection policy: %s", err.Error()) @@ -209,7 +340,8 @@ func EnsureProtectionPolicyExists(ctx context.Context, arr *array.PowerStoreArra // EnsureReplicationRuleExists ensures replication rule exists func EnsureReplicationRuleExists(ctx context.Context, arr *array.PowerStoreArray, - vgName string, remoteSystemID string, rpoEnum gopowerstore.RPOEnum) (string, error) { + vgName string, remoteSystemID string, rpoEnum gopowerstore.RPOEnum, +) (string, error) { rrName := "rr-" + vgName rr, err := arr.Client.GetReplicationRuleByName(ctx, rrName) if err != nil { @@ -228,8 +360,8 @@ func EnsureReplicationRuleExists(ctx context.Context, arr *array.PowerStoreArray } // GetReplicationCapabilities is a getter for replication capabilities -func (s *Service) GetReplicationCapabilities(ctx context.Context, req *csiext.GetReplicationCapabilityRequest) (*csiext.GetReplicationCapabilityResponse, error) { - var rep = new(csiext.GetReplicationCapabilityResponse) +func (s *Service) GetReplicationCapabilities(_ context.Context, _ *csiext.GetReplicationCapabilityRequest) (*csiext.GetReplicationCapabilityResponse, error) { + rep := new(csiext.GetReplicationCapabilityResponse) rep.Capabilities = []*csiext.ReplicationCapability{ { Type: &csiext.ReplicationCapability_Rpc{ @@ -304,8 +436,9 @@ func (s *Service) GetReplicationCapabilities(ctx context.Context, req *csiext.Ge // ExecuteAction is a method to execute an action request func (s *Service) ExecuteAction(ctx context.Context, - req *csiext.ExecuteActionRequest) (*csiext.ExecuteActionResponse, error) { - + req *csiext.ExecuteActionRequest, +) (*csiext.ExecuteActionResponse, error) { + log := log.WithContext(ctx) var reqID string localParams := req.GetProtectionGroupAttributes() protectionGroupID := req.GetProtectionGroupId() @@ -332,30 +465,29 @@ func (s *Service) ExecuteAction(ctx context.Context, if err != nil { return nil, err } - var client = pstoreClient + client := pstoreClient var execAction gopowerstore.ActionType - var params *gopowerstore.FailoverParams = nil + var params *gopowerstore.FailoverParams switch action { case csiext.ActionTypes_FAILOVER_REMOTE.String(): - execAction = gopowerstore.RS_ACTION_FAILOVER + execAction = gopowerstore.RsActionFailover params = &gopowerstore.FailoverParams{IsPlanned: true, Reverse: false} case csiext.ActionTypes_UNPLANNED_FAILOVER_LOCAL.String(): - execAction = gopowerstore.RS_ACTION_FAILOVER + execAction = gopowerstore.RsActionFailover params = &gopowerstore.FailoverParams{IsPlanned: false, Reverse: false} case csiext.ActionTypes_SUSPEND.String(): - execAction = gopowerstore.RS_ACTION_PAUSE + execAction = gopowerstore.RsActionPause case csiext.ActionTypes_RESUME.String(): - execAction = gopowerstore.RS_ACTION_RESUME + execAction = gopowerstore.RsActionResume case csiext.ActionTypes_SYNC.String(): - execAction = gopowerstore.RS_ACTION_SYNC + execAction = gopowerstore.RsActionSync case csiext.ActionTypes_REPROTECT_LOCAL.String(): - execAction = gopowerstore.RS_ACTION_REPROTECT + execAction = gopowerstore.RsActionReprotect default: return nil, status.Errorf(codes.Unknown, "The requested action does not match with supported actions") } resErr := ExecuteAction(&rs, client, execAction, params) if resErr != nil { - return nil, resErr } @@ -391,9 +523,8 @@ func ExecuteAction(session *gopowerstore.ReplicationSession, pstoreClient gopowe _, err := pstoreClient.ExecuteActionOnReplicationSession(context.Background(), session.ID, action, failoverParams) - if err != nil { - if apiError, ok := err.(gopowerstore.APIError); ok && !apiError.UnableToFailoverFromDestination() { + if apiError, ok := err.(gopowerstore.APIError); ok && apiError.UnableToFailoverFromDestination() { log.Error(fmt.Sprintf("Fail over: Failed to modify RS (%s) - Error (%s)", session.ID, err.Error())) return status.Errorf(codes.Internal, "Execute action: Failed to modify RS (%s) - Error (%s)", session.ID, err.Error()) } @@ -408,22 +539,22 @@ func validateRSState(session *gopowerstore.ReplicationSession, action gopowersto state := session.State log.Infof("replication session is in %s", state) switch action { - case gopowerstore.RS_ACTION_RESUME: + case gopowerstore.RsActionResume: if state == "OK" { log.Infof("RS (%s) is already in desired state: (%s)", session.ID, state) return true, false, nil } - case gopowerstore.RS_ACTION_REPROTECT: + case gopowerstore.RsActionReprotect: if state == "OK" { log.Infof("RS (%s) is already in desired state: (%s)", session.ID, state) return true, false, nil } - case gopowerstore.RS_ACTION_PAUSE: + case gopowerstore.RsActionPause: if state == "Paused" || state == "Paused_For_Migration" || state == "Paused_For_NDU" { log.Infof("RS (%s) is already in desired state: (%s)", session.ID, state) return true, false, nil } - case gopowerstore.RS_ACTION_FAILOVER: + case gopowerstore.RsActionFailover: if state == "Failing_Over" { return false, false, nil } @@ -435,13 +566,13 @@ func validateRSState(session *gopowerstore.ReplicationSession, action gopowersto return false, true, nil } -// DeleteStorageProtectionGroup deletes storage protection group -func (s *Service) DeleteStorageProtectionGroup(ctx context.Context, - req *csiext.DeleteStorageProtectionGroupRequest) (*csiext.DeleteStorageProtectionGroupResponse, error) { +func (s *Service) DeleteStorageProtectionGroup( + ctx context.Context, + req *csiext.DeleteStorageProtectionGroupRequest, +) (*csiext.DeleteStorageProtectionGroupResponse, error) { localParams := req.GetProtectionGroupAttributes() groupID := req.GetProtectionGroupId() globalID, ok := localParams[s.replicationContextPrefix+"globalID"] - if !ok { return nil, status.Error(codes.InvalidArgument, "missing globalID in protection group attributes") } @@ -454,54 +585,76 @@ func (s *Service) DeleteStorageProtectionGroup(ctx context.Context, "GlobalID": globalID, "ProtectedStorageGroup": groupID, } + log := log.WithContext(ctx).WithFields(fields) + + log.Info("Deleting storage protection group") - log.WithFields(fields).Info("Deleting storage protection group") + nasServerID, hasNas := localParams[s.replicationContextPrefix+"NasServerID"] + // Skip deletion logic for NFS replication sessions + // Storage Protection Group (i.e., Protection Policy) cannot be deleted from the PowerStore array + // if it is assigned to a NAS server. Modifying NAS to unassign the policy is currently unsupported, + // so deletion is effectively blocked for sync/async NAS contexts. + if hasNas && nasServerID != "" { + log.Info("NFS context detected — skipping deletion logic") + return &csiext.DeleteStorageProtectionGroupResponse{}, nil + } + // Block: Unassign PP and delete VolumeGroup vg, err := arr.GetClient().GetVolumeGroup(ctx, groupID) if apiErr, ok := err.(gopowerstore.APIError); ok && !apiErr.NotFound() { + log.Errorf("Failed to get Volume Group: %v", apiErr) return nil, status.Errorf(codes.Internal, "Error: Unable to get Volume Group") } if vg.ID != "" { if vg.ProtectionPolicyID != "" { _, err := arr.GetClient().ModifyVolumeGroup(ctx, &gopowerstore.VolumeGroupModify{ - ProtectionPolicyId: "", + ProtectionPolicyID: "", }, groupID) if apiErr, ok := err.(gopowerstore.APIError); ok && !apiErr.NotFound() { + log.Errorf("Unable to un-assign PP from Volume Group: %v", apiErr) return nil, status.Errorf(codes.Internal, "Error: Unable to un-assign PP from Volume Group") } } _, err = arr.Client.DeleteVolumeGroup(ctx, groupID) - if apiError, ok := err.(gopowerstore.APIError); ok && !apiError.NotFound() { - return nil, status.Errorf(codes.Internal, "Error: %s: Unable to delete Volume Group", apiError.Error()) + if apiErr, ok := err.(gopowerstore.APIError); ok && !apiErr.NotFound() { + log.Errorf("Unable to delete Volume Group: %v", apiErr) + return nil, status.Errorf(codes.Internal, "Error: Unable to delete Volume Group") } } - log.WithFields(fields).Info("Deleting protection policy") - vgName, ok := localParams[s.replicationContextPrefix+"VolumeGroupName"] if !ok { return nil, status.Errorf(codes.Internal, "Error: Unable to get volume group name") } + + // Delete Protection Policy + log.Info("Deleting protection policy") pp, err := arr.GetClient().GetProtectionPolicyByName(ctx, "pp-"+vgName) if apiErr, ok := err.(gopowerstore.APIError); ok && !apiErr.NotFound() { - return nil, status.Errorf(codes.Internal, "Error: Unable to get the PP") + log.Errorf("Error retrieving protection policy: %v", apiErr) + return nil, status.Errorf(codes.Internal, "Error: Unable to get protection policy") } - if pp.ID != "" && len(pp.Volumes) == 0 && len(pp.VolumeGroups) == 0 { + if pp.ID != "" && + len(pp.Volumes) == 0 && + len(pp.VolumeGroups) == 0 { _, err := arr.Client.DeleteProtectionPolicy(ctx, pp.ID) if apiErr, ok := err.(gopowerstore.APIError); ok && !apiErr.NotFound() { - return nil, status.Errorf(codes.Internal, "Error: Unable to delete PP") + log.Errorf("Unable to delete protection policy: %v", apiErr) + return nil, status.Errorf(codes.Internal, "Error: Unable to delete protection policy") } } - log.WithFields(fields).Info("Deleting replication rule") - + // Delete Replication Rule + log.Info("Deleting replication rule") rr, err := arr.GetClient().GetReplicationRuleByName(ctx, "rr-"+vgName) if apiErr, ok := err.(gopowerstore.APIError); ok && !apiErr.NotFound() { - return nil, status.Errorf(codes.Internal, "Error: RR not found") + log.Errorf("Error retrieving replication rule: %v", apiErr) + return nil, status.Errorf(codes.Internal, "Error: Unable to get replication rule") } if rr.ID != "" && len(rr.ProtectionPolicies) == 0 { _, err = arr.GetClient().DeleteReplicationRule(ctx, rr.ID) if apiErr, ok := err.(gopowerstore.APIError); ok && !apiErr.NotFound() { + log.Errorf("Unable to delete replication rule: %v", apiErr) return nil, status.Errorf(codes.Internal, "Error: Unable to delete replication rule") } } @@ -511,7 +664,9 @@ func (s *Service) DeleteStorageProtectionGroup(ctx context.Context, // DeleteLocalVolume deletes a volume on the local storage array upon request from a remote replication controller. func (s *Service) DeleteLocalVolume(ctx context.Context, - req *csiext.DeleteLocalVolumeRequest) (*csiext.DeleteLocalVolumeResponse, error) { + req *csiext.DeleteLocalVolumeRequest, +) (*csiext.DeleteLocalVolumeResponse, error) { + log := log.WithContext(ctx) log.Info("Deleting local volume " + req.VolumeHandle + " per request from remote replication controller") // req.VolumeHandle is of format //. We only need the IDs. @@ -571,7 +726,9 @@ func (s *Service) DeleteLocalVolume(ctx context.Context, // GetStorageProtectionGroupStatus gets storage protection group status func (s *Service) GetStorageProtectionGroupStatus(ctx context.Context, - req *csiext.GetStorageProtectionGroupStatusRequest) (*csiext.GetStorageProtectionGroupStatusResponse, error) { + req *csiext.GetStorageProtectionGroupStatusRequest, +) (*csiext.GetStorageProtectionGroupStatusResponse, error) { + log := log.WithContext(ctx) localParams := req.GetProtectionGroupAttributes() groupID := req.GetProtectionGroupId() @@ -597,21 +754,21 @@ func (s *Service) GetStorageProtectionGroupStatus(ctx context.Context, var state csiext.StorageProtectionGroupStatus_State switch rs.State { - case gopowerstore.RS_STATE_OK: + case gopowerstore.RsStateOk: state = csiext.StorageProtectionGroupStatus_SYNCHRONIZED break - case gopowerstore.RS_STATE_FAILED_OVER: + case gopowerstore.RsStateFailedOver: state = csiext.StorageProtectionGroupStatus_FAILEDOVER break - case gopowerstore.RS_STATE_PAUSED, gopowerstore.RS_STATE_PAUSED_FOR_MIGRATION, gopowerstore.RS_STATE_PAUSED_FOR_NDU, gopowerstore.RS_STATE_SYSTEM_PAUSED: + case gopowerstore.RsStatePaused, gopowerstore.RsStatePausedForMigration, gopowerstore.RsStatePausedForNdu, gopowerstore.RsStateSystemPaused: state = csiext.StorageProtectionGroupStatus_SUSPENDED break - case gopowerstore.RS_STATE_FAILING_OVER, gopowerstore.RS_STATE_FAILING_OVER_FOR_DR, gopowerstore.RS_STATE_RESUMING, - gopowerstore.RS_STATE_REPROTECTING, gopowerstore.RS_STATE_PARTIAL_CUTOVER_FOR_MIGRATION, gopowerstore.RS_STATE_SYNCHRONIZING, - gopowerstore.RS_STATE_INITIALIZING: + case gopowerstore.RsStateFailingOver, gopowerstore.RsStateFailingOverForDR, gopowerstore.RsStateResuming, + gopowerstore.RsStateReprotecting, gopowerstore.RsStatePartialCutoverForMigration, gopowerstore.RsStateSynchronizing, + gopowerstore.RsStateInitializing: state = csiext.StorageProtectionGroupStatus_SYNC_IN_PROGRESS break - case gopowerstore.RS_STATE_ERROR: + case gopowerstore.RsStateError: state = csiext.StorageProtectionGroupStatus_INVALID break default: @@ -631,7 +788,12 @@ func (s *Service) GetStorageProtectionGroupStatus(ctx context.Context, // WithRP appends Replication Prefix to provided string func (s *Service) WithRP(key string) string { - return s.replicationPrefix + "/" + key + replicationPrefix := s.replicationPrefix + if replicationPrefix == "" { + replicationPrefix = ReplicationPrefix + } + + return replicationPrefix + "/" + key } func getRemoteCSIVolume(volumeID string, size int64) *csiext.Volume { diff --git a/pkg/controller/replication_test.go b/pkg/controller/replication_test.go index 6928a679..f9ff609a 100755 --- a/pkg/controller/replication_test.go +++ b/pkg/controller/replication_test.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2021-2023 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2021-2024 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -8,40 +8,39 @@ * http://www.apache.org/licenses/LICENSE-2.0 * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * */ -package controller_test +package controller import ( "context" "net/http" "github.com/dell/csi-powerstore/v2/pkg/array" - "github.com/dell/csi-powerstore/v2/pkg/controller" csiext "github.com/dell/dell-csi-extensions/replication" "github.com/dell/gopowerstore" "github.com/dell/gopowerstore/api" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" + ginkgo "github.com/onsi/ginkgo" + gomega "github.com/onsi/gomega" "github.com/stretchr/testify/mock" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) -var _ = Describe("Replication", func() { - BeforeEach(func() { +var _ = ginkgo.Describe("Replication", func() { + ginkgo.BeforeEach(func() { setVariables() }) - Describe("calling GetStorageProtectionGroupStatus()", func() { - When("getting storage protection group status and state is ok", func() { - It("should return synchronized status", func() { + ginkgo.Describe("calling GetStorageProtectionGroupStatus()", func() { + ginkgo.When("getting storage protection group status and state is ok", func() { + ginkgo.It("should return synchronized status", func() { clientMock.On("GetReplicationSessionByLocalResourceID", mock.Anything, mock.Anything).Return( - gopowerstore.ReplicationSession{State: gopowerstore.RS_STATE_OK}, nil) + gopowerstore.ReplicationSession{State: gopowerstore.RsStateOk}, nil) req := new(csiext.GetStorageProtectionGroupStatusRequest) params := make(map[string]string) @@ -49,17 +48,17 @@ var _ = Describe("Replication", func() { req.ProtectionGroupAttributes = params res, err := ctrlSvc.GetStorageProtectionGroupStatus(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res.Status.State).To(Equal( + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res.Status.State).To(gomega.Equal( csiext.StorageProtectionGroupStatus_SYNCHRONIZED, )) }) }) - When("getting storage protection group status and state is failed over", func() { - It("should return failed over status", func() { + ginkgo.When("getting storage protection group status and state is failed over", func() { + ginkgo.It("should return failed over status", func() { clientMock.On("GetReplicationSessionByLocalResourceID", mock.Anything, mock.Anything).Return( - gopowerstore.ReplicationSession{State: gopowerstore.RS_STATE_FAILED_OVER}, nil) + gopowerstore.ReplicationSession{State: gopowerstore.RsStateFailedOver}, nil) req := new(csiext.GetStorageProtectionGroupStatusRequest) params := make(map[string]string) @@ -67,17 +66,17 @@ var _ = Describe("Replication", func() { req.ProtectionGroupAttributes = params res, err := ctrlSvc.GetStorageProtectionGroupStatus(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res.Status.State).To(Equal( + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res.Status.State).To(gomega.Equal( csiext.StorageProtectionGroupStatus_FAILEDOVER, )) }) }) - When("getting storage protection group status and state is paused (for several reasons)", func() { - It("should return suspended status (if paused)", func() { + ginkgo.When("getting storage protection group status and state is paused (for several reasons)", func() { + ginkgo.It("should return suspended status (if paused)", func() { clientMock.On("GetReplicationSessionByLocalResourceID", mock.Anything, mock.Anything).Return( - gopowerstore.ReplicationSession{State: gopowerstore.RS_STATE_PAUSED}, nil) + gopowerstore.ReplicationSession{State: gopowerstore.RsStatePaused}, nil) req := new(csiext.GetStorageProtectionGroupStatusRequest) params := make(map[string]string) @@ -85,14 +84,14 @@ var _ = Describe("Replication", func() { req.ProtectionGroupAttributes = params res, err := ctrlSvc.GetStorageProtectionGroupStatus(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res.Status.State).To(Equal( + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res.Status.State).To(gomega.Equal( csiext.StorageProtectionGroupStatus_SUSPENDED, )) }) - It("should return suspended status (if paused for migration)", func() { + ginkgo.It("should return suspended status (if paused for migration)", func() { clientMock.On("GetReplicationSessionByLocalResourceID", mock.Anything, mock.Anything).Return( - gopowerstore.ReplicationSession{State: gopowerstore.RS_STATE_PAUSED_FOR_MIGRATION}, nil) + gopowerstore.ReplicationSession{State: gopowerstore.RsStatePausedForMigration}, nil) req := new(csiext.GetStorageProtectionGroupStatusRequest) params := make(map[string]string) @@ -100,14 +99,14 @@ var _ = Describe("Replication", func() { req.ProtectionGroupAttributes = params res, err := ctrlSvc.GetStorageProtectionGroupStatus(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res.Status.State).To(Equal( + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res.Status.State).To(gomega.Equal( csiext.StorageProtectionGroupStatus_SUSPENDED, )) }) - It("should return suspended status (if paused for NDU)", func() { + ginkgo.It("should return suspended status (if paused for NDU)", func() { clientMock.On("GetReplicationSessionByLocalResourceID", mock.Anything, mock.Anything).Return( - gopowerstore.ReplicationSession{State: gopowerstore.RS_STATE_PAUSED_FOR_NDU}, nil) + gopowerstore.ReplicationSession{State: gopowerstore.RsStatePausedForNdu}, nil) req := new(csiext.GetStorageProtectionGroupStatusRequest) params := make(map[string]string) @@ -115,14 +114,14 @@ var _ = Describe("Replication", func() { req.ProtectionGroupAttributes = params res, err := ctrlSvc.GetStorageProtectionGroupStatus(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res.Status.State).To(Equal( + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res.Status.State).To(gomega.Equal( csiext.StorageProtectionGroupStatus_SUSPENDED, )) }) - It("should return suspended status (if system paused)", func() { + ginkgo.It("should return suspended status (if system paused)", func() { clientMock.On("GetReplicationSessionByLocalResourceID", mock.Anything, mock.Anything).Return( - gopowerstore.ReplicationSession{State: gopowerstore.RS_STATE_SYSTEM_PAUSED}, nil) + gopowerstore.ReplicationSession{State: gopowerstore.RsStateSystemPaused}, nil) req := new(csiext.GetStorageProtectionGroupStatusRequest) params := make(map[string]string) @@ -130,17 +129,17 @@ var _ = Describe("Replication", func() { req.ProtectionGroupAttributes = params res, err := ctrlSvc.GetStorageProtectionGroupStatus(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res.Status.State).To(Equal( + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res.Status.State).To(gomega.Equal( csiext.StorageProtectionGroupStatus_SUSPENDED, )) }) }) - When("getting storage protection group status and state is updating (in progress)", func() { - It("should return 'sync in progress' status (if failing over)", func() { + ginkgo.When("getting storage protection group status and state is updating (in progress)", func() { + ginkgo.It("should return 'sync in progress' status (if failing over)", func() { clientMock.On("GetReplicationSessionByLocalResourceID", mock.Anything, mock.Anything).Return( - gopowerstore.ReplicationSession{State: gopowerstore.RS_STATE_FAILING_OVER}, nil) + gopowerstore.ReplicationSession{State: gopowerstore.RsStateFailingOver}, nil) req := new(csiext.GetStorageProtectionGroupStatusRequest) params := make(map[string]string) @@ -148,14 +147,14 @@ var _ = Describe("Replication", func() { req.ProtectionGroupAttributes = params res, err := ctrlSvc.GetStorageProtectionGroupStatus(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res.Status.State).To(Equal( + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res.Status.State).To(gomega.Equal( csiext.StorageProtectionGroupStatus_SYNC_IN_PROGRESS, )) }) - It("should return 'sync in progress' status (if failing over for DR)", func() { + ginkgo.It("should return 'sync in progress' status (if failing over for DR)", func() { clientMock.On("GetReplicationSessionByLocalResourceID", mock.Anything, mock.Anything).Return( - gopowerstore.ReplicationSession{State: gopowerstore.RS_STATE_FAILING_OVER_FOR_DR}, nil) + gopowerstore.ReplicationSession{State: gopowerstore.RsStateFailingOverForDR}, nil) req := new(csiext.GetStorageProtectionGroupStatusRequest) params := make(map[string]string) @@ -163,14 +162,14 @@ var _ = Describe("Replication", func() { req.ProtectionGroupAttributes = params res, err := ctrlSvc.GetStorageProtectionGroupStatus(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res.Status.State).To(Equal( + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res.Status.State).To(gomega.Equal( csiext.StorageProtectionGroupStatus_SYNC_IN_PROGRESS, )) }) - It("should return 'sync in progress' status (if resuming)", func() { + ginkgo.It("should return 'sync in progress' status (if resuming)", func() { clientMock.On("GetReplicationSessionByLocalResourceID", mock.Anything, mock.Anything).Return( - gopowerstore.ReplicationSession{State: gopowerstore.RS_STATE_RESUMING}, nil) + gopowerstore.ReplicationSession{State: gopowerstore.RsStateResuming}, nil) req := new(csiext.GetStorageProtectionGroupStatusRequest) params := make(map[string]string) @@ -178,14 +177,14 @@ var _ = Describe("Replication", func() { req.ProtectionGroupAttributes = params res, err := ctrlSvc.GetStorageProtectionGroupStatus(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res.Status.State).To(Equal( + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res.Status.State).To(gomega.Equal( csiext.StorageProtectionGroupStatus_SYNC_IN_PROGRESS, )) }) - It("should return 'sync in progress' status (if reprotecting)", func() { + ginkgo.It("should return 'sync in progress' status (if reprotecting)", func() { clientMock.On("GetReplicationSessionByLocalResourceID", mock.Anything, mock.Anything).Return( - gopowerstore.ReplicationSession{State: gopowerstore.RS_STATE_REPROTECTING}, nil) + gopowerstore.ReplicationSession{State: gopowerstore.RsStateReprotecting}, nil) req := new(csiext.GetStorageProtectionGroupStatusRequest) params := make(map[string]string) @@ -193,14 +192,14 @@ var _ = Describe("Replication", func() { req.ProtectionGroupAttributes = params res, err := ctrlSvc.GetStorageProtectionGroupStatus(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res.Status.State).To(Equal( + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res.Status.State).To(gomega.Equal( csiext.StorageProtectionGroupStatus_SYNC_IN_PROGRESS, )) }) - It("should return 'sync in progress' status (if cutover for migration)", func() { + ginkgo.It("should return 'sync in progress' status (if cutover for migration)", func() { clientMock.On("GetReplicationSessionByLocalResourceID", mock.Anything, mock.Anything).Return( - gopowerstore.ReplicationSession{State: gopowerstore.RS_STATE_PARTIAL_CUTOVER_FOR_MIGRATION}, nil) + gopowerstore.ReplicationSession{State: gopowerstore.RsStatePartialCutoverForMigration}, nil) req := new(csiext.GetStorageProtectionGroupStatusRequest) params := make(map[string]string) @@ -208,14 +207,14 @@ var _ = Describe("Replication", func() { req.ProtectionGroupAttributes = params res, err := ctrlSvc.GetStorageProtectionGroupStatus(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res.Status.State).To(Equal( + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res.Status.State).To(gomega.Equal( csiext.StorageProtectionGroupStatus_SYNC_IN_PROGRESS, )) }) - It("should return 'sync in progress' status (if synchronizing)", func() { + ginkgo.It("should return 'sync in progress' status (if synchronizing)", func() { clientMock.On("GetReplicationSessionByLocalResourceID", mock.Anything, mock.Anything).Return( - gopowerstore.ReplicationSession{State: gopowerstore.RS_STATE_SYNCHRONIZING}, nil) + gopowerstore.ReplicationSession{State: gopowerstore.RsStateSynchronizing}, nil) req := new(csiext.GetStorageProtectionGroupStatusRequest) params := make(map[string]string) @@ -223,14 +222,14 @@ var _ = Describe("Replication", func() { req.ProtectionGroupAttributes = params res, err := ctrlSvc.GetStorageProtectionGroupStatus(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res.Status.State).To(Equal( + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res.Status.State).To(gomega.Equal( csiext.StorageProtectionGroupStatus_SYNC_IN_PROGRESS, )) }) - It("should return 'sync in progress' status (if initializing)", func() { + ginkgo.It("should return 'sync in progress' status (if initializing)", func() { clientMock.On("GetReplicationSessionByLocalResourceID", mock.Anything, mock.Anything).Return( - gopowerstore.ReplicationSession{State: gopowerstore.RS_STATE_INITIALIZING}, nil) + gopowerstore.ReplicationSession{State: gopowerstore.RsStateInitializing}, nil) req := new(csiext.GetStorageProtectionGroupStatusRequest) params := make(map[string]string) @@ -238,17 +237,17 @@ var _ = Describe("Replication", func() { req.ProtectionGroupAttributes = params res, err := ctrlSvc.GetStorageProtectionGroupStatus(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res.Status.State).To(Equal( + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res.Status.State).To(gomega.Equal( csiext.StorageProtectionGroupStatus_SYNC_IN_PROGRESS, )) }) }) - When("getting storage protection group status and state is error", func() { - It("should return invalid status", func() { + ginkgo.When("getting storage protection group status and state is error", func() { + ginkgo.It("should return invalid status", func() { clientMock.On("GetReplicationSessionByLocalResourceID", mock.Anything, mock.Anything).Return( - gopowerstore.ReplicationSession{State: gopowerstore.RS_STATE_ERROR}, nil) + gopowerstore.ReplicationSession{State: gopowerstore.RsStateError}, nil) req := new(csiext.GetStorageProtectionGroupStatusRequest) params := make(map[string]string) @@ -256,15 +255,15 @@ var _ = Describe("Replication", func() { req.ProtectionGroupAttributes = params res, err := ctrlSvc.GetStorageProtectionGroupStatus(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res.Status.State).To(Equal( + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res.Status.State).To(gomega.Equal( csiext.StorageProtectionGroupStatus_INVALID, )) }) }) - When("getting storage protection group status and state does not match with known protection group states", func() { - It("should return unknown status", func() { + ginkgo.When("getting storage protection group status and state does not match with known protection group states", func() { + ginkgo.It("should return unknown status", func() { clientMock.On("GetReplicationSessionByLocalResourceID", mock.Anything, mock.Anything).Return( gopowerstore.ReplicationSession{}, nil) @@ -274,46 +273,44 @@ var _ = Describe("Replication", func() { req.ProtectionGroupAttributes = params res, err := ctrlSvc.GetStorageProtectionGroupStatus(context.Background(), req) - Expect(err).To(BeNil()) - Expect(res.Status.State).To(Equal( + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res.Status.State).To(gomega.Equal( csiext.StorageProtectionGroupStatus_UNKNOWN, )) }) }) - When("GlobalID is missing", func() { - It("should fail", func() { - + ginkgo.When("GlobalID is missing", func() { + ginkgo.It("should fail", func() { req := new(csiext.GetStorageProtectionGroupStatusRequest) res, err := ctrlSvc.GetStorageProtectionGroupStatus(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To( - ContainSubstring("missing globalID in protection group attributes"), + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To( + gomega.ContainSubstring("missing globalID in protection group attributes"), ) }) }) - When("Array with specified globalID couldn't be found", func() { - It("should fail", func() { - + ginkgo.When("Array with specified globalID couldn't be found", func() { + ginkgo.It("should fail", func() { req := new(csiext.GetStorageProtectionGroupStatusRequest) params := make(map[string]string) params["globalID"] = "SOMETHING WRONG" req.ProtectionGroupAttributes = params res, err := ctrlSvc.GetStorageProtectionGroupStatus(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To( - ContainSubstring("can't find array with global id"), + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To( + gomega.ContainSubstring("can't find array with global id"), ) }) }) - When("Invalid client response", func() { - It("should fail", func() { + ginkgo.When("Invalid client response", func() { + ginkgo.It("should fail", func() { clientMock.On("GetReplicationSessionByLocalResourceID", mock.Anything, mock.Anything).Return( gopowerstore.ReplicationSession{}, status.Errorf(codes.InvalidArgument, "Invalid client response")) @@ -323,110 +320,104 @@ var _ = Describe("Replication", func() { req.ProtectionGroupAttributes = params res, err := ctrlSvc.GetStorageProtectionGroupStatus(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To( - ContainSubstring("Invalid client response"), + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To( + gomega.ContainSubstring("Invalid client response"), ) }) }) }) - Describe("calling ExecuteAction()", func() { - When("action is RS_ACTION_RESUME and state is OK", func() { - It("return nil", func() { + ginkgo.Describe("calling ExecuteAction()", func() { + ginkgo.When("action is RsActionResume and state is OK", func() { + ginkgo.It("return nil", func() { clientMock.On("ExecuteActionOnReplicationSession", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("", nil) session := gopowerstore.ReplicationSession{ID: "test", State: "OK"} - action := gopowerstore.RS_ACTION_RESUME + action := gopowerstore.RsActionResume failoverParams := gopowerstore.FailoverParams{} - err := controller.ExecuteAction(&session, clientMock, action, &failoverParams) + err := ExecuteAction(&session, clientMock, action, &failoverParams) - Expect(err).To(BeNil()) + gomega.Expect(err).To(gomega.BeNil()) }) }) - When("action is RS_ACTION_REPROTECT and state is not OK", func() { - It("return nil", func() { + ginkgo.When("action is RsActionReprotect and state is not OK", func() { + ginkgo.It("return nil", func() { clientMock.On("ExecuteActionOnReplicationSession", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("", nil) session := gopowerstore.ReplicationSession{ID: "test", State: "OK"} - action := gopowerstore.RS_ACTION_REPROTECT + action := gopowerstore.RsActionReprotect failoverParams := gopowerstore.FailoverParams{} - err := controller.ExecuteAction(&session, clientMock, action, &failoverParams) - - Expect(err).To(BeNil()) + err := ExecuteAction(&session, clientMock, action, &failoverParams) + gomega.Expect(err).To(gomega.BeNil()) }) }) - When("action is RS_ACTION_PAUSE and state is Paused", func() { - It("return nil", func() { + ginkgo.When("action is RsActionPause and state is Paused", func() { + ginkgo.It("return nil", func() { clientMock.On("ExecuteActionOnReplicationSession", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("", nil) session := gopowerstore.ReplicationSession{ID: "test", State: "Paused"} - action := gopowerstore.RS_ACTION_PAUSE + action := gopowerstore.RsActionPause failoverParams := gopowerstore.FailoverParams{} - err := controller.ExecuteAction(&session, clientMock, action, &failoverParams) + err := ExecuteAction(&session, clientMock, action, &failoverParams) - Expect(err).To(BeNil()) + gomega.Expect(err).To(gomega.BeNil()) }) }) - When("action is RS_ACTION_FAILOVER and state is Failing_Over", func() { - It("return nil", func() { + ginkgo.When("action is RsActionFailover and state is Failing_Over", func() { + ginkgo.It("return nil", func() { clientMock.On("ExecuteActionOnReplicationSession", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("", nil) session := gopowerstore.ReplicationSession{ID: "test", State: "Failing_Over"} - action := gopowerstore.RS_ACTION_FAILOVER + action := gopowerstore.RsActionFailover failoverParams := gopowerstore.FailoverParams{} - err := controller.ExecuteAction(&session, clientMock, action, &failoverParams) - - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To( - ContainSubstring("Execute action: RS (test) is still executing previous action")) + err := ExecuteAction(&session, clientMock, action, &failoverParams) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To( + gomega.ContainSubstring("Execute action: RS (test) is still executing previous action")) }) }) - When("action is RS_ACTION_FAILOVER and state is Failed_Over", func() { - It("return nil", func() { + ginkgo.When("action is RsActionFailover and state is Failed_Over", func() { + ginkgo.It("return nil", func() { clientMock.On("ExecuteActionOnReplicationSession", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("", nil) session := gopowerstore.ReplicationSession{ID: "test", State: "Failed_Over"} - action := gopowerstore.RS_ACTION_FAILOVER + action := gopowerstore.RsActionFailover failoverParams := gopowerstore.FailoverParams{} - err := controller.ExecuteAction(&session, clientMock, action, &failoverParams) - - Expect(err).To(BeNil()) + err := ExecuteAction(&session, clientMock, action, &failoverParams) + gomega.Expect(err).To(gomega.BeNil()) }) - }) - Describe("calling DeleteLocalVolume()", func() { - When("Volume ID is missing", func() { - It("should fail", func() { + ginkgo.Describe("calling DeleteLocalVolume()", func() { + ginkgo.When("Volume ID is missing", func() { + ginkgo.It("should fail", func() { req := new(csiext.DeleteLocalVolumeRequest) res, err := ctrlSvc.DeleteLocalVolume(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To( - ContainSubstring("can't delete volume of improper handle format")) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To( + gomega.ContainSubstring("can't delete volume of improper handle format")) }) }) - When("Array with specified globalID couldn't be found", func() { - It("should fail", func() { - + ginkgo.When("Array with specified globalID couldn't be found", func() { + ginkgo.It("should fail", func() { req := new(csiext.DeleteLocalVolumeRequest) handle := "valid-id/SOMETHING-WRONG/iscsi" req.VolumeHandle = handle res, err := ctrlSvc.DeleteLocalVolume(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To( - ContainSubstring("can't find array with global ID")) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To( + gomega.ContainSubstring("can't find array with global ID")) }) }) - When("the volume cannot be found on the powerstore array", func() { - It("should fail with 'not found'", func() { - + ginkgo.When("the volume cannot be found on the powerstore array", func() { + ginkgo.It("should fail with 'not found'", func() { // GetVolume should return a NotFound error. clientMock.On("GetVolume", mock.Anything, validBaseVolID).Return( gopowerstore.Volume{}, gopowerstore.WrapErr(gopowerstore.NewNotFoundError()), @@ -437,12 +428,12 @@ var _ = Describe("Replication", func() { } res, err := ctrlSvc.DeleteLocalVolume(context.Background(), req) - Expect(res).To(Equal( + gomega.Expect(res).To(gomega.Equal( &csiext.DeleteLocalVolumeResponse{}, )) - Expect(err).To(BeNil()) + gomega.Expect(err).To(gomega.BeNil()) }) - It("should fail to get the volume", func() { + ginkgo.It("should fail to get the volume", func() { // GetVolume should return a non-nil error, and not be a NotFoundError clientMock.On("GetVolume", mock.Anything, validBaseVolID).Return( gopowerstore.Volume{}, gopowerstore.WrapErr(gopowerstore.NewAPIError()), @@ -453,43 +444,142 @@ var _ = Describe("Replication", func() { } res, err := ctrlSvc.DeleteLocalVolume(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(ContainSubstring( + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring( "Unable to get volume for deletion", )) }) }) + ginkgo.When("the volume group cannot be found", func() { + ginkgo.It("should fail to get the volume group", func() { + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID}, nil) + clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID). + Return(gopowerstore.VolumeGroups{}, gopowerstore.WrapErr(gopowerstore.NewAPIError())) + + req := &csiext.DeleteLocalVolumeRequest{ + VolumeHandle: validBaseVolID + "/" + firstValidID + "/" + "iscsi", + } + res, err := ctrlSvc.DeleteLocalVolume(context.Background(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + }) + }) + ginkgo.When("the volume is part of volume group", func() { + ginkgo.It("should fail to delete the volume", func() { + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID}, nil) + clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID). + Return(gopowerstore.VolumeGroups{VolumeGroup: []gopowerstore.VolumeGroup{{ID: validGroupID}}}, nil) + + req := &csiext.DeleteLocalVolumeRequest{ + VolumeHandle: validBaseVolID + "/" + firstValidID + "/" + "iscsi", + } + res, err := ctrlSvc.DeleteLocalVolume(context.Background(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring( + "Unable to delete volume", + )) + }) + }) + ginkgo.When("the volume is still protected", func() { + ginkgo.It("should fail to delete the volume", func() { + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, ProtectionPolicyID: validPolicyID}, nil) + clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID). + Return(gopowerstore.VolumeGroups{}, nil) + + req := &csiext.DeleteLocalVolumeRequest{ + VolumeHandle: validBaseVolID + "/" + firstValidID + "/" + "iscsi", + } + res, err := ctrlSvc.DeleteLocalVolume(context.Background(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring( + "Unable to delete volume", + )) + }) + }) + ginkgo.When("the delete volume call failed", func() { + ginkgo.It("should fail to delete the volume", func() { + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID}, nil) + clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID). + Return(gopowerstore.VolumeGroups{}, nil) + clientMock.On("DeleteVolume", + mock.Anything, + mock.AnythingOfType("*gopowerstore.VolumeDelete"), + validBaseVolID). + Return(gopowerstore.EmptyResponse(""), gopowerstore.NewAPIError()) + + req := &csiext.DeleteLocalVolumeRequest{ + VolumeHandle: validBaseVolID + "/" + firstValidID + "/" + "iscsi", + } + res, err := ctrlSvc.DeleteLocalVolume(context.Background(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring( + "Unable to delete volume", + )) + }) + }) + ginkgo.When("the delete local volume is requested", func() { + ginkgo.It("should succeed to delete the local volume", func() { + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID}, nil) + clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID). + Return(gopowerstore.VolumeGroups{}, nil) + clientMock.On("DeleteVolume", + mock.Anything, + mock.AnythingOfType("*gopowerstore.VolumeDelete"), + validBaseVolID). + Return(gopowerstore.EmptyResponse(""), nil) + + req := &csiext.DeleteLocalVolumeRequest{ + VolumeHandle: validBaseVolID + "/" + firstValidID + "/" + "iscsi", + } + res, err := ctrlSvc.DeleteLocalVolume(context.Background(), req) + + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).ToNot(gomega.BeNil()) + }) + }) }) - Describe("calling DeleteStorageProtectionGroup()", func() { - When("GlobalID is missing", func() { - It("should fail", func() { + + ginkgo.Describe("calling DeleteStorageProtectionGroup()", func() { + ginkgo.When("GlobalID is missing", func() { + ginkgo.It("should fail", func() { req := new(csiext.DeleteStorageProtectionGroupRequest) res, err := ctrlSvc.DeleteStorageProtectionGroup(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To( - ContainSubstring("missing globalID in protection group attributes")) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To( + gomega.ContainSubstring("missing globalID in protection group attributes")) }) }) - When("Array with specified globalID couldn't be found", func() { - It("should fail", func() { - + ginkgo.When("Array with specified globalID couldn't be found", func() { + ginkgo.It("should fail", func() { req := new(csiext.DeleteStorageProtectionGroupRequest) params := make(map[string]string) params["globalID"] = "SOMETHING WRONG" req.ProtectionGroupAttributes = params res, err := ctrlSvc.DeleteStorageProtectionGroup(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To( - ContainSubstring("can't find array with global id")) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To( + gomega.ContainSubstring("can't find array with global id")) }) }) - When("can't get volume group", func() { - It("should fail", func() { + ginkgo.When("can't get volume group", func() { + ginkgo.It("should fail", func() { clientMock.On("GetVolumeGroup", mock.Anything, mock.Anything).Return( gopowerstore.VolumeGroup{}, gopowerstore.APIError{ErrorMsg: &api.ErrorMsg{StatusCode: http.StatusBadRequest}}) @@ -502,14 +592,14 @@ var _ = Describe("Replication", func() { res, err := ctrlSvc.DeleteStorageProtectionGroup(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To( - ContainSubstring("Error: Unable to get Volume Group")) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To( + gomega.ContainSubstring("Error: Unable to get Volume Group")) }) }) - When("can't get volume group name", func() { - It("should fail", func() { + ginkgo.When("can't get volume group name", func() { + ginkgo.It("should fail", func() { clientMock.On("GetVolumeGroup", mock.Anything, mock.Anything).Return( gopowerstore.VolumeGroup{}, gopowerstore.APIError{ErrorMsg: &api.ErrorMsg{StatusCode: http.StatusNotFound}}) req := new(csiext.DeleteStorageProtectionGroupRequest) @@ -521,14 +611,14 @@ var _ = Describe("Replication", func() { res, err := ctrlSvc.DeleteStorageProtectionGroup(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To( - ContainSubstring("Error: Unable to get volume group name")) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To( + gomega.ContainSubstring("Error: Unable to get volume group name")) }) }) - When("Can't unassign the protection policy from volume group", func() { - It("should fail", func() { + ginkgo.When("Can't unassign the protection policy from volume group", func() { + ginkgo.It("should fail", func() { vg := gopowerstore.VolumeGroup{} vg.ProtectionPolicyID = validPolicyID vg.ID = validGroupID @@ -546,14 +636,14 @@ var _ = Describe("Replication", func() { res, err := ctrlSvc.DeleteStorageProtectionGroup(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To( - ContainSubstring("Error: Unable to un-assign PP from Volume Group")) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To( + gomega.ContainSubstring("Error: Unable to un-assign PP from Volume Group")) }) }) - When("Can't delete volume group", func() { - It("should fail", func() { + ginkgo.When("Can't delete volume group", func() { + ginkgo.It("should fail", func() { vg := gopowerstore.VolumeGroup{} vg.ProtectionPolicyID = "" vg.ID = validGroupID @@ -570,14 +660,14 @@ var _ = Describe("Replication", func() { req.ProtectionGroupAttributes = params res, err := ctrlSvc.DeleteStorageProtectionGroup(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To( - ContainSubstring("Error: : Unable to delete Volume Group")) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To( + gomega.ContainSubstring("Error: Unable to delete Volume Group")) }) }) - When("Can't get the protection policy", func() { - It("should fail", func() { + ginkgo.When("Can't get the protection policy", func() { + ginkgo.It("should fail", func() { vg := gopowerstore.VolumeGroup{} vg.ProtectionPolicyID = validPolicyID vg.ID = validGroupID @@ -605,15 +695,15 @@ var _ = Describe("Replication", func() { req.ProtectionGroupAttributes = params res, err := ctrlSvc.DeleteStorageProtectionGroup(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To( - ContainSubstring("Error: Unable to get the PP")) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To( + gomega.ContainSubstring("Error: Unable to get protection policy")) }) }) - When("The replication rule couldn't be found", func() { - It("should fail", func() { + ginkgo.When("The replication rule couldn't be found", func() { + ginkgo.It("should fail", func() { vg := gopowerstore.VolumeGroup{} vg.ProtectionPolicyID = validPolicyID vg.ID = validGroupID @@ -643,14 +733,14 @@ var _ = Describe("Replication", func() { req.ProtectionGroupAttributes = params res, err := ctrlSvc.DeleteStorageProtectionGroup(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To( - ContainSubstring("Error: RR not found")) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To( + gomega.ContainSubstring("Error: Unable to get replication rule")) }) }) - When("The replication rule can't be deleted", func() { - It("should fail", func() { + ginkgo.When("The replication rule can't be deleted", func() { + ginkgo.It("should fail", func() { vg := gopowerstore.VolumeGroup{} vg.ProtectionPolicyID = validPolicyID vg.ID = validGroupID @@ -684,19 +774,31 @@ var _ = Describe("Replication", func() { res, err := ctrlSvc.DeleteStorageProtectionGroup(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To( - ContainSubstring("Error: Unable to delete replication rule")) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To( + gomega.ContainSubstring("Error: Unable to delete replication rule")) + }) + }) + ginkgo.When("NFS context is detected via NasServerID", func() { + ginkgo.It("should skip deletion logic and return success", func() { + req := new(csiext.DeleteStorageProtectionGroupRequest) + params := make(map[string]string) + params["globalID"] = firstValidID + params["NasServerID"] = "nas-server-id" + req.ProtectionGroupAttributes = params + res, err := ctrlSvc.DeleteStorageProtectionGroup(context.Background(), req) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).ToNot(gomega.BeNil()) }) }) }) - Describe("calling GetReplicationCapabilities()", func() { - When("basic parameters are declared", func() { - It("should pass", func() { + ginkgo.Describe("calling GetReplicationCapabilities()", func() { + ginkgo.When("basic parameters are declared", func() { + ginkgo.It("should pass", func() { context := context.Background() req := new(csiext.GetReplicationCapabilityRequest) - var rep = new(csiext.GetReplicationCapabilityResponse) + rep := new(csiext.GetReplicationCapabilityResponse) rep.Capabilities = []*csiext.ReplicationCapability{ { Type: &csiext.ReplicationCapability_Rpc{ @@ -767,17 +869,15 @@ var _ = Describe("Replication", func() { }, } res, err := ctrlSvc.GetReplicationCapabilities(context, req) - Expect(err).To(BeNil()) - Expect(res).To(Equal(rep)) - + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(rep)) }) }) }) - Describe("calling ExecuteAction()", func() { - When("action is unknown", func() { - It("should fail", func() { - + ginkgo.Describe("calling ExecuteAction()", func() { + ginkgo.When("action is unknown", func() { + ginkgo.It("should fail", func() { action := &csiext.Action{ ActionTypes: csiext.ActionTypes_UNKNOWN_ACTION, } @@ -792,13 +892,11 @@ var _ = Describe("Replication", func() { RemoteProtectionGroupAttributes: nil, } _, err := ctrlSvc.ExecuteAction(context.Background(), req) - Expect(err).NotTo(BeNil()) - + gomega.Expect(err).NotTo(gomega.BeNil()) }) }) - When("Array can't be found", func() { - It("should fail", func() { - + ginkgo.When("Array can't be found", func() { + ginkgo.It("should fail", func() { action := &csiext.Action{ ActionTypes: csiext.ActionTypes_FAILOVER_REMOTE, } @@ -818,15 +916,13 @@ var _ = Describe("Replication", func() { } _, err := ctrlSvc.ExecuteAction(context.Background(), req) - Expect(err).NotTo(BeNil()) - Expect(err.Error()).To( - ContainSubstring("can't find array with global id ")) - + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To( + gomega.ContainSubstring("can't find array with global id ")) }) }) - When("the action is not supported", func() { - It("should fail", func() { - + ginkgo.When("the action is not supported", func() { + ginkgo.It("should fail", func() { action := &csiext.Action{ ActionTypes: csiext.ActionTypes_UNKNOWN_ACTION, } @@ -845,16 +941,14 @@ var _ = Describe("Replication", func() { RemoteProtectionGroupAttributes: nil, } _, err := ctrlSvc.ExecuteAction(context.Background(), req) - Expect(err).NotTo(BeNil()) - Expect(err.Error()).To( - ContainSubstring("The requested action does not match with supported actions")) - + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To( + gomega.ContainSubstring("The requested action does not match with supported actions")) }) }) - When("the replication session is executing previous action. the action type is unplanned failover local", func() { - It("should fail", func() { - + ginkgo.When("the replication session is executing previous action. the action type is unplanned failover local", func() { + ginkgo.It("should fail", func() { action := &csiext.Action{ ActionTypes: csiext.ActionTypes_UNPLANNED_FAILOVER_LOCAL, } @@ -876,15 +970,13 @@ var _ = Describe("Replication", func() { } _, err := ctrlSvc.ExecuteAction(context.Background(), req) - Expect(err).NotTo(BeNil()) - Expect(err.Error()).To( - ContainSubstring("Execute action: RS (test) is still executing previous action")) - + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To( + gomega.ContainSubstring("Execute action: RS (test) is still executing previous action")) }) }) - When("the action type is suspend", func() { - It("pass", func() { - + ginkgo.When("the action type is suspend", func() { + ginkgo.It("pass", func() { action := &csiext.Action{ ActionTypes: csiext.ActionTypes_SUSPEND, } @@ -906,13 +998,11 @@ var _ = Describe("Replication", func() { } _, err := ctrlSvc.ExecuteAction(context.Background(), req) - Expect(err).To(BeNil()) - + gomega.Expect(err).To(gomega.BeNil()) }) }) - When("the replication session is executing previous action. the action type is failover remote.", func() { - It("should fail", func() { - + ginkgo.When("the replication session is executing previous action. the action type is failover remote.", func() { + ginkgo.It("should fail", func() { action := &csiext.Action{ ActionTypes: csiext.ActionTypes_FAILOVER_REMOTE, } @@ -934,21 +1024,19 @@ var _ = Describe("Replication", func() { } _, err := ctrlSvc.ExecuteAction(context.Background(), req) - Expect(err).NotTo(BeNil()) - Expect(err.Error()).To( - ContainSubstring("Execute action: RS (test) is still executing previous action")) - + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To( + gomega.ContainSubstring("Execute action: RS (test) is still executing previous action")) }) }) - When("the replication session can't be modified due to sync action type.", func() { - It("should fail", func() { - + ginkgo.When("the replication session can't be modified due to sync action type.", func() { + ginkgo.It("should fail", func() { action := &csiext.Action{ ActionTypes: csiext.ActionTypes_SYNC, } session := gopowerstore.ReplicationSession{ID: "test", State: "Failed_Over"} - clientMock.On("ExecuteActionOnReplicationSession", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(gopowerstore.EmptyResponse(""), gopowerstore.APIError{ErrorMsg: &api.ErrorMsg{StatusCode: http.StatusNotFound}}) + clientMock.On("ExecuteActionOnReplicationSession", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(gopowerstore.EmptyResponse(""), gopowerstore.APIError{ErrorMsg: &api.ErrorMsg{StatusCode: http.StatusBadRequest}}) clientMock.On("GetReplicationSessionByLocalResourceID", mock.Anything, mock.Anything).Return(session, nil) params := make(map[string]string) @@ -964,15 +1052,13 @@ var _ = Describe("Replication", func() { } _, err := ctrlSvc.ExecuteAction(context.Background(), req) - Expect(err).NotTo(BeNil()) - Expect(err.Error()).To( - ContainSubstring("Execute action: Failed to modify RS (test) - Error ()")) - + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To( + gomega.ContainSubstring("Execute action: Failed to modify RS (test) - Error ()")) }) }) - When("the action type is resume", func() { - It("should pass", func() { - + ginkgo.When("the action type is resume", func() { + ginkgo.It("should pass", func() { action := &csiext.Action{ ActionTypes: csiext.ActionTypes_RESUME, } @@ -994,13 +1080,11 @@ var _ = Describe("Replication", func() { } _, err := ctrlSvc.ExecuteAction(context.Background(), req) - Expect(err).To(BeNil()) - + gomega.Expect(err).To(gomega.BeNil()) }) }) - When("the action type is sync", func() { - It("should pass", func() { - + ginkgo.When("the action type is sync", func() { + ginkgo.It("should pass", func() { action := &csiext.Action{ ActionTypes: csiext.ActionTypes_SYNC, } @@ -1022,13 +1106,11 @@ var _ = Describe("Replication", func() { } _, err := ctrlSvc.ExecuteAction(context.Background(), req) - Expect(err).To(BeNil()) - + gomega.Expect(err).To(gomega.BeNil()) }) }) - When("the action type is reprotect local", func() { - It("should pass", func() { - + ginkgo.When("the action type is reprotect local", func() { + ginkgo.It("should pass", func() { action := &csiext.Action{ ActionTypes: csiext.ActionTypes_REPROTECT_LOCAL, } @@ -1050,12 +1132,9 @@ var _ = Describe("Replication", func() { } _, err := ctrlSvc.ExecuteAction(context.Background(), req) - Expect(err).To(BeNil()) - + gomega.Expect(err).To(gomega.BeNil()) }) }) - }) - }) }) diff --git a/pkg/controller/snapshotter.go b/pkg/controller/snapshotter.go index 5b6a872f..1cd264fd 100644 --- a/pkg/controller/snapshotter.go +++ b/pkg/controller/snapshotter.go @@ -81,7 +81,7 @@ func (f FilesystemSnapshot) GetID() string { // GetSourceID returns ID of the volume/fs that snapshot was created from func (f FilesystemSnapshot) GetSourceID() string { - return f.ParentId + return f.ParentID } // GetSize returns current size of the snapshot @@ -103,8 +103,7 @@ type VolumeSnapshotter interface { } // SCSISnapshotter is a implementation of VolumeSnapshotter for SCSI based (FC, iSCSI) volumes -type SCSISnapshotter struct { -} +type SCSISnapshotter struct{} // GetExistingSnapshot queries storage array if given snapshot of the Volume already exists func (*SCSISnapshotter) GetExistingSnapshot(ctx context.Context, snapName string, client gopowerstore.Client) (GeneralSnapshot, error) { @@ -126,8 +125,7 @@ func (*SCSISnapshotter) Create(ctx context.Context, snapName string, sourceID st } // NfsSnapshotter is a implementation of VolumeSnapshotter for NFS volumes -type NfsSnapshotter struct { -} +type NfsSnapshotter struct{} // GetExistingSnapshot queries storage array if given snapshot of the FileSystem already exists func (*NfsSnapshotter) GetExistingSnapshot(ctx context.Context, snapName string, client gopowerstore.Client) (GeneralSnapshot, error) { diff --git a/pkg/helpers/utils.go b/pkg/helpers/utils.go new file mode 100644 index 00000000..d9f63802 --- /dev/null +++ b/pkg/helpers/utils.go @@ -0,0 +1,89 @@ +/* + * + * Copyright © 2021-2025 Dell Inc. or its subsidiaries. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package helpers + +import ( + "errors" + "net" + + "github.com/dell/csmlog" +) + +// Instantiate csmlog at package level +var log = csmlog.GetLogger() + +// InterfaceProvider allows mocking net.Interfaces +type InterfaceProvider interface { + Interfaces() ([]net.Interface, error) +} + +// AddrProvider allows mocking iface.Addrs() +type AddrProvider interface { + Addrs(iface net.Interface) ([]net.Addr, error) +} + +// Default providers for production use +type defaultProvider struct{} + +func (p defaultProvider) Interfaces() ([]net.Interface, error) { + return net.Interfaces() +} + +func (p defaultProvider) Addrs(iface net.Interface) ([]net.Addr, error) { + return iface.Addrs() +} + +func GetNodeIP() (net.IP, error) { + return GetNodeIPWithProvider(defaultProvider{}, defaultProvider{}) +} + +// GetNodeIP retrieves the first valid non-loopback IPv4 addres +func GetNodeIPWithProvider(ifProvider InterfaceProvider, addrProvider AddrProvider) (net.IP, error) { + interfaces, err := ifProvider.Interfaces() + if err != nil { + return nil, err + } + + for _, iface := range interfaces { + if iface.Flags&net.FlagUp == 0 || iface.Flags&net.FlagLoopback != 0 { + continue + } + + addrs, err := addrProvider.Addrs(iface) + if err != nil { + continue + } + + for _, addr := range addrs { + var ip net.IP + switch v := addr.(type) { + case *net.IPNet: + ip = v.IP + case *net.IPAddr: + ip = v.IP + } + + if ip == nil || ip.IsLoopback() || ip.To4() == nil { + continue + } + return ip, nil + } + } + return nil, errors.New("no valid non-loopback IPv4 address found") +} diff --git a/pkg/helpers/utils_test.go b/pkg/helpers/utils_test.go new file mode 100644 index 00000000..ff7e5220 --- /dev/null +++ b/pkg/helpers/utils_test.go @@ -0,0 +1,200 @@ +/* + * + * Copyright © 2021-2025 Dell Inc. or its subsidiaries. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package helpers + +import ( + "errors" + "net" + "testing" + + "github.com/stretchr/testify/assert" +) + +// Mock implementations +type mockInterfaceProvider struct { + interfaces []net.Interface + err error +} + +func (m mockInterfaceProvider) Interfaces() ([]net.Interface, error) { + return m.interfaces, m.err +} + +type mockAddrProvider struct { + addrMap map[string][]net.Addr + errMap map[string]error +} + +func (m mockAddrProvider) Addrs(iface net.Interface) ([]net.Addr, error) { + if err, ok := m.errMap[iface.Name]; ok { + return nil, err + } + return m.addrMap[iface.Name], nil +} + +func TestGetNodeIP_AllCases(t *testing.T) { + tests := []struct { + name string + ifProvider InterfaceProvider + addrProvider AddrProvider + wantErr bool + expectedIP string + }{ + { + name: "error from net.Interfaces", + ifProvider: mockInterfaceProvider{ + err: errors.New("interface error"), + }, + addrProvider: mockAddrProvider{}, + wantErr: true, + }, + { + name: "no interfaces returned", + ifProvider: mockInterfaceProvider{ + interfaces: []net.Interface{}, + }, + addrProvider: mockAddrProvider{}, + wantErr: true, + }, + { + name: "all interfaces are down or loopback", + ifProvider: mockInterfaceProvider{ + interfaces: []net.Interface{ + {Name: "lo", Flags: net.FlagLoopback}, + {Name: "eth0", Flags: 0}, + }, + }, + addrProvider: mockAddrProvider{}, + wantErr: true, + }, + { + name: "interface returns error on Addrs()", + ifProvider: mockInterfaceProvider{ + interfaces: []net.Interface{ + {Name: "eth0", Flags: net.FlagUp}, + }, + }, + addrProvider: mockAddrProvider{ + errMap: map[string]error{ + "eth0": errors.New("addr error"), + }, + }, + wantErr: true, + }, + { + name: "addresses are loopback or non-IPv4", + ifProvider: mockInterfaceProvider{ + interfaces: []net.Interface{ + {Name: "eth0", Flags: net.FlagUp}, + }, + }, + addrProvider: mockAddrProvider{ + addrMap: map[string][]net.Addr{ + "eth0": { + &net.IPNet{IP: net.ParseIP("127.0.0.1")}, + &net.IPNet{IP: net.ParseIP("::1")}, + }, + }, + }, + wantErr: true, + }, + { + name: "valid IPv4 address found", + ifProvider: mockInterfaceProvider{ + interfaces: []net.Interface{ + {Name: "eth0", Flags: net.FlagUp}, + }, + }, + addrProvider: mockAddrProvider{ + addrMap: map[string][]net.Addr{ + "eth0": { + &net.IPNet{IP: net.ParseIP("192.168.1.100")}, + }, + }, + }, + wantErr: false, + expectedIP: "192.168.1.100", + }, + { + name: "valid IP found in second interface", + ifProvider: mockInterfaceProvider{ + interfaces: []net.Interface{ + {Name: "lo", Flags: net.FlagLoopback}, + {Name: "eth1", Flags: net.FlagUp}, + }, + }, + addrProvider: mockAddrProvider{ + addrMap: map[string][]net.Addr{ + "eth1": { + &net.IPAddr{IP: net.ParseIP("10.0.0.5")}, + }, + }, + }, + wantErr: false, + expectedIP: "10.0.0.5", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ip, err := GetNodeIPWithProvider(tt.ifProvider, tt.addrProvider) + if tt.wantErr { + assert.Error(t, err) + assert.Nil(t, ip) + } else { + assert.NoError(t, err) + assert.NotNil(t, ip) + assert.Equal(t, tt.expectedIP, ip.String()) + } + }) + } +} + +func TestDefaultProvider_Interfaces(t *testing.T) { + provider := defaultProvider{} + + interfaces, err := provider.Interfaces() + assert.NoError(t, err) + assert.NotNil(t, interfaces) + assert.Greater(t, len(interfaces), 0, "Expected at least one network interface") +} + +func TestDefaultProvider_Addrs(t *testing.T) { + provider := defaultProvider{} + + interfaces, err := provider.Interfaces() + assert.NoError(t, err) + assert.NotEmpty(t, interfaces) + + for _, iface := range interfaces { + _, err := provider.Addrs(iface) + // Some interfaces may not have addresses, but the call shouldn't fail + assert.NoError(t, err) + } +} + +func TestGetNodeIP_Integration(t *testing.T) { + ip, err := GetNodeIP() + if err != nil { + t.Logf("No valid IP found: %v", err) + } else { + assert.NotNil(t, ip) + t.Logf("Found IP: %s", ip.String()) + } +} diff --git a/pkg/common/envvars.go b/pkg/identifiers/envvars.go similarity index 67% rename from pkg/common/envvars.go rename to pkg/identifiers/envvars.go index f040691d..3063cab6 100644 --- a/pkg/common/envvars.go +++ b/pkg/identifiers/envvars.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2021-2023 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2021-2024 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ * */ -package common +package identifiers const ( // EnvDriverName is the name of the csi driver (provisioner) @@ -30,7 +30,7 @@ const ( // node name EnvKubeNodeName = "X_CSI_POWERSTORE_KUBE_NODE_NAME" - //EnvKubeConfigPath indicates kubernetes configuration path that has to be used by CSI Driver + // EnvKubeConfigPath indicates kubernetes configuration path that has to be used by CSI Driver EnvKubeConfigPath = "KUBECONFIG" // EnvNodeNamePrefix is the name of the environment variable which stores prefix which will be @@ -44,10 +44,6 @@ const ( // to execute iSCSI commands EnvNodeChrootPath = "X_CSI_POWERSTORE_NODE_CHROOT_PATH" - // EnvCtrlRootPath is the name of the environment variable which store path to directory where - // the host root is mounted - EnvCtrlRootPath = "X_CSI_POWERSTORE_CTRL_ROOT_PATH" - // EnvTmpDir is the name of the environment variable which store path to the folder which will be used // for csi-powerstore temporary files EnvTmpDir = "X_CSI_POWERSTORE_TMP_DIR" // #nosec G101 @@ -72,6 +68,10 @@ const ( // Used to provide NFS volumes behind NAT EnvExternalAccess = "X_CSI_POWERSTORE_EXTERNAL_ACCESS" // #nosec G101 + // EnvExclusiveAccess indicates whether only externalAccess entries should be added to the NFS export. + // If true, node IP is excluded, and only IP/CIDR from externalAccess is used. + EnvExclusiveAccess = "X_CSI_POWERSTORE_EXCLUSIVE_ACCESS" + // EnvArrayConfigFilePath is filepath to powerstore arrays config file EnvArrayConfigFilePath = "X_CSI_POWERSTORE_CONFIG_PATH" @@ -96,21 +96,45 @@ const ( // EnvNfsAcls specifies acls to be set on NFS mount directory EnvNfsAcls = "X_CSI_NFS_ACLS" - // EnvK8sVisibilityAutoRegistration specifies if k8s cluster should be automatically registered to PowerStore Array - EnvK8sVisibilityAutoRegistration = "X_CSI_K8S_VISIBILITY_AUTO_REGISTRATION" - // EnvMetadataRetrieverEndpoint specifies the endpoint address for csi-metadata-retriever sidecar EnvMetadataRetrieverEndpoint = "CSI_RETRIEVER_ENDPOINT" // EnvAllowAutoRoundOffFilesystemSize specifies if auto round off minimum filesystem size is enabled EnvAllowAutoRoundOffFilesystemSize = "CSI_AUTO_ROUND_OFF_FILESYSTEM_SIZE" - //EnvPodmonEnabled indicates that podmon is enabled + // EnvPodmonEnabled indicates that podmon is enabled EnvPodmonEnabled = "X_CSI_PODMON_ENABLED" - //EnvPodmonAPIPORT indicates the port to be used for exposing podmon API health, ToDo: Rename to var EnvPodmonArrayConnectivityAPIPORT + // EnvPodmonAPIPORT indicates the port to be used for exposing podmon API health, ToDo: Rename to var EnvPodmonArrayConnectivityAPIPORT EnvPodmonAPIPORT = "X_CSI_PODMON_API_PORT" - //EnvPodmonArrayConnectivityPollRate indicates the polling frequency to check array connectivity + // EnvPodmonArrayConnectivityPollRate indicates the polling frequency to check array connectivity EnvPodmonArrayConnectivityPollRate = "X_CSI_PODMON_ARRAY_CONNECTIVITY_POLL_RATE" + + // EnvMultiNASThreshold specifies the failure threshold used to put NAS in cooldown. + EnvMultiNASFailureThreshold = "X_CSI_MULTI_NAS_FAILURE_THRESHOLD" + + // EnvMultiNASCooldownPeriod specifies the cooldown period for multiple NAS devices. + EnvMultiNASCooldownPeriod = "X_CSI_MULTI_NAS_COOLDOWN_PERIOD" + + // EnvDriverNamespace is the namespace where the powerstore driver is deployed + EnvDriverNamespace = "X_CSI_DRIVER_NAMESPACE" + + // EnvPowerstoreAPITimeout specifies the timeout for Powerstore REST API calls + EnvPowerstoreAPITimeout = "X_CSI_POWERSTORE_API_TIMEOUT" + + // EnvPodmonArrayConnectivityTimeout specifies the timeout for array connectivity for podmon + EnvPodmonArrayConnectivityTimeout = "X_CSI_PODMON_ARRAY_CONNECTIVITY_TIMEOUT" + + // EnvVolumeDisconnectMaxRetries specifies the maximum number of retry attempts for volume disconnection + EnvVolumeDisconnectMaxRetries = "X_CSI_VOLUME_DISCONNECT_MAX_RETRIES" + + // EnvVolumeDisconnectRetryInterval specifies the wait time (in seconds) between volume disconnection retries + EnvVolumeDisconnectRetryInterval = "X_CSI_VOLUME_DISCONNECT_RETRY_INTERVAL" + + // EnvVolumeDisconnectTimeoutSeconds specifies the timeout duration (in seconds) for each volume disconnection attempt + EnvVolumeDisconnectTimeoutSeconds = "X_CSI_VOLUME_DISCONNECT_TIMEOUT_SECONDS" + + // EnvCSMDREnabled indicates if CSM-DR is enabled + EnvCSMDREnabled = "X_CSM_DR_ENABLED" ) diff --git a/pkg/common/fs/fs.go b/pkg/identifiers/fs/fs.go similarity index 89% rename from pkg/common/fs/fs.go rename to pkg/identifiers/fs/fs.go index 1431fd1c..9206d165 100644 --- a/pkg/common/fs/fs.go +++ b/pkg/identifiers/fs/fs.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2021-2023 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2021-2025 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,13 +28,17 @@ import ( "os" "os/exec" "path/filepath" + "strings" "syscall" "time" + "github.com/dell/csmlog" "github.com/dell/gofsutil" - log "github.com/sirupsen/logrus" ) +// Instantiate csmlog on a package level +var log = csmlog.GetLogger() + // A FileInfo describes a file and is returned by Stat and Lstat. type FileInfo interface { Name() string // base name of the file @@ -69,7 +73,7 @@ type Interface interface { // wrapper ParseProcMounts(ctx context.Context, content io.Reader) ([]gofsutil.Info, error) MkFileIdempotent(path string) (bool, error) - //Network + // Network NetDial(endpoint string) (net.Conn, error) } @@ -98,6 +102,7 @@ type UtilInterface interface { ResizeMultipath(ctx context.Context, deviceName string) error FindFSType(ctx context.Context, mountpoint string) (fsType string, err error) GetMpathNameFromDevice(ctx context.Context, device string) (string, error) + GetNVMeController(device string) (string, error) } // Fs implementation of FsInterface that uses default os/file calls @@ -116,8 +121,8 @@ func (fs *Fs) OpenFile(name string, flag int, perm os.FileMode) (*os.File, error } // WriteString is a wrapper of file.WriteString -func (fs *Fs) WriteString(file *os.File, string string) (int, error) { - return file.WriteString(string) // #nosec G304 +func (fs *Fs) WriteString(file *os.File, str string) (int, error) { + return file.WriteString(str) // #nosec G304 } // Create is a wrapper of os.Create @@ -188,7 +193,8 @@ func (fs *Fs) ExecCommandOutput(name string, args ...string) ([]byte, error) { // ParseProcMounts is wrapper of gofsutil.ReadProcMountsFrom global function func (fs *Fs) ParseProcMounts( ctx context.Context, - content io.Reader) ([]gofsutil.Info, error) { + content io.Reader, +) ([]gofsutil.Info, error) { r, _, err := gofsutil.ReadProcMountsFrom(ctx, content, false, gofsutil.ProcMountsFields, gofsutil.DefaultEntryScanFunc()) return r, err @@ -196,22 +202,33 @@ func (fs *Fs) ParseProcMounts( // NetDial is a wrapper for net.Dial func. Uses UDP and 80 port. func (fs *Fs) NetDial(endpoint string) (net.Conn, error) { - return net.Dial("udp", fmt.Sprintf("%s:80", endpoint)) + splittedURL := strings.Split(endpoint, ":") + if len(splittedURL) == 1 { + // if we are here then its plain driver installation + endpoint = fmt.Sprintf("%s:%s", endpoint, "80") + } + log.Infof("Using final endpoint %s", endpoint) + return net.Dial("udp", endpoint) } // MkFileIdempotent creates file if there is none func (fs *Fs) MkFileIdempotent(path string) (bool, error) { st, err := fs.Stat(path) if fs.IsNotExist(err) { - file, err := fs.OpenFile(path, os.O_CREATE, 0600) + file, err := fs.OpenFile(path, os.O_CREATE, 0o600) if err != nil { - log.WithField("path", path).WithError(err).Error("Unable to create file") + log.WithFields(csmlog.Fields{ + "path": path, + }).Error("Unable to create file" + err.Error()) return false, err } if err = file.Close(); err != nil { return false, fmt.Errorf("could not close file") } - log.WithField("path", path).Debug("created file") + log.WithFields(csmlog.Fields{ + "path": path, + }).Debug("created file") + return true, nil } if st.IsDir() { diff --git a/pkg/common/fs/fs_test.go b/pkg/identifiers/fs/fs_test.go similarity index 58% rename from pkg/common/fs/fs_test.go rename to pkg/identifiers/fs/fs_test.go index 4fd745f9..3c9cb0a8 100644 --- a/pkg/common/fs/fs_test.go +++ b/pkg/identifiers/fs/fs_test.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2021 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2021-2024 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,7 +19,9 @@ package fs import ( + "context" "os" + "strings" "testing" "github.com/dell/gofsutil" @@ -35,7 +37,7 @@ type FsTestSuite struct { func (suite *FsTestSuite) SetupSuite() { suite.fs = &Fs{Util: &gofsutil.FS{}} suite.tmp = "./tmp" - err := os.Mkdir(suite.tmp, 0750) + err := os.Mkdir(suite.tmp, 0o750) if err != nil { suite.T().Error("couldn't create the tmp folder") } @@ -74,7 +76,7 @@ func (suite *FsTestSuite) TestCreate() { func (suite *FsTestSuite) TestWriteFile() { str := "random string \n hello" data := []byte(str) - err := suite.fs.WriteFile(suite.tmp+"/create", data, 0640) + err := suite.fs.WriteFile(suite.tmp+"/create", data, 0o640) suite.Assert().NoError(err) bytes, err := suite.fs.ReadFile(suite.tmp + "/create") @@ -82,12 +84,23 @@ func (suite *FsTestSuite) TestWriteFile() { suite.Assert().Equal(bytes, data) } +func (suite *FsTestSuite) TestOpenFile() { + file, err := suite.fs.OpenFile(suite.tmp+"/file", os.O_CREATE, 0o600) + suite.Assert().NoError(err) + + err = suite.fs.Chmod(suite.tmp+"/file", os.ModeSticky) + suite.Assert().NoError(err) + + err = file.Close() + suite.Assert().NoError(err) +} + func (suite *FsTestSuite) TestMkDir() { - err := suite.fs.Mkdir(suite.tmp+"/dir", 0750) + err := suite.fs.Mkdir(suite.tmp+"/dir", 0o750) suite.Assert().NoError(err) suite.Assert().DirExists(suite.tmp + "/dir") - err = suite.fs.MkdirAll(suite.tmp+"/1/2/3", 0750) + err = suite.fs.MkdirAll(suite.tmp+"/1/2/3", 0o750) suite.Assert().NoError(err) suite.Assert().DirExists(suite.tmp + "/1/2") @@ -101,7 +114,7 @@ func (suite *FsTestSuite) TestMkFileIdempotent() { suite.Assert().NoError(err) suite.Assert().Equal(true, created) - err = suite.fs.Mkdir(suite.tmp+"/mydir", 0750) + err = suite.fs.Mkdir(suite.tmp+"/mydir", 0o750) suite.Assert().NoError(err) _, err = suite.fs.MkFileIdempotent(suite.tmp + "/mydir") suite.Assert().EqualError(err, "existing path is a directory") @@ -121,6 +134,45 @@ func (suite *FsTestSuite) TestExecCommand() { suite.Assert().NotEmpty(out) } +func (suite *FsTestSuite) TestIsDeviceOrResourceBusy() { + err := suite.fs.Remove(suite.tmp + "/busy") + res := suite.fs.IsDeviceOrResourceBusy(err) + suite.Assert().False(res) +} + +func (suite *FsTestSuite) TestParseProcMounts() { + input := `17 60 0:16 / /sys rw,nosuid,nodev,noexec,relatime shared:6 - sysfs sysfs rw,seclabel + 18 60 0:3 / /proc rw,nosuid,nodev,noexec,relatime shared:5 - proc proc rw + 19 60 0:5 / /dev rw,nosuid shared:2 - devtmpfs devtmpfs rw,seclabel,size=1930460k,nr_inodes=482615,mode=755 + 20 17 0:15 / /sys/kernel/security rw,nosuid,nodev,noexec,relatime shared:7 - securityfs securityfs rw + 21 19 0:17 / /dev/shm rw,nosuid,nodev shared:3 - tmpfs tmpfs rw,seclabel + 22 19 0:11 / /dev/pts rw,nosuid,noexec,relatime shared:4 - devpts devpts rw,seclabel,gid=5,mode=620,ptmxmode=000 + 23 60 0:18 / /run rw,nosuid,nodev shared:23 - tmpfs tmpfs rw,seclabel,mode=755 + 24 17 0:19 / /sys/fs/cgroup ro,nosuid,nodev,noexec shared:8 - tmpfs tmpfs ro,seclabel,mode=755` + mounts, err := suite.fs.ParseProcMounts(context.Background(), strings.NewReader(input)) + suite.Assert().NoError(err) + suite.Assert().NotEmpty(mounts) +} + +func (suite *FsTestSuite) TestNetDial() { + conn, err := suite.fs.NetDial("localhost") + suite.Assert().NoError(err) + conn.Close() +} + +func (suite *FsTestSuite) TestNetDialWithPort() { + conn, err := suite.fs.NetDial("localhost:9400") + suite.Assert().NoError(err) + suite.Assert().NotNil(conn) + defer conn.Close() +} + +func (suite *FsTestSuite) TestNetDialWithHttpsPort() { + conn, err := suite.fs.NetDial("https://localhost:9400") + suite.Assert().Error(err) + suite.Assert().Nil(conn) +} + func (suite *FsTestSuite) TestGetUtil() { util := suite.fs.GetUtil() suite.Assert().NotNil(util) diff --git a/pkg/common/common.go b/pkg/identifiers/identifiers.go similarity index 56% rename from pkg/common/common.go rename to pkg/identifiers/identifiers.go index a108bbb3..19818360 100644 --- a/pkg/common/common.go +++ b/pkg/identifiers/identifiers.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2021-2023 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2021-2026 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,8 +16,8 @@ * */ -// Package common provides common constants, variables and function used in both controller and node services. -package common +// Package identifiers provides common constants, variables and function used in both controller and node services. +package identifiers import ( "bytes" @@ -28,34 +28,39 @@ import ( "net" "os" "regexp" + "slices" "sort" "strconv" "strings" "sync" "time" - "github.com/apparentlymart/go-cidr/cidr" - "github.com/container-storage-interface/spec/lib/go/csi" "github.com/dell/csi-powerstore/v2/core" - "github.com/dell/csi-powerstore/v2/pkg/common/fs" + "github.com/dell/csi-powerstore/v2/pkg/identifiers/fs" + "github.com/dell/csmlog" "github.com/dell/gobrick" csictx "github.com/dell/gocsi/context" - "github.com/dell/gocsi/utils" + csiutils "github.com/dell/gocsi/utils/csi" "github.com/dell/gopowerstore" - log "github.com/sirupsen/logrus" + "github.com/apparentlymart/go-cidr/cidr" + "github.com/container-storage-interface/spec/lib/go/csi" ) +// Instantiate csmlog on a package level +var log = csmlog.GetLogger() + // Name contains default name of the driver, can be overridden var Name = "csi-powerstore.dellemc.com" // APIPort port for API calls var APIPort string +// Update when the manifest version changes. +var ManifestSemver string + // Manifest contains additional information about the driver var Manifest = map[string]string{ - "url": "https://github.com/dell/csi-powerstore", - "semver": core.SemVer, - "commit": core.CommitSha32, + "semver": ManifestSemver, "formed": core.CommitTime.Format(time.RFC1123), } @@ -136,30 +141,56 @@ const ( AutoDetectTransport TransportType = "AUTO" // NoneTransport indicates that no SCSI transport protocol needed NoneTransport TransportType = "NONE" - // PublishContextDeviceWWN indicates publish context device wwn - PublishContextDeviceWWN = "DEVICE_WWN" - // PublishContextLUNAddress indicates publish context LUN address - PublishContextLUNAddress = "LUN_ADDRESS" - // PublishContextISCSIPortalsPrefix indicates publish context iSCSI portals prefix - PublishContextISCSIPortalsPrefix = "PORTAL" - // PublishContextISCSITargetsPrefix indicates publish context iSCSI targets prefix - PublishContextISCSITargetsPrefix = "TARGET" - // PublishContextNVMETCPPortalsPrefix indicates publish context NVMeTCP portals prefix - PublishContextNVMETCPPortalsPrefix = "NVMETCPPORTAL" - // PublishContextNVMETCPTargetsPrefix indicates publish context NVMe targets prefix - PublishContextNVMETCPTargetsPrefix = "NVMETCPTARGET" - // PublishContextNVMEFCPortalsPrefix indicates publish context NVMe targets prefix - PublishContextNVMEFCPortalsPrefix = "NVMEFCPORTAL" - // PublishContextNVMEFCTargetsPrefix indicates publish context NVMe targets prefix - PublishContextNVMEFCTargetsPrefix = "NVMEFCTARGET" + // TargetMapDeviceWWN indicates target map device wwn + TargetMapDeviceWWN = "DEVICE_WWN" + // TargetMapLUNAddress indicates publish context LUN address + TargetMapLUNAddress = "LUN_ADDRESS" + // TargetMapISCSIPortalsPrefix indicates target map iSCSI portals prefix + TargetMapISCSIPortalsPrefix = "PORTAL" + // TargetMapISCSITargetsPrefix indicates target map iSCSI targets prefix + TargetMapISCSITargetsPrefix = "TARGET" + // TargetMapNVMETCPPortalsPrefix indicates target map NVMeTCP portals prefix + TargetMapNVMETCPPortalsPrefix = "NVMETCPPORTAL" + // TargetMapNVMETCPTargetsPrefix indicates target mapNVMe targets prefix + TargetMapNVMETCPTargetsPrefix = "NVMETCPTARGET" + // TargetMapNVMEFCPortalsPrefix indicates publish context NVMe targets prefix + TargetMapNVMEFCPortalsPrefix = "NVMEFCPORTAL" + // TargetMapNVMEFCTargetsPrefix indicates target map NVMe targets prefix + TargetMapNVMEFCTargetsPrefix = "NVMEFCTARGET" // NVMETCPTransport indicates that NVMe/TCP is chosen as the transport protocol NVMETCPTransport TransportType = "NVMETCP" // NVMEFCTransport indicates that NVMe/FC is chosen as the transport protocol NVMEFCTransport TransportType = "NVMEFC" - // PublishContextFCWWPNPrefix indicates publish context FC WWPN prefix - PublishContextFCWWPNPrefix = "FCWWPN" + // TargetMapFCWWPNPrefix indicates target map FC WWPN prefix + TargetMapFCWWPNPrefix = "FCWWPN" + // TargetMapRemoteDeviceWWN indicates target map device wwn of remote device + TargetMapRemoteDeviceWWN = "REMOTE_DEVICE_WWN" + // TargetMapRemoteLUNAddress indicates target map LUN address of remote device + TargetMapRemoteLUNAddress = "REMOTE_LUN_ADDRESS" + // TargetMapRemoteISCSIPortalsPrefix indicates target map iSCSI portals prefix of remote array + TargetMapRemoteISCSIPortalsPrefix = "REMOTE_PORTAL" + // TargetMapRemoteISCSITargetsPrefix indicates target map iSCSI targets prefix of remote array + TargetMapRemoteISCSITargetsPrefix = "REMOTE_TARGET" + // TargetMapRemoteNVMETCPPortalsPrefix indicates target map NVMeTCP portals prefix of remote array + TargetMapRemoteNVMETCPPortalsPrefix = "REMOTE_NVMETCPPORTAL" + // TargetMapRemoteNVMETCPTargetsPrefix indicates target map NVMe targets prefix of remote array + TargetMapRemoteNVMETCPTargetsPrefix = "REMOTE_NVMETCPTARGET" + // TargetMapRemoteNVMEFCPortalsPrefix indicates target map NVMe targets prefix of remote array + TargetMapRemoteNVMEFCPortalsPrefix = "REMOTE_NVMEFCPORTAL" + // TargetMapRemoteNVMEFCTargetsPrefix indicates target map NVMe targets prefix of remote array + TargetMapRemoteNVMEFCTargetsPrefix = "REMOTE_NVMEFCTARGET" + // TargetMapRemoteFCWWPNPrefix indicates target map FC WWPN prefix of remote array + TargetMapRemoteFCWWPNPrefix = "REMOTE_FCWWPN" // WWNPrefix indicates WWN prefix WWNPrefix = "naa." + // SyncMode indicates Synchronous Replication + SyncMode = "SYNC" + // AsyncMode indicats Asynchronous Replication + AsyncMode = "ASYNC" + // MetroMode indicates Metro Replication + MetroMode = "METRO" + // Zero indicates value zero for RPO + Zero = "Zero" contextLogFieldsKey key = iota @@ -169,19 +200,40 @@ const ( // DefaultPodmonPollRate is the default polling frequency to check for array connectivity DefaultPodmonPollRate = 60 - // Timeout for making http requests - Timeout = time.Second * 5 - // ArrayStatus is the endPoint for polling to check array status ArrayStatus = "/array-status" + + // KeyNodeID represents key for node id + KeyNodeID = "csi.volume.kubernetes.io/nodeid" ) +// PodmonArrayConnectivityTimeout specifies timeout for making http requests to node services by podmon +var PodmonArrayConnectivityTimeout = GetPodmonArrayConnectivityTimeout() + +// DefaultPodmonArrayConnectivityTimeout specifies default timeout for making http requests to node services by podmon +var DefaultPodmonArrayConnectivityTimeout = 10 * time.Second + +// PowerstoreRESTApiTimeout specifies timeout for making http requests by Powerstore client +var PowerstoreRESTApiTimeout = GetPowerStoreRESTApiTimeout() + +// DefaultPowerstoreRESTApiTimeout specifies default timeout for making http requests by Powerstore client +var DefaultPowerstoreRESTApiTimeout = 120 * time.Second + +// DefaultVolumeDisconnectMaxRetries specifies the default maximum number of retry attempts for volume disconnection +var DefaultVolumeDisconnectMaxRetries = 5 + +// DefaultVolumeDisconnectRetryInterval specifies the default wait time between volume disconnection retries +var DefaultVolumeDisconnectRetryInterval = 5 * time.Second + +// DefaultVolumeDisconnectTimeout specifies the default timeout duration for each volume disconnection attempt +var DefaultVolumeDisconnectTimeout = 120 * time.Second + // TransportType differentiates different SCSI transport protocols (FC, iSCSI, Auto, None) type TransportType string // RmSockFile removes socket files that left after previous installation func RmSockFile(f fs.Interface) { - proto, addr, err := utils.GetCSIEndpoint() + proto, addr, err := csiutils.GetCSIEndpoint() if err != nil { log.Errorf("Error: failed to get CSI endpoint: %s\n", err.Error()) } @@ -206,8 +258,37 @@ func RmSockFile(f fs.Interface) { // GetIPListFromString returns list of ips in string form found in input string // A return value of nil indicates no match func GetIPListFromString(input string) []string { - re := regexp.MustCompile(`(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}`) - return re.FindAllString(input, -1) + ipRe := regexp.MustCompile(`\b((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}|localhost\b)`) + urlRe := regexp.MustCompile(`https?://([a-zA-Z0-9.-]+)(:[0-9]+)?(/.*)?`) + + ipMatches := ipRe.FindAllString(input, -1) + urlMatches := urlRe.FindAllStringSubmatch(input, -1) + + var matches []string + matches = append(matches, ipMatches...) + for _, match := range urlMatches { + if len(match) > 1 { + if strings.Contains(match[1], ".") { + if !isDomain(match[1]) { + continue + } + } + if !slices.Contains(matches, match[1]) { + matches = append(matches, match[1]) + } + } + } + + return matches +} + +func isDomain(str string) bool { + for _, r := range str { + if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') { + return true + } + } + return false } func parseMask(ipaddr string) (mask string, err error) { @@ -261,44 +342,19 @@ func GetIPListWithMaskFromString(input string) (string, error) { return ip, nil } -// SetLogFields returns modified context with fields inserted as values by using contextLogFieldsKey key -func SetLogFields(ctx context.Context, fields log.Fields) context.Context { - if ctx == nil { - ctx = context.Background() - } - return context.WithValue(ctx, contextLogFieldsKey, fields) -} - // RandomString returns a random string of specified length. // String is generated by using crypto/rand. -func RandomString(len int) string { - b := make([]byte, len) +func RandomString(length int) string { + b := make([]byte, length) _, err := rand.Read(b) if err != nil { - log.Errorf("Can't generate random string; error = %v", err) + return err.Error() } suff := fmt.Sprintf("%x", b[0:]) return suff } -// GetLogFields extracts log fields from context by using contextLogFieldsKey key -func GetLogFields(ctx context.Context) log.Fields { - if ctx == nil { - return log.Fields{} - } - fields, ok := ctx.Value(contextLogFieldsKey).(log.Fields) - if !ok { - fields = log.Fields{} - } - csiReqID, ok := ctx.Value(csictx.RequestIDKey).(string) - if !ok { - return fields - } - fields["RequestID"] = csiReqID - return fields -} - -// GetISCSITargetsInfoFromStorage returns list of gobrick compatible iscsi tragets by querying PowerStore array +// GetISCSITargetsInfoFromStorage returns list of gobrick compatible iscsi targets by querying PowerStore array func GetISCSITargetsInfoFromStorage(client gopowerstore.Client, volumeApplianceID string) ([]gobrick.ISCSITargetInfo, error) { addrInfo, err := client.GetStorageISCSITargetAddresses(context.Background()) if err != nil { @@ -311,15 +367,43 @@ func GetISCSITargetsInfoFromStorage(client gopowerstore.Client, volumeApplianceI }) var result []gobrick.ISCSITargetInfo for _, t := range addrInfo { - //volumeApplianceID will be empty in case the call is from NodeGetInfo + // volumeApplianceID will be empty in case the call is from NodeGetInfo if t.ApplianceID == volumeApplianceID || volumeApplianceID == "" { - result = append(result, gobrick.ISCSITargetInfo{Target: t.IPPort.TargetIqn, Portal: fmt.Sprintf("%s:3260", t.Address)}) + result = append(result, gobrick.ISCSITargetInfo{Target: t.IPPort.TargetIqn, Portal: fmt.Sprintf("%s:3260", t.Address), NetworkID: t.NetworkID}) } } return result, nil } -// GetFCTargetsInfoFromStorage returns list of gobrick compatible FC tragets by querying PowerStore array +// GetNVMETCPTargetsInfoFromStorage returns list of gobrick compatible NVME TCP targets by querying PowerStore array +func GetNVMETCPTargetsInfoFromStorage(client gopowerstore.Client, volumeApplianceID string) ([]gobrick.NVMeTargetInfo, error) { + clusterInfo, err := client.GetCluster(context.Background()) + if err != nil { + log.Error(err.Error()) + return []gobrick.NVMeTargetInfo{}, err + } + nvmeNQN := clusterInfo.NVMeNQN + + addrInfo, err := client.GetStorageNVMETCPTargetAddresses(context.Background()) + if err != nil { + log.Error(err.Error()) + return []gobrick.NVMeTargetInfo{}, err + } + // sort data by id + sort.Slice(addrInfo, func(i, j int) bool { + return addrInfo[i].ID < addrInfo[j].ID + }) + var result []gobrick.NVMeTargetInfo + for _, t := range addrInfo { + // volumeApplianceID will be empty in case the call is from NodeGetInfo + if t.ApplianceID == volumeApplianceID || volumeApplianceID == "" { + result = append(result, gobrick.NVMeTargetInfo{Target: nvmeNQN, Portal: fmt.Sprintf("%s:4420", t.Address), NetworkID: t.NetworkID}) + } + } + return result, nil +} + +// GetFCTargetsInfoFromStorage returns list of gobrick compatible FC targets by querying PowerStore array func GetFCTargetsInfoFromStorage(client gopowerstore.Client, volumeApplianceID string) ([]gobrick.FCTargetInfo, error) { fcPorts, err := client.GetFCPorts(context.Background()) if err != nil { @@ -377,19 +461,19 @@ func ParseCIDR(externalAccessCIDR string) (string, error) { if !strings.Contains(externalAccessCIDR, "/") { // if externalAccess is a plane ip we can add /32 from our end externalAccessCIDR += "/32" - log.Debug("externalAccess after appending netMask bit:", externalAccessCIDR) + log.Debugf("externalAccess after appending netMask bit: %s", externalAccessCIDR) } ip, ipnet, err := net.ParseCIDR(externalAccessCIDR) if err != nil { return "", err } - log.Debug("Parsed CIDR:", externalAccessCIDR, "-> ip:", ip, " net:", ipnet) + log.Debugf("Parsed CIDR: %s -> ip: %v and net: %v", externalAccessCIDR, ip, ipnet) start, _ := cidr.AddressRange(ipnet) fromString, err := GetIPListWithMaskFromString(externalAccessCIDR) if err != nil { return "", err } - log.Debug("IP with Mask:", fromString) + log.Debugf("IP with Mask: %s", fromString) s := strings.Split(fromString, "/") // ExernalAccess IP consists of Starting range IP of CIDR+Mask and hence concatenating the same to remove from the array @@ -434,15 +518,16 @@ func Contains(slice []string, element string) bool { func ExternalAccessAlreadyAdded(export gopowerstore.NFSExport, externalAccess string) bool { externalAccess, _ = ParseCIDR(externalAccess) if Contains(export.RWRootHosts, externalAccess) || Contains(export.RWHosts, externalAccess) || Contains(export.RORootHosts, externalAccess) || Contains(export.ROHosts, externalAccess) { - log.Debug("ExternalAccess is already added into Host Access list on array: ", externalAccess) + log.Debugf("ExternalAccess is already added into Host Access list on array: %s ", externalAccess) return true } - log.Debug("Going to add externalAccess into Host Access list on array: ", externalAccess) + log.Debugf("Going to add externalAccess into Host Access list on array: %s", externalAccess) return false } // SetPollingFrequency reads the pollingFrequency from Env, sets default vale if ENV not found func SetPollingFrequency(ctx context.Context) int64 { + log := log.WithContext(ctx) var pollingFrequency int64 if pollRateEnv, ok := csictx.LookupEnv(ctx, EnvPodmonArrayConnectivityPollRate); ok { if pollingFrequency, _ = strconv.ParseInt(pollRateEnv, 10, 32); pollingFrequency != 0 { @@ -456,6 +541,7 @@ func SetPollingFrequency(ctx context.Context) int64 { // SetAPIPort set the port for running server func SetAPIPort(ctx context.Context) { + log := log.WithContext(ctx) if port, ok := csictx.LookupEnv(ctx, EnvPodmonAPIPORT); ok && strings.TrimSpace(port) != "" { APIPort = fmt.Sprintf(":%s", port) log.Debugf("set podmon API port to %s", APIPort) @@ -470,8 +556,122 @@ func SetAPIPort(ctx context.Context) { func ReachableEndPoint(endpoint string) bool { // this endpoint has IP:PORT _, err := net.DialTimeout("tcp", endpoint, 2*time.Second) + return err == nil +} + +func GetMountFlags(vc *csi.VolumeCapability) []string { + if vc != nil { + if mount := vc.GetMount(); mount != nil { + return mount.GetMountFlags() + } + } + return nil +} + +// IsNFSServiceEnabled checks if NFS service is enabled for the given PowerStore array. +func IsNFSServiceEnabled(ctx context.Context, client gopowerstore.Client) (bool, error) { + nasList, err := client.GetNASServers(ctx) if err != nil { - return false + return false, fmt.Errorf("failed to get NAS servers: %w", err) + } + + for _, nas := range nasList { + for _, nasServer := range nas.NfsServers { + if nasServer.IsNFSv4Enabled || nasServer.IsNFSv3Enabled { + return true, nil + } + } + } + return false, nil +} + +// GetTimeoutFromEnv retrieves a timeout value from the specified environment variable or returns default value. +func GetTimeoutFromEnv(envvar string) (time.Duration, error) { + var timeout time.Duration + var err error + if duration, ok := csictx.LookupEnv(context.Background(), envvar); ok { + timeout, err = time.ParseDuration(duration) + if err != nil { + return -1, err + } + } else { + return -1, errors.New("failed to get timeout from env") + } + log.Infof("%s set to: %v", envvar, timeout) + return timeout, nil +} + +// GetIntFromEnv retrieves an integer value from the specified environment variable or returns an error. +func GetIntFromEnv(envvar string) (int, error) { + if valStr, ok := csictx.LookupEnv(context.Background(), envvar); ok { + val, err := strconv.Atoi(valStr) + if err != nil { + return -1, err + } + log.Infof("%s set to: %d", envvar, val) + return val, nil } - return true + return -1, errors.New("failed to get integer value from env") +} + +// GetPodmonArrayConnectivityPollRate retrieves a timeout value for podmon node array connectivity check from env var or returns default value. +func GetPodmonArrayConnectivityTimeout() time.Duration { + timeout, err := GetTimeoutFromEnv(EnvPodmonArrayConnectivityTimeout) + if err != nil { + log.Debugf("failed to get timeout from env %s, using default value %d", EnvPodmonArrayConnectivityTimeout, DefaultPodmonArrayConnectivityTimeout) + return DefaultPodmonArrayConnectivityTimeout + } + return timeout +} + +// GetPowerStoreRESTApiTimeout retrieves a timeout value for PowerStore REST API requests from env var or returns default value.. +func GetPowerStoreRESTApiTimeout() time.Duration { + timeout, err := GetTimeoutFromEnv(EnvPowerstoreAPITimeout) + if err != nil { + log.Debugf("failed to get timeout from env %s, using default value %d", EnvPowerstoreAPITimeout, DefaultPowerstoreRESTApiTimeout) + return DefaultPowerstoreRESTApiTimeout + } + return timeout +} + +// GetVolumeDisconnectMaxRetries retrieves the maximum number of retry attempts for volume disconnection from env var or returns the default value. +func GetVolumeDisconnectMaxRetries() int { + retries, err := GetIntFromEnv(EnvVolumeDisconnectMaxRetries) + if err != nil { + log.Debugf("failed to get max retries from env %s, using default value %d", EnvVolumeDisconnectMaxRetries, DefaultVolumeDisconnectMaxRetries) + return DefaultVolumeDisconnectMaxRetries + } + return retries +} + +// GetVolumeDisconnectRetryInterval retrieves the wait time between volume disconnection retries from env var or returns the default value. +func GetVolumeDisconnectRetryInterval() time.Duration { + timeout, err := GetTimeoutFromEnv(EnvVolumeDisconnectRetryInterval) + if err != nil { + log.Debugf("failed to get RetryInterval from env %s, using default value %d", EnvVolumeDisconnectRetryInterval, DefaultVolumeDisconnectRetryInterval) + return DefaultVolumeDisconnectRetryInterval + } + return timeout +} + +// GetVolumeDisconnectTimeout retrieves a timeout value for volume disconnection from env var or returns the default value. +func GetVolumeDisconnectTimeout() time.Duration { + timeout, err := GetTimeoutFromEnv(EnvVolumeDisconnectTimeoutSeconds) + if err != nil { + log.Debugf("failed to get timeout from env %s, using default value %d", EnvVolumeDisconnectTimeoutSeconds, DefaultVolumeDisconnectTimeout) + return DefaultVolumeDisconnectTimeout + } + return timeout +} + +// HostAlreadyPresentInNFSExport checks if the given host IP is already present in any of the NFS export host lists. +func HostAlreadyPresentInNFSExport(export gopowerstore.NFSExport, ip string) bool { + host := ip + "/255.255.255.255" + if Contains(export.ROHosts, host) || + Contains(export.RORootHosts, host) || + Contains(export.RWHosts, host) || Contains(export.RWRootHosts, host) { + log.Debug("Host IP is already present in NFS Export") + return true + } + return false } diff --git a/pkg/identifiers/identifiers_test.go b/pkg/identifiers/identifiers_test.go new file mode 100644 index 00000000..6f52695c --- /dev/null +++ b/pkg/identifiers/identifiers_test.go @@ -0,0 +1,830 @@ +/* + * + * Copyright © 2021-2025 Dell Inc. or its subsidiaries. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package identifiers_test + +import ( + "context" + "errors" + "fmt" + "os" + "reflect" + "testing" + "time" + + "github.com/dell/csi-powerstore/v2/mocks" + identifiers "github.com/dell/csi-powerstore/v2/pkg/identifiers" + csiutils "github.com/dell/gocsi/utils/csi" + "github.com/dell/gopowerstore" + gopowerstoremock "github.com/dell/gopowerstore/mocks" + "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +func TestRmSockFile(t *testing.T) { + sockPath := "unix:///var/run/csi/csi.sock" + trimmedSockPath := "/var/run/csi/csi.sock" + _ = os.Setenv(csiutils.CSIEndpoint, sockPath) + + t.Run("removed socket", func(_ *testing.T) { + fsMock := new(mocks.FsInterface) + fsMock.On("Stat", trimmedSockPath).Return(&mocks.FileInfo{}, nil) + fsMock.On("RemoveAll", trimmedSockPath).Return(nil) + + identifiers.RmSockFile(fsMock) + }) + + t.Run("failed to remove socket", func(_ *testing.T) { + fsMock := new(mocks.FsInterface) + fsMock.On("Stat", trimmedSockPath).Return(&mocks.FileInfo{}, nil) + fsMock.On("RemoveAll", trimmedSockPath).Return(fmt.Errorf("some error")) + + identifiers.RmSockFile(fsMock) + }) + + t.Run("not found", func(_ *testing.T) { + fsMock := new(mocks.FsInterface) + fsMock.On("Stat", trimmedSockPath).Return(&mocks.FileInfo{}, os.ErrNotExist) + + identifiers.RmSockFile(fsMock) + }) + + t.Run("may or may not exist", func(_ *testing.T) { + fsMock := new(mocks.FsInterface) + fsMock.On("Stat", trimmedSockPath).Return(&mocks.FileInfo{}, fmt.Errorf("some other error")) + + identifiers.RmSockFile(fsMock) + }) + + t.Run("no endpoint set", func(_ *testing.T) { + fsMock := new(mocks.FsInterface) + _ = os.Setenv(csiutils.CSIEndpoint, "") + + identifiers.RmSockFile(fsMock) + }) +} + +func TestGetISCSITargetsInfoFromStorage(t *testing.T) { + t.Run("api error", func(t *testing.T) { + e := errors.New("some error") + clientMock := new(gopowerstoremock.Client) + clientMock.On("GetStorageISCSITargetAddresses", context.Background()).Return([]gopowerstore.IPPoolAddress{}, e) + _, err := identifiers.GetISCSITargetsInfoFromStorage(clientMock, "A1") + assert.EqualError(t, err, e.Error()) + }) + + t.Run("no error", func(t *testing.T) { + clientMock := new(gopowerstoremock.Client) + clientMock.On("GetStorageISCSITargetAddresses", context.Background()). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn"}, + }, + }, nil) + iscsiTargetsInfo, err := identifiers.GetISCSITargetsInfoFromStorage(clientMock, "") + assert.NotNil(t, iscsiTargetsInfo) + assert.NoError(t, err) + }) +} + +func TestGetNVMETCPTargetsInfoFromStorage(t *testing.T) { + t.Run("api error", func(t *testing.T) { + e := errors.New("some error") + clientMock := new(gopowerstoremock.Client) + clientMock.On("GetCluster", context.Background()).Return(gopowerstore.Cluster{}, e) + clientMock.On("GetStorageNVMETCPTargetAddresses", context.Background()).Return([]gopowerstore.IPPoolAddress{}, e) + _, err := identifiers.GetNVMETCPTargetsInfoFromStorage(clientMock, "A1") + assert.EqualError(t, err, e.Error()) + }) + + t.Run("no error", func(t *testing.T) { + clientMock := new(gopowerstoremock.Client) + clientMock.On("GetCluster", context.Background()).Return(gopowerstore.Cluster{}, nil) + clientMock.On("GetStorageNVMETCPTargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn"}, + }, + }, nil) + nvmetcpTargetInfo, err := identifiers.GetNVMETCPTargetsInfoFromStorage(clientMock, "") + assert.NotNil(t, nvmetcpTargetInfo) + assert.NoError(t, err) + }) +} + +func TestGetFCTargetsInfoFromStorage(t *testing.T) { + t.Run("api error", func(t *testing.T) { + e := errors.New("some error") + clientMock := new(gopowerstoremock.Client) + clientMock.On("GetFCPorts", context.Background()).Return([]gopowerstore.FcPort{}, e) + _, err := identifiers.GetFCTargetsInfoFromStorage(clientMock, "A1") + assert.EqualError(t, err, e.Error()) + }) + + t.Run("no error", func(t *testing.T) { + clientMock := new(gopowerstoremock.Client) + clientMock.On("GetFCPorts", mock.Anything). + Return([]gopowerstore.FcPort{ + { + Wwn: "58:cc:f0:93:48:a0:03:a3", + ApplianceID: "A1", + IsLinkUp: true, + }, + }, nil) + fcTargetInfo, err := identifiers.GetFCTargetsInfoFromStorage(clientMock, "A1") + assert.NotNil(t, fcTargetInfo) + assert.NoError(t, err) + }) +} + +func TestIsK8sMetadataSupported(t *testing.T) { + t.Run("api error", func(t *testing.T) { + e := errors.New("some error") + clientMock := new(gopowerstoremock.Client) + clientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(0.0), e) + version := identifiers.IsK8sMetadataSupported(clientMock) + assert.Equal(t, version, false) + }) + + t.Run("no error", func(t *testing.T) { + clientMock := new(gopowerstoremock.Client) + clientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(3.0), nil) + version := identifiers.IsK8sMetadataSupported(clientMock) + assert.Equal(t, version, true) + }) +} + +func TestGetNVMEFCTargetInfoFromStorage(t *testing.T) { + t.Run("api error", func(t *testing.T) { + e := errors.New("some error") + clientMock := new(gopowerstoremock.Client) + clientMock.On("GetCluster", context.Background()).Return(gopowerstore.Cluster{}, e) + clientMock.On("GetFCPorts", context.Background()).Return([]gopowerstore.FcPort{}, e) + _, err := identifiers.GetNVMEFCTargetInfoFromStorage(clientMock, "A1") + assert.EqualError(t, err, e.Error()) + }) + + t.Run("no error", func(t *testing.T) { + clientMock := new(gopowerstoremock.Client) + clientMock.On("GetCluster", context.Background()).Return(gopowerstore.Cluster{}, nil) + clientMock.On("GetFCPorts", mock.Anything). + Return([]gopowerstore.FcPort{ + { + Wwn: "58:cc:f0:93:48:a0:03:a3", + IsLinkUp: true, + }, + }, nil) + nvmefcTargetInfo, err := identifiers.GetNVMEFCTargetInfoFromStorage(clientMock, "") + assert.NotNil(t, nvmefcTargetInfo) + assert.NoError(t, err) + }) +} + +func TestHasRequiredTopology(t *testing.T) { + nfsTopology := &csi.Topology{Segments: map[string]string{"csi-powerstore.dellemc.com/10.0.0.0-nfs": "true"}} + iscsiTopology := &csi.Topology{Segments: map[string]string{"csi-powerstore.dellemc.com/10.0.0.0-iscsi": "true"}} + + type args struct { + topologies []*csi.Topology + arrIP string + requiredTopology string + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "only nfs is present in topologies", + args: args{topologies: []*csi.Topology{nfsTopology}, arrIP: "10.0.0.0", requiredTopology: "nfs"}, + want: true, + }, + { + name: "nfs & iscsi is present in topologies", + args: args{topologies: []*csi.Topology{iscsiTopology, nfsTopology}, arrIP: "10.0.0.0", requiredTopology: "nfs"}, + want: true, + }, + { + name: "nfs is not present in topologies", + args: args{topologies: []*csi.Topology{iscsiTopology}, arrIP: "10.0.0.0", requiredTopology: "nfs"}, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equalf(t, tt.want, identifiers.HasRequiredTopology(tt.args.topologies, tt.args.arrIP, tt.args.requiredTopology), "HasRequiredTopology(%v, %v, %v)", tt.args.topologies, tt.args.arrIP, tt.args.requiredTopology) + }) + } +} + +func TestGetNfsTopology(t *testing.T) { + t.Run("nfs topology is true", func(t *testing.T) { + topology := identifiers.GetNfsTopology("10.0.0.0") + assert.Equal(t, topology, []*csi.Topology{{Segments: map[string]string{"csi-powerstore.dellemc.com/10.0.0.0-nfs": "true"}}}) + }) + + t.Run("nfs topology should not be false", func(t *testing.T) { + topology := identifiers.GetNfsTopology("10.0.0.0") + assert.NotEqual(t, topology, []*csi.Topology{{Segments: map[string]string{"csi-powerstore.dellemc.com/10.0.0.0-nfs": "false"}}}) + }) +} + +func Test_contains(t *testing.T) { + type args struct { + slice []string + element string + } + tests := []struct { + name string + args args + want bool + }{ + {"elementPresent", args{slice: []string{"firstElement", "secondElement"}, element: "secondElement"}, true}, + {"elementNotPresent", args{slice: []string{"firstElement", "secondElement"}, element: "thirdElement"}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := identifiers.Contains(tt.args.slice, tt.args.element); got != tt.want { + t.Errorf("contains() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestExternalAccessAlreadyAdded(t *testing.T) { + type args struct { + export gopowerstore.NFSExport + externalAccess string + } + tests := []struct { + name string + args args + want bool + }{ + {"externalAccessPresentInRWHosts", args{export: gopowerstore.NFSExport{RWHosts: []string{"10.0.0.0/255.255.255.255"}}, externalAccess: "10.0.0.0"}, true}, + {"externalAccessNotPresentInRWHosts", args{export: gopowerstore.NFSExport{RWHosts: []string{"10.232.0.0/255.255.255.255"}}, externalAccess: "10.10.0.0"}, false}, + {"externalAccessPresentInROHosts", args{export: gopowerstore.NFSExport{ROHosts: []string{"10.0.0.0/255.255.255.255"}}, externalAccess: "10.0.0.0"}, true}, + {"externalAccessNotPresentInROHosts", args{export: gopowerstore.NFSExport{ROHosts: []string{"10.232.0.0/255.255.255.255"}}, externalAccess: "10.10.0.0"}, false}, + {"externalAccessPresentInRWRootHosts", args{export: gopowerstore.NFSExport{RWRootHosts: []string{"10.0.0.0/255.255.255.255"}}, externalAccess: "10.0.0.0"}, true}, + {"externalAccessNotPresentInRWRootHosts", args{export: gopowerstore.NFSExport{RWRootHosts: []string{"10.232.0.0/255.255.255.255"}}, externalAccess: "10.10.0.0"}, false}, + {"externalAccessPresentInRORootHosts", args{export: gopowerstore.NFSExport{RORootHosts: []string{"10.0.0.0/255.255.255.255"}}, externalAccess: "10.0.0.0"}, true}, + {"externalAccessNotPresentInRORootHosts", args{export: gopowerstore.NFSExport{RORootHosts: []string{"10.232.0.0/255.255.255.255"}}, externalAccess: "10.10.0.0"}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := identifiers.ExternalAccessAlreadyAdded(tt.args.export, tt.args.externalAccess); got != tt.want { + t.Errorf("ExternalAccessAlreadyAdded() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestParseCIDR(t *testing.T) { + type args struct { + externalAccessCIDR string + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + {"Valid IP with net mask", args{externalAccessCIDR: "10.232.58.2/16"}, "10.232.0.0/255.255.0.0", false}, + {"Valid IP without net mask", args{externalAccessCIDR: "10.232.58.2"}, "10.232.58.2/255.255.255.255", false}, + {"InValid IP without net mask", args{externalAccessCIDR: "10.232.58"}, "", true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := identifiers.ParseCIDR(tt.args.externalAccessCIDR) + if (err != nil) != tt.wantErr { + t.Errorf("ParseCIDR() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("ParseCIDR() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestSetPollingFrequency(t *testing.T) { + type args struct { + ctx context.Context + } + tests := []struct { + name string + args args + want int64 + }{ + {"Setting environament variable", args{ctx: context.TODO()}, 100}, + {"Expecting default value to be set", args{ctx: context.TODO()}, 60}, + } + for i, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if i == 0 { + os.Setenv("X_CSI_PODMON_ARRAY_CONNECTIVITY_POLL_RATE", "100") + } + // need to import this function because the package name in this file is not common + // @TO-DO rename package name to common + if got := identifiers.SetPollingFrequency(tt.args.ctx); got != tt.want { + t.Errorf("SetPollingFrequency() = %v, want %v", got, tt.want) + } + os.Unsetenv("X_CSI_PODMON_ARRAY_CONNECTIVITY_POLL_RATE") + }) + } +} + +func Test_setAPIPort(t *testing.T) { + type args struct { + ctx context.Context + } + tests := []struct { + name string + args args + }{ + {"Fetching port number from Environment variable", args{ctx: context.TODO()}}, + {"Fetching & setting default port number", args{ctx: context.TODO()}}, + } + + for i, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if i == 0 { + os.Setenv("X_CSI_PODMON_API_PORT", "8090") + identifiers.SetAPIPort(tt.args.ctx) + if identifiers.APIPort != ":8090" { + t.Errorf("setAPIPort() error, want 8090 port found %v", identifiers.APIPort) + } + os.Unsetenv("X_CSI_PODMON_API_PORT") + } + identifiers.SetAPIPort(tt.args.ctx) + if identifiers.APIPort != ":8083" { + t.Errorf("setAPIPort() error, want 8083 port found %v", identifiers.APIPort) + } + }) + } +} + +func TestRandomString(t *testing.T) { + type args struct { + len int + } + tests := []struct { + name string + args args + }{ + {"Generating some random string", args{len: 5}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Since each byte in the slice is represented by two hex characters in the resulting string, the length of the string returned by the function will be len * 2. + if got := identifiers.RandomString(tt.args.len); len(got) != 5*2 { + t.Errorf("RandomString() = %v, have len %d and want 5*2", got, len(got)) + } + }) + } +} + +func TestGetIPListWithMaskFromString(t *testing.T) { + type args struct { + input string + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + {"Valid IP without subnet mask, Test 1", args{input: "10.1.1.2"}, "10.1.1.2", false}, + {"Invalid IP without subnet maskTest 2", args{input: "10.256.1.2"}, "", true}, + {"Invalid IP with subnet mask, Test 3", args{input: "10.256.1.2/24"}, "", true}, + {"Valid IP with subnet mask, Test 4", args{input: "10.1.1.2/24"}, "10.1.1.2/255.255.255.0", false}, + {"Invalid IP with subnet maskTest 5", args{input: "10.256.1.2/24/25"}, "", true}, + {"Invalid IP with Invalid subnet mask, Test 6", args{input: "10.255.1.2/24/25"}, "", true}, + {"Invalid IP with Invalid subnet mask, Test 7", args{input: "10.255.1.2/38"}, "", true}, + {"Invalid IP with Invalid subnet mask, Test 8", args{input: "10.255.1.2/x"}, "", true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := identifiers.GetIPListWithMaskFromString(tt.args.input) + if (err != nil) != tt.wantErr { + t.Errorf("GetIPListWithMaskFromString() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("GetIPListWithMaskFromString() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestGetIPListFromString(t *testing.T) { + type args struct { + input string + } + x := []string{} + x = nil + tests := []struct { + name string + args args + want []string + }{ + {"Valid IP, Test 1", args{input: "10.255.1.2"}, []string{"10.255.1.2"}}, + {"InValid IP, Test 2", args{input: "10.256.1.2"}, x}, + {"Valid Localhost", args{input: "https://localhost:9400"}, []string{"localhost"}}, + {"Valid Domain", args{input: "https://example.com:9443"}, []string{"example.com"}}, + {"Valid CSI NodeID", args{input: "csi-node-b61220be1acc441abdd8b00e34542e5d-1.1.1.1"}, []string{"1.1.1.1"}}, + {"Valid CSI NodeID", args{input: "csi-node-tar2222.infralab.ptec-2.2.2.2"}, []string{"2.2.2.2"}}, + {"Valid Multi-Segment Domain", args{input: "https://abc.example.com/page"}, []string{"abc.example.com"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := identifiers.GetIPListFromString(tt.args.input); !reflect.DeepEqual(got, tt.want) { + t.Errorf("GetIPListFromString() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestReachableEndPoint(t *testing.T) { + type args struct { + endpoint string + } + tests := []struct { + name string + args args + want bool + }{ + {"Unreachable IP, ", args{endpoint: "10.255.1.2:100"}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := identifiers.ReachableEndPoint(tt.args.endpoint); got != tt.want { + t.Errorf("ReachableEndPoint() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestGetMountFlags(t *testing.T) { + tests := []struct { + name string + vc *csi.VolumeCapability + expected []string + }{ + { + name: "Nil VolumeCapability", + vc: nil, + expected: nil, + }, + { + name: "Nil Mount", + vc: &csi.VolumeCapability{}, + expected: nil, + }, + { + name: "With Mount Flags", + vc: &csi.VolumeCapability{ + AccessType: &csi.VolumeCapability_Mount{ + Mount: &csi.VolumeCapability_MountVolume{ + MountFlags: []string{"ro", "noexec"}, + }, + }, + }, + expected: []string{"ro", "noexec"}, + }, + { + name: "Empty Mount Flags", + vc: &csi.VolumeCapability{ + AccessType: &csi.VolumeCapability_Mount{ + Mount: &csi.VolumeCapability_MountVolume{ + MountFlags: []string{}, + }, + }, + }, + expected: []string{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := identifiers.GetMountFlags(tt.vc) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestIsNFSServiceEnabled(t *testing.T) { + // Define mock client + clientMock := new(gopowerstoremock.Client) + // Initialise variable for nas servers + nasServers := []gopowerstore.NAS{ + { + NfsServers: []gopowerstore.NFSServerInstance{ + { + ID: "4444", + IsNFSv4Enabled: true, + }, + }, + }, + } + + // Test cases + t.Run("nfs service is enabled", func(t *testing.T) { + clientMock.On("GetNASServers", mock.Anything, mock.Anything).Return(nasServers, nil) + result, err := identifiers.IsNFSServiceEnabled(context.Background(), clientMock) + assert.NoError(t, err) + assert.True(t, result, "Expected result to be true") + }) + + t.Run("nfs service is not enabled", func(t *testing.T) { + nasServers[0].NfsServers[0].IsNFSv4Enabled = false + clientMock.On("GetNASServers", mock.Anything, mock.Anything).Return(nasServers, nil) + result, err := identifiers.IsNFSServiceEnabled(context.Background(), clientMock) + assert.NoError(t, err) + assert.False(t, result, "Expected result to be false") + }) +} + +func TestGetPowerStoreAPITimeout(t *testing.T) { + tests := []struct { + name string + expected time.Duration + setupFunc func() + teardownFunc func() + }{ + { + name: "env variable is not set", + expected: 120 * time.Second, + }, + { + name: "env variable is set to valid value", + expected: 10 * time.Second, + setupFunc: func() { os.Setenv("X_CSI_POWERSTORE_API_TIMEOUT", "10s") }, + teardownFunc: func() { os.Unsetenv("X_CSI_POWERSTORE_API_TIMEOUT") }, + }, + { + name: "env variable is set to invalid value", + expected: 120 * time.Second, + setupFunc: func() { os.Setenv("X_CSI_POWERSTORE_API_TIMEOUT", "abc") }, + teardownFunc: func() { os.Unsetenv("X_CSI_POWERSTORE_API_TIMEOUT") }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.setupFunc != nil { + tt.setupFunc() + defer tt.teardownFunc() + } + actual := identifiers.GetPowerStoreRESTApiTimeout() + if actual != tt.expected { + t.Errorf("GetTimeout() = %v, want %v", actual, tt.expected) + } + }) + } +} + +func TestGetPodmonArrayConnectivityTimeout(t *testing.T) { + tests := []struct { + name string + expected time.Duration + setupFunc func() + teardownFunc func() + }{ + { + name: "env variable is not set", + expected: 10 * time.Second, + }, + { + name: "env variable is set to valid value", + expected: 25 * time.Second, + setupFunc: func() { os.Setenv("X_CSI_PODMON_ARRAY_CONNECTIVITY_TIMEOUT", "25s") }, + teardownFunc: func() { os.Unsetenv("X_CSI_PODMON_ARRAY_CONNECTIVITY_TIMEOUT") }, + }, + { + name: "env variable is set to invalid value", + expected: 10 * time.Second, + setupFunc: func() { os.Setenv("X_CSI_PODMON_ARRAY_CONNECTIVITY_TIMEOUT", "abc") }, + teardownFunc: func() { os.Unsetenv("X_CSI_PODMON_ARRAY_CONNECTIVITY_TIMEOUT") }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.setupFunc != nil { + tt.setupFunc() + defer tt.teardownFunc() + } + + actual := identifiers.GetPodmonArrayConnectivityTimeout() + if actual != tt.expected { + t.Errorf("GetTimeout() = %v, want %v", actual, tt.expected) + } + }) + } +} + +func TestGetVolumeDisconnectTimeout(t *testing.T) { + tests := []struct { + name string + expected time.Duration + setupFunc func() + teardownFunc func() + }{ + { + name: "env variable is not set", + expected: 120 * time.Second, // DefaultVolumeDisconnectTimeout + }, + { + name: "env variable is set to valid value", + expected: 45 * time.Second, + setupFunc: func() { os.Setenv("X_CSI_VOLUME_DISCONNECT_TIMEOUT_SECONDS", "45s") }, + teardownFunc: func() { os.Unsetenv("X_CSI_VOLUME_DISCONNECT_TIMEOUT_SECONDS") }, + }, + { + name: "env variable is set to invalid value", + expected: 120 * time.Second, + setupFunc: func() { os.Setenv("X_CSI_VOLUME_DISCONNECT_TIMEOUT_SECONDS", "invalid") }, + teardownFunc: func() { os.Unsetenv("X_CSI_VOLUME_DISCONNECT_TIMEOUT_SECONDS") }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.setupFunc != nil { + tt.setupFunc() + defer tt.teardownFunc() + } + + actual := identifiers.GetVolumeDisconnectTimeout() + if actual != tt.expected { + t.Errorf("GetVolumeDisconnectTimeout() = %v, want %v", actual, tt.expected) + } + }) + } +} + +func TestGetVolumeDisconnectRetryInterval(t *testing.T) { + tests := []struct { + name string + expected time.Duration + setupFunc func() + teardownFunc func() + }{ + { + name: "env variable is not set", + expected: 5 * time.Second, // DefaultVolumeDisconnectRetryInterval + }, + { + name: "env variable is set to valid value", + expected: 15 * time.Second, + setupFunc: func() { os.Setenv("X_CSI_VOLUME_DISCONNECT_RETRY_INTERVAL", "15s") }, + teardownFunc: func() { os.Unsetenv("X_CSI_VOLUME_DISCONNECT_RETRY_INTERVAL") }, + }, + { + name: "env variable is set to invalid value", + expected: 5 * time.Second, + setupFunc: func() { os.Setenv("X_CSI_VOLUME_DISCONNECT_RETRY_INTERVAL", "invalid") }, + teardownFunc: func() { os.Unsetenv("X_CSI_VOLUME_DISCONNECT_RETRY_INTERVAL") }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.setupFunc != nil { + tt.setupFunc() + defer tt.teardownFunc() + } + + actual := identifiers.GetVolumeDisconnectRetryInterval() + if actual != tt.expected { + t.Errorf("GetVolumeDisconnectRetryInterval() = %v, want %v", actual, tt.expected) + } + }) + } +} + +func TestGetVolumeDisconnectMaxRetries(t *testing.T) { + tests := []struct { + name string + expected int + setupFunc func() + teardownFunc func() + }{ + { + name: "env variable is not set", + expected: 5, // DefaultVolumeDisconnectMaxRetries + }, + { + name: "env variable is set to valid value", + expected: 7, + setupFunc: func() { os.Setenv("X_CSI_VOLUME_DISCONNECT_MAX_RETRIES", "7") }, + teardownFunc: func() { os.Unsetenv("X_CSI_VOLUME_DISCONNECT_MAX_RETRIES") }, + }, + { + name: "env variable is set to invalid value", + expected: 5, + setupFunc: func() { os.Setenv("X_CSI_VOLUME_DISCONNECT_MAX_RETRIES", "invalid") }, + teardownFunc: func() { os.Unsetenv("X_CSI_VOLUME_DISCONNECT_MAX_RETRIES") }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.setupFunc != nil { + tt.setupFunc() + defer tt.teardownFunc() + } + + actual := identifiers.GetVolumeDisconnectMaxRetries() + if actual != tt.expected { + t.Errorf("GetVolumeDisconnectMaxRetries() = %v, want %v", actual, tt.expected) + } + }) + } +} + +func TestHostAlreadyPresentInNFSExport(t *testing.T) { + tests := []struct { + name string + export gopowerstore.NFSExport + ip string + want bool + }{ + { + name: "present_in_RWHosts", + export: gopowerstore.NFSExport{RWHosts: []string{"10.0.0.1/255.255.255.255"}}, + ip: "10.0.0.1", + want: true, + }, + { + name: "present_in_RWRootHosts", + export: gopowerstore.NFSExport{RWRootHosts: []string{"192.168.0.5/255.255.255.255"}}, + ip: "192.168.0.5", + want: true, + }, + { + name: "present_in_ROHosts", + export: gopowerstore.NFSExport{ROHosts: []string{"172.16.10.10/255.255.255.255"}}, + ip: "172.16.10.10", + want: true, + }, + { + name: "present_in_RORootHosts", + export: gopowerstore.NFSExport{RORootHosts: []string{"10.10.10.10/255.255.255.255"}}, + ip: "10.10.10.10", + want: true, + }, + { + name: "not_present_different_ip", + export: gopowerstore.NFSExport{RWHosts: []string{"10.0.0.2/255.255.255.255"}, ROHosts: []string{"10.0.0.3/255.255.255.255"}}, + ip: "10.0.0.1", + want: false, + }, + { + name: "not_present_empty_lists", + export: gopowerstore.NFSExport{}, + ip: "10.0.0.1", + want: false, + }, + { + name: "present_in_multiple_lists", + export: gopowerstore.NFSExport{RWHosts: []string{"1.2.3.4/255.255.255.255"}, RORootHosts: []string{"5.6.7.8/255.255.255.255"}}, + ip: "1.2.3.4", + want: true, + }, + { + name: "ip_with_trailing_spaces_not_present", + export: gopowerstore.NFSExport{RWHosts: []string{"8.8.8.8/255.255.255.255"}}, + ip: "8.8.8.8 ", + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := identifiers.HostAlreadyPresentInNFSExport(tt.export, tt.ip) + if got != tt.want { + t.Errorf("HostAlreadyPresentInNFSExport() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/identifiers/k8sutils/k8sutils.go b/pkg/identifiers/k8sutils/k8sutils.go new file mode 100644 index 00000000..a00ae308 --- /dev/null +++ b/pkg/identifiers/k8sutils/k8sutils.go @@ -0,0 +1,245 @@ +/* + Copyright (c) 2023-2025 Dell Inc, or its subsidiaries. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package k8sutils + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + + "github.com/dell/csmlog" + k8score "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" +) + +type K8sClient struct { + Clientset kubernetes.Interface +} + +// Instantiate csmlog on a package level +var log = csmlog.GetLogger() + +// Kube Kubeclient +var Kubeclient *K8sClient + +// used for unit testing - +// allows CreateKubeClientSet to be mocked +var InClusterConfigFunc = func() (*rest.Config, error) { + return rest.InClusterConfig() +} + +var NewForConfigFunc = func(config *rest.Config) (kubernetes.Interface, error) { + return kubernetes.NewForConfig(config) +} + +// CreateKubeClientSet creates kubeclient set if not created already +func CreateKubeClientSet(kubeconfig ...string) (*K8sClient, error) { + Kubeclient = &K8sClient{} + + config, err := InClusterConfigFunc() + if err != nil { + if len(kubeconfig) == 0 { + return nil, err + } + + config, err = clientcmd.BuildConfigFromFlags("", kubeconfig[0]) + if err != nil { + return nil, err + } + } + + Kubeclient.Clientset, err = NewForConfigFunc(config) + if err != nil { + return nil, fmt.Errorf("failed to create Kubernetes clientset: %s", err.Error()) + } + + return Kubeclient, nil +} + +func (k8s *K8sClient) GetNode(ctx context.Context, kubeNodeName string) (*k8score.Node, error) { + if k8s.Clientset == nil { + return nil, fmt.Errorf("unable to get node %q, kubernetes client is uninitialized", kubeNodeName) + } + + node, err := k8s.Clientset.CoreV1().Nodes().Get(ctx, kubeNodeName, v1.GetOptions{}) + if err != nil { + return nil, err + } + + return node, nil +} + +// GetNodeLabels retrieves the kubernetes node object and returns its labels +func (k8s *K8sClient) GetNodeLabels(ctx context.Context, kubeNodeName string) (map[string]string, error) { + if k8s.Clientset == nil { + return nil, fmt.Errorf("unable to get node labels for node %q, kubernetes client is uninitialized", kubeNodeName) + } + + node, err := k8s.GetNode(ctx, kubeNodeName) + if err != nil { + return nil, err + } + + return node.Labels, nil +} + +// GetNodeLabels retrieves the kubernetes node object and returns its labels +func (k8s *K8sClient) SetNodeLabel(ctx context.Context, kubeNodeName string, labelKey string, labelValue string) error { + if k8s.Clientset == nil { + return fmt.Errorf("unable to get node labels for node %q, kubernetes client is uninitialized", kubeNodeName) + } + + node, err := k8s.GetNode(ctx, kubeNodeName) + if err != nil { + return err + } + + // Update the node with the new labels + node.Labels[labelKey] = labelValue + _, err = k8s.Clientset.CoreV1().Nodes().Update(ctx, node, v1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("failed to update node %s labels: %v", kubeNodeName, err.Error()) + } + + return nil +} + +// AddNVMeLabels adds a hostnqn uuid label to the specified Kubernetes node +func (k8s *K8sClient) AddNVMeLabels(ctx context.Context, kubeNodeName string, labelKey string, labelValue []string) error { + if k8s.Clientset == nil { + return fmt.Errorf("unable to add NVMe labels to node %q, kubernetes client is uninitialized", kubeNodeName) + } + + // Get the current node + node, err := k8s.GetNode(ctx, kubeNodeName) + if err != nil { + return fmt.Errorf("failed to get node %s: %v", kubeNodeName, err.Error()) + } + + // Initialize node labels if it is nil + if node.Labels == nil { + node.Labels = make(map[string]string) + } + + // Fetch the uuids from hostnqns + var uuids []string + for _, nqn := range labelValue { + parts := strings.Split(nqn, ":") + if len(parts) == 3 { // nqn format is nqn.yyyy-mm.nvmexpress:uuid:xxxx-yyyy-zzzz + uuids = append(uuids, parts[2]) // Extract the UUID + } + } + + // Update the node with the new labels + node.Labels[labelKey] = strings.Join(uuids, ",") + _, err = k8s.Clientset.CoreV1().Nodes().Update(ctx, node, v1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("failed to update node %s labels: %v", kubeNodeName, err.Error()) + } + return nil +} + +// GetNVMeUUIDs returns map of hosts with their hostnqn uuids +func (k8s *K8sClient) GetNVMeUUIDs(ctx context.Context) (map[string]string, error) { + nodeUUIDs := make(map[string]string) + if k8s.Clientset == nil { + return nodeUUIDs, errors.New("unable to get NVMe UUIDs, kubernetes client is uninitialized") + } + + // Retrieve the list of nodes + nodes, err := k8s.Clientset.CoreV1().Nodes().List(ctx, v1.ListOptions{}) + if err != nil { + return nodeUUIDs, fmt.Errorf("failed to get node list: %v", err.Error()) + } + + // Iterate over all nodes to check their labels + for _, node := range nodes.Items { + labels := node.Labels + if uuid, exists := labels["hostnqn-uuid"]; exists { + nodeUUIDs[node.Name] = uuid + } + } + + return nodeUUIDs, nil +} + +func (k8s *K8sClient) GetNodeByCSINodeID(ctx context.Context, driverKey string, csiNodeID string, keyNodeID string) (*k8score.Node, error) { + if k8s.Clientset == nil { + return nil, errors.New("unable to get node, kubernetes client is uninitialized") + } + + // Retrieve the list of nodes + nodes, err := k8s.Clientset.CoreV1().Nodes().List(ctx, v1.ListOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get node list: %v", err.Error()) + } + + for _, node := range nodes.Items { + if annotation, exists := node.Annotations[keyNodeID]; exists { + var nodeIDMap map[string]string + if err := json.Unmarshal([]byte(annotation), &nodeIDMap); err != nil { + continue + } + + if value, found := nodeIDMap[driverKey]; found && value == csiNodeID { + return &node, nil + } + } + } + + return nil, fmt.Errorf("failed to find a Node matching csiNodeId %s", csiNodeID) +} + +// ListVolumes lists all persistent volumes. +func (k8s *K8sClient) ListPersistentVolumes(ctx context.Context) (*k8score.PersistentVolumeList, error) { + if k8s.Clientset == nil { + return nil, errors.New("unable to list volumes, kubernetes client is uninitialized") + } + + return k8s.Clientset.CoreV1().PersistentVolumes().List(ctx, v1.ListOptions{}) +} + +// GetEvents gets events for the named resource of the given kind in the given namespace. +// If name is empty, it gets all events for the given kind in the given namespace. +func (k8s *K8sClient) GetEvents(ctx context.Context, kind, name, namespace string) (*k8score.EventList, error) { + if k8s.Clientset == nil { + return nil, errors.New("unable to get events, kubernetes client is uninitialized") + } + + var fieldSelector string + if name != "" { + fieldSelector = fmt.Sprintf("involvedObject.name=%s", name) + } + if kind != "" { + fieldSelector = strings.Join([]string{fieldSelector, fmt.Sprintf("involvedObject.kind=%s", kind)}, ",") + } + + log.Debugf("getting events for %q %q in namespace %q", kind, name, namespace) + + return k8s.Clientset.CoreV1().Events(namespace).List(ctx, v1.ListOptions{ + FieldSelector: fieldSelector, + }) +} + +var GetNodeByCSINodeID = func(ctx context.Context, driverKey string, csiNodeID string, keyNodeID string) (*k8score.Node, error) { + return Kubeclient.GetNodeByCSINodeID(ctx, driverKey, csiNodeID, keyNodeID) +} diff --git a/pkg/identifiers/k8sutils/k8sutils_test.go b/pkg/identifiers/k8sutils/k8sutils_test.go new file mode 100644 index 00000000..eb229516 --- /dev/null +++ b/pkg/identifiers/k8sutils/k8sutils_test.go @@ -0,0 +1,565 @@ +/* + * Copyright © 2025 Dell Inc. or its subsidiaries. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package k8sutils + +import ( + "context" + "errors" + "os" + "reflect" + "testing" + + "github.com/dell/csi-powerstore/v2/pkg/identifiers" + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/rest" +) + +const ( + testEventKind string = "PersistentVolume" + testNamespace string = "default" + testVolName string = "csivol-aabccdd" +) + +var ( + testPV *corev1.PersistentVolume = &corev1.PersistentVolume{ + TypeMeta: v1.TypeMeta{ + Kind: "PersistentVolume", + }, + ObjectMeta: v1.ObjectMeta{ + Name: testVolName, + Namespace: "", + }, + Spec: corev1.PersistentVolumeSpec{ + Capacity: corev1.ResourceList{ + "storage": resource.MustParse("10Gi"), + }, + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimDelete, + StorageClassName: "powerstore-block", + }, + } + testVolumeEvent *corev1.Event = &corev1.Event{ + TypeMeta: v1.TypeMeta{ + APIVersion: "v1", + Kind: "Event", + }, + ObjectMeta: v1.ObjectMeta{ + Name: "event1", + Namespace: "default", + }, + InvolvedObject: corev1.ObjectReference{ + APIVersion: "v1", + Name: testVolName, + Kind: testEventKind, + }, + Type: corev1.EventTypeWarning, + Reason: "Minor", + } + testPodEvent *corev1.Event = &corev1.Event{ + TypeMeta: v1.TypeMeta{ + APIVersion: "v1", + Kind: "Event", + }, + ObjectMeta: v1.ObjectMeta{ + Name: "powerstore-node-event", + Namespace: "powerstore", + }, + InvolvedObject: corev1.ObjectReference{ + Name: "powerstore-node-aabbccdd", + Kind: "Pod", + Namespace: "powerstore", + }, + Type: corev1.EventTypeNormal, + Reason: "Scheduled", + } +) + +func GetMockNodeWithLabels() *corev1.Node { + return &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Annotations: map[string]string{ + "csi.volume.kubernetes.io/nodeid": "{\"node1\":\"myCsiNode\"}", + }, + Labels: map[string]string{ + "max-powerstore-volumes-per-node": "2", + "hostnqn-uuid": "uuid1", + }, + }, + } +} + +func GetMockNodeWithoutLabels() *corev1.Node { + return &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + }, + } +} + +func TestCreateKubeClientSet(t *testing.T) { + var tempConfigFunc func() (*rest.Config, error) // must return getInClusterConfig to its original value + var tempClientsetFunc func(config *rest.Config) (kubernetes.Interface, error) // must return getK8sClientset to its original value + + tests := []struct { + name string + before func(*testing.T) error + after func() + wantErr bool + }{ + { + name: "success: manually set InClusterConfig with mock", + before: func(_ *testing.T) error { + Kubeclient = nil // reset Clientset before each run + tempConfigFunc = InClusterConfigFunc + InClusterConfigFunc = func() (*rest.Config, error) { return &rest.Config{}, nil } + return nil + }, + after: func() { InClusterConfigFunc = tempConfigFunc }, + wantErr: false, + }, + { + name: "failure: unmocked config function", + before: func(tt *testing.T) error { + Kubeclient = nil // reset Clientset before each run + tempConfigFunc = InClusterConfigFunc + // Mock InClusterConfigFunc to return an error to simulate failure + InClusterConfigFunc = func() (*rest.Config, error) { + return nil, errors.New("unable to load in-cluster configuration") + } + // Clear KUBECONFIG to ensure fallback also fails + tt.Setenv(identifiers.EnvKubeConfigPath, "") + return nil + }, + after: func() { + InClusterConfigFunc = tempConfigFunc + }, + wantErr: true, + }, + { + name: "failure: error returned by kubernetes.NewForConfig", + before: func(_ *testing.T) error { // overrides to get past a mock and inject a failure + Kubeclient = nil // reset Clientset before each run + tempConfigFunc = InClusterConfigFunc + tempClientsetFunc = NewForConfigFunc + InClusterConfigFunc = func() (*rest.Config, error) { return &rest.Config{}, nil } + NewForConfigFunc = func(_ *rest.Config) (kubernetes.Interface, error) { + return nil, assert.AnError + } + return nil + }, + after: func() { // restore functions to their defaults + InClusterConfigFunc = tempConfigFunc + NewForConfigFunc = tempClientsetFunc + }, + wantErr: true, + }, + { + name: "fail to get in-cluster config and no kubeconfig provided", + before: func(tt *testing.T) error { + Kubeclient = nil + // keep InClusterConfigFunc as-is + // Clear KUBECONFIG to ensure fallback also fails + tt.Setenv(identifiers.EnvKubeConfigPath, "") + // ensure failure of InClusterConfigFunc + tt.Setenv("KUBERNETES_SERVICE_HOST", "") + return nil + }, + after: func() {}, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.before(t) + defer tt.after() + + _, err := CreateKubeClientSet() + + if tt.wantErr { + assert.Error(t, err) + assert.Nil(t, Kubeclient.Clientset) + } else { + assert.NoError(t, err) + assert.NotNil(t, Kubeclient.Clientset) + } + + // Reset Clientset for the second test call + Kubeclient = nil + + // Test 2: Call CreateKubeClientSet(kubeConfig) with parameters + // For the failure test case, we need to ensure this also fails + if tt.name == "failure: unmocked config function" { + // For this test case, pass an invalid kubeconfig path to ensure failure + _, err = CreateKubeClientSet("/invalid/path/to/kubeconfig") + } else { + // For other tests, use the original kubeconfig (if any) + _, err = CreateKubeClientSet(os.Getenv(identifiers.EnvKubeConfigPath)) + } + + if tt.wantErr { + assert.Error(t, err) + assert.Nil(t, Kubeclient.Clientset) + } else { + assert.NoError(t, err) + assert.NotNil(t, Kubeclient.Clientset) + } + }) + } +} + +func TestGetNode(t *testing.T) { + t.Run("GetNode success", func(t *testing.T) { + Kubeclient = &K8sClient{ + Clientset: fake.NewSimpleClientset(GetMockNodeWithoutLabels()), + } + + _, err := Kubeclient.GetNode(context.Background(), "node1") + assert.NoError(t, err) + }) + + t.Run("GetNode no client", func(t *testing.T) { + Kubeclient = &K8sClient{} + + _, err := Kubeclient.GetNode(context.Background(), "node1") + assert.Error(t, err) + }) + + t.Run("GetNode not found", func(t *testing.T) { + Kubeclient = &K8sClient{ + Clientset: fake.NewSimpleClientset(), + } + + _, err := Kubeclient.GetNode(context.Background(), "node1") + assert.Error(t, err) + assert.Contains(t, err.Error(), "not found") + }) +} + +func TestGetNodeLabels(t *testing.T) { + t.Run("GetNodeLabels success", func(t *testing.T) { + Kubeclient = &K8sClient{ + Clientset: fake.NewSimpleClientset(GetMockNodeWithLabels()), + } + + labels, err := Kubeclient.GetNodeLabels(context.Background(), "node1") + assert.NoError(t, err) + assert.Equal(t, len(labels), 2) + }) + + t.Run("GetNodeLabels success - no labels", func(t *testing.T) { + Kubeclient = &K8sClient{ + Clientset: fake.NewSimpleClientset(GetMockNodeWithoutLabels()), + } + + labels, err := Kubeclient.GetNodeLabels(context.Background(), "node1") + assert.NoError(t, err) + assert.Equal(t, len(labels), 0) + }) + + t.Run("GetNodeLabels no client", func(t *testing.T) { + Kubeclient = &K8sClient{} + + _, err := Kubeclient.GetNodeLabels(context.Background(), "node1") + assert.Error(t, err) + }) + + t.Run("GetNodeLabels not found", func(t *testing.T) { + Kubeclient = &K8sClient{ + Clientset: fake.NewSimpleClientset(), + } + + _, err := Kubeclient.GetNodeLabels(context.Background(), "node1") + assert.Error(t, err) + assert.Contains(t, err.Error(), "not found") + }) +} + +func TestSetNodeLabel(t *testing.T) { + t.Run("SetNodeLabel success", func(t *testing.T) { + Kubeclient = &K8sClient{ + Clientset: fake.NewSimpleClientset(GetMockNodeWithLabels()), + } + + err := Kubeclient.SetNodeLabel(context.Background(), "node1", "topology.kubernetes.io/zone", "zone1") + assert.NoError(t, err) + + labels, err := Kubeclient.GetNodeLabels(context.Background(), "node1") + assert.NoError(t, err) + + assert.Equal(t, labels["topology.kubernetes.io/zone"], "zone1") + }) + + t.Run("SetNodeLabel no client", func(t *testing.T) { + Kubeclient = &K8sClient{} + + err := Kubeclient.SetNodeLabel(context.Background(), "", "", "") + assert.Error(t, err) + }) + + t.Run("SetNodeLabel not found", func(t *testing.T) { + Kubeclient = &K8sClient{ + Clientset: fake.NewSimpleClientset(), + } + + err := Kubeclient.SetNodeLabel(context.Background(), "node1", "", "") + assert.Error(t, err) + assert.Contains(t, err.Error(), "not found") + }) +} + +func TestGetNVMeUUIDs(t *testing.T) { + t.Run("GetNVMeUUIDs success", func(t *testing.T) { + Kubeclient = &K8sClient{ + Clientset: fake.NewSimpleClientset(GetMockNodeWithLabels()), + } + + nodeUUIDs, err := Kubeclient.GetNVMeUUIDs(context.Background()) + assert.NoError(t, err) + assert.Equal(t, map[string]string{"node1": "uuid1"}, nodeUUIDs) + }) + + t.Run("GetNVMeUUIDs success - no uuids", func(t *testing.T) { + Kubeclient = &K8sClient{ + Clientset: fake.NewSimpleClientset(GetMockNodeWithoutLabels()), + } + + nodeUUIDs, err := Kubeclient.GetNVMeUUIDs(context.Background()) + assert.NoError(t, err) + assert.Equal(t, len(nodeUUIDs), 0) + }) + + t.Run("GetNVMeUUIDs no client", func(t *testing.T) { + Kubeclient = &K8sClient{} + + _, err := Kubeclient.GetNVMeUUIDs(context.Background()) + assert.Error(t, err) + }) +} + +func TestAddNVMeLabels(t *testing.T) { + t.Run("AddNVMeLabels success", func(t *testing.T) { + Kubeclient = &K8sClient{ + Clientset: fake.NewSimpleClientset(GetMockNodeWithLabels()), + } + + err := Kubeclient.AddNVMeLabels(context.Background(), "node1", "newNVME", []string{"nqn.yyyy-mm.nvmexpress:uuid:xxxx-yyyy-zzzz"}) + assert.NoError(t, err) + }) + + t.Run("AddNVMeLabels success - empty labels", func(t *testing.T) { + Kubeclient = &K8sClient{ + Clientset: fake.NewSimpleClientset(GetMockNodeWithoutLabels()), + } + + err := Kubeclient.AddNVMeLabels(context.Background(), "node1", "newNVME", []string{"nqn.yyyy-mm.nvmexpress:uuid:xxxx-yyyy-zzzz"}) + assert.NoError(t, err) + }) + + t.Run("AddNVMeLabels no client", func(t *testing.T) { + Kubeclient = &K8sClient{} + + err := Kubeclient.AddNVMeLabels(context.Background(), "node1", "newNVME", []string{"nqn.yyyy-mm.nvmexpress:uuid:xxxx-yyyy-zzzz"}) + assert.Error(t, err) + }) +} + +func TestGetNodeByCSINodeID(t *testing.T) { + t.Run("GetNodeByCSINodeID success", func(t *testing.T) { + Kubeclient = &K8sClient{ + Clientset: fake.NewSimpleClientset(GetMockNodeWithLabels()), + } + + _, err := Kubeclient.GetNodeByCSINodeID(context.Background(), "node1", "myCsiNode", "csi.volume.kubernetes.io/nodeid") + assert.NoError(t, err) + }) + + t.Run("GetNodeByCSINodeID failed", func(t *testing.T) { + Kubeclient = &K8sClient{ + Clientset: fake.NewSimpleClientset(GetMockNodeWithoutLabels()), + } + + _, err := Kubeclient.GetNodeByCSINodeID(context.Background(), "node1", "myCsiNode", "csi.volume.kubernetes.io/nodeid") + assert.Error(t, err) + }) + + t.Run("GetNodeByCSINodeID failed - no node found", func(t *testing.T) { + Kubeclient = &K8sClient{ + Clientset: fake.NewSimpleClientset(GetMockNodeWithoutLabels()), + } + + _, err := Kubeclient.GetNodeByCSINodeID(context.Background(), "invalidNode", "myCsiNode", "csi.volume.kubernetes.io/nodeid") + assert.Error(t, err) + }) + + t.Run("GetNodeByCSINodeID no client", func(t *testing.T) { + Kubeclient = &K8sClient{} + + _, err := Kubeclient.GetNodeByCSINodeID(context.Background(), "node1", "myCsiNode", "csi.volume.kubernetes.io/nodeid") + assert.Error(t, err) + }) +} + +func TestK8sClient_ListPersistentVolumes(t *testing.T) { + tests := []struct { + name string // description of this test case + // Named input parameters for receiver constructor. + k8s func() *K8sClient + want *corev1.PersistentVolumeList + wantErr bool + }{ + { + name: "uninitialized kube client", + k8s: func() *K8sClient { + return &K8sClient{} + }, + want: nil, + wantErr: true, + }, + { + name: "success", + k8s: func() *K8sClient { + return &K8sClient{ + Clientset: fake.NewClientset([]runtime.Object{testPV}...), + } + }, + want: &corev1.PersistentVolumeList{Items: []corev1.PersistentVolume{*testPV}}, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + k8s := tt.k8s() + + got, gotErr := k8s.ListPersistentVolumes(context.Background()) + if gotErr != nil { + if !tt.wantErr { + t.Errorf("ListPersistentVolumes() failed: %v", gotErr) + } + return + } + if tt.wantErr { + t.Fatal("ListPersistentVolumes() succeeded unexpectedly") + } + + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ListPersistentVolumes() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestK8sClient_GetEvents(t *testing.T) { + type args struct { + name string + kind string + namespace string + } + tests := []struct { + name string // description of this test case + // Named input parameters for receiver constructor. + k8s func() *K8sClient + args args + want *corev1.EventList + wantErr bool + }{ + { + name: "client is uninitialized", + k8s: func() *K8sClient { return &K8sClient{} }, + args: args{ + name: testVolName, + kind: testEventKind, + namespace: "", + }, + want: nil, + wantErr: true, + }, + // NOTE: "k8s.io/client-go/kubernetes/fake" as of version v0.34.0 does not support filtering events + // based on FieldSelector, so we cannot accurately test the functionality of the GetEvents() func. + // { + // name: "without a kind", + // k8s: func() *K8sClient { + // client := fake.NewClientset([]runtime.Object{testVolumeEvent, testPodEvent}...) + // return &K8sClient{ + // Clientset: client, + // } + // }, + // args: args{ + // name: testVolName, + // kind: "", + // namespace: "", + // }, + // want: &corev1.EventList{Items: []corev1.Event{*testVolumeEvent}}, + // wantErr: false, + // }, + { + name: "gets all events in the default namespace", + k8s: func() *K8sClient { + client := fake.NewClientset([]runtime.Object{testVolumeEvent, testPodEvent}...) + return &K8sClient{ + Clientset: client, + } + }, + args: args{ + name: "", + kind: "", + namespace: "default", + }, + want: &corev1.EventList{Items: []corev1.Event{*testVolumeEvent}}, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + k8s := tt.k8s() + got, gotErr := k8s.GetEvents(context.Background(), tt.args.kind, tt.args.name, tt.args.namespace) + if gotErr != nil { + if !tt.wantErr { + t.Errorf("GetEvents() failed: %v", gotErr) + } + return + } + if tt.wantErr { + t.Fatal("GetEvents() succeeded unexpectedly") + } + if !reflect.DeepEqual(tt.want, got) { + t.Errorf("GetEvents() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestGetNodeByCSINodeIDVar(t *testing.T) { + t.Run("GetNodeByCSINodeIDVar success", func(t *testing.T) { + Kubeclient = &K8sClient{ + Clientset: fake.NewSimpleClientset(GetMockNodeWithLabels()), + } + + _, err := GetNodeByCSINodeID(context.Background(), "node1", "myCsiNode", "csi.volume.kubernetes.io/nodeid") + assert.NoError(t, err) + }) +} diff --git a/pkg/common/logger.go b/pkg/identifiers/logger.go similarity index 70% rename from pkg/common/logger.go rename to pkg/identifiers/logger.go index 78b0e458..f79c5a53 100644 --- a/pkg/common/logger.go +++ b/pkg/identifiers/logger.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2021 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2021-2025 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,27 +16,26 @@ * */ -package common +package identifiers import ( "context" - log "github.com/sirupsen/logrus" ) // CustomLogger is logger wrapper that can be passed to gopowerstore, gobrick allowing to logging context fields with each call type CustomLogger struct{} -// Info is a wrapper of logrus Info method +// Info is a wrapper of csmlog Info method func (lg *CustomLogger) Info(ctx context.Context, format string, args ...interface{}) { - log.WithFields(GetLogFields(ctx)).Infof(format, args...) + log.WithContext(ctx).Infof(format, args...) } -// Debug is a wrapper of logrus Debug method +// Debug is a wrapper of csmlog Debug method func (lg *CustomLogger) Debug(ctx context.Context, format string, args ...interface{}) { - log.WithFields(GetLogFields(ctx)).Debugf(format, args...) + log.WithContext(ctx).Debugf(format, args...) } -// Error is a wrapper of logrus Error method +// Error is a wrapper of csmlog Error method func (lg *CustomLogger) Error(ctx context.Context, format string, args ...interface{}) { - log.WithFields(GetLogFields(ctx)).Errorf(format, args...) + log.WithContext(ctx).Errorf(format, args...) } diff --git a/pkg/identity/identity.go b/pkg/identity/identity.go index a762ad4b..22d9acc9 100644 --- a/pkg/identity/identity.go +++ b/pkg/identity/identity.go @@ -21,8 +21,9 @@ package identity import ( "context" + "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/golang/protobuf/ptypes/wrappers" + "google.golang.org/protobuf/types/known/wrapperspb" ) // NewIdentityService creates new identity service @@ -44,7 +45,7 @@ type Service struct { } // GetPluginInfo returns general information about plugin (driver) such as name, version and manifest -func (s Service) GetPluginInfo(context context.Context, request *csi.GetPluginInfoRequest) (*csi.GetPluginInfoResponse, error) { +func (s Service) GetPluginInfo(_ context.Context, _ *csi.GetPluginInfoRequest) (*csi.GetPluginInfoResponse, error) { return &csi.GetPluginInfoResponse{ Name: s.name, VendorVersion: s.version, @@ -53,7 +54,7 @@ func (s Service) GetPluginInfo(context context.Context, request *csi.GetPluginIn } // GetPluginCapabilities returns capabilities that are supported by the driver -func (s Service) GetPluginCapabilities(context context.Context, request *csi.GetPluginCapabilitiesRequest) (*csi.GetPluginCapabilitiesResponse, error) { +func (s Service) GetPluginCapabilities(_ context.Context, _ *csi.GetPluginCapabilitiesRequest) (*csi.GetPluginCapabilitiesResponse, error) { var rep csi.GetPluginCapabilitiesResponse rep.Capabilities = []*csi.PluginCapability{ { @@ -90,6 +91,6 @@ func (s Service) GetPluginCapabilities(context context.Context, request *csi.Get } // Probe returns current state of the driver and if it is ready to receive requests -func (s Service) Probe(context context.Context, request *csi.ProbeRequest) (*csi.ProbeResponse, error) { - return &csi.ProbeResponse{Ready: &wrappers.BoolValue{Value: s.ready}}, nil +func (s Service) Probe(_ context.Context, _ *csi.ProbeRequest) (*csi.ProbeResponse, error) { + return &csi.ProbeResponse{Ready: &wrapperspb.BoolValue{Value: s.ready}}, nil } diff --git a/pkg/identity/identity_test.go b/pkg/identity/identity_test.go index a8b9b34d..8b8e1aea 100644 --- a/pkg/identity/identity_test.go +++ b/pkg/identity/identity_test.go @@ -22,36 +22,36 @@ import ( "context" "testing" + "github.com/dell/csi-powerstore/v2/pkg/identifiers" "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/dell/csi-powerstore/v2/pkg/common" - "github.com/golang/protobuf/ptypes/wrappers" - . "github.com/onsi/ginkgo" + ginkgo "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" - . "github.com/onsi/gomega" + gomega "github.com/onsi/gomega" + "google.golang.org/protobuf/types/known/wrapperspb" ) var idntySvc *Service func TestCSIIdentityService(t *testing.T) { - RegisterFailHandler(Fail) + gomega.RegisterFailHandler(ginkgo.Fail) junitReporter := reporters.NewJUnitReporter("idnty-svc.xml") - RunSpecsWithDefaultAndCustomReporters(t, "CSIIdentityService testing suite", []Reporter{junitReporter}) + ginkgo.RunSpecsWithDefaultAndCustomReporters(t, "CSIIdentityService testing suite", []ginkgo.Reporter{junitReporter}) } func setVariables() { - idntySvc = NewIdentityService(common.Name, "v1.3.0", common.Manifest) + idntySvc = NewIdentityService(identifiers.Name, "v1.3.0", identifiers.Manifest) } -var _ = Describe("CSIIdentityService", func() { - BeforeEach(func() { +var _ = ginkgo.Describe("CSIIdentityService", func() { + ginkgo.BeforeEach(func() { setVariables() }) - Describe("calling GetPluginInfo()", func() { - It("should return correct info", func() { + ginkgo.Describe("calling GetPluginInfo()", func() { + ginkgo.It("should return correct info", func() { res, err := idntySvc.GetPluginInfo(context.Background(), &csi.GetPluginInfoRequest{}) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.GetPluginInfoResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.GetPluginInfoResponse{ Name: idntySvc.name, VendorVersion: idntySvc.version, Manifest: idntySvc.manifest, @@ -59,11 +59,11 @@ var _ = Describe("CSIIdentityService", func() { }) }) - Describe("calling GetPluginCapabilities()", func() { - It("should return correct capabilities", func() { + ginkgo.Describe("calling GetPluginCapabilities()", func() { + ginkgo.It("should return correct capabilities", func() { res, err := idntySvc.GetPluginCapabilities(context.Background(), &csi.GetPluginCapabilitiesRequest{}) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.GetPluginCapabilitiesResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.GetPluginCapabilitiesResponse{ Capabilities: []*csi.PluginCapability{ { Type: &csi.PluginCapability_Service_{ @@ -99,11 +99,11 @@ var _ = Describe("CSIIdentityService", func() { }) }) - Describe("calling Probe()", func() { - It("should return current status'", func() { + ginkgo.Describe("calling Probe()", func() { + ginkgo.It("should return current status'", func() { res, err := idntySvc.Probe(context.Background(), &csi.ProbeRequest{}) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.ProbeResponse{Ready: &wrappers.BoolValue{Value: idntySvc.ready}})) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.ProbeResponse{Ready: &wrapperspb.BoolValue{Value: idntySvc.ready}})) }) }) }) diff --git a/pkg/interceptors/interceptors.go b/pkg/interceptors/interceptors.go index fb16506b..ad682df8 100644 --- a/pkg/interceptors/interceptors.go +++ b/pkg/interceptors/interceptors.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2021-2023 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2021-2025 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,19 +26,19 @@ import ( "sync" "time" - "github.com/akutz/gosync" - "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/dell/csi-powerstore/v2/pkg/common" controller "github.com/dell/csi-powerstore/v2/pkg/controller" + "github.com/dell/csi-powerstore/v2/pkg/identifiers" "github.com/dell/gocsi/middleware/serialvolume" + "github.com/akutz/gosync" + "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" + "github.com/dell/csmlog" csictx "github.com/dell/gocsi/context" - mwtypes "github.com/dell/gocsi/middleware/serialvolume/types" - log "github.com/sirupsen/logrus" + mwtypes "github.com/dell/gocsi/middleware/serialvolume/lockprovider" xctx "golang.org/x/net/context" "github.com/dell/csi-metadata-retriever/retriever" @@ -46,10 +46,14 @@ import ( "github.com/kubernetes-csi/csi-lib-utils/metrics" ) +// Instantial csmlog on a package level +var log = csmlog.GetLogger() + type rewriteRequestIDInterceptor struct{} func (r *rewriteRequestIDInterceptor) handleServer(ctx context.Context, req interface{}, - info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { + _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler, +) (interface{}, error) { // Retrieve the gRPC metadata from the incoming context. md, mdOK := metadata.FromIncomingContext(ctx) @@ -78,7 +82,7 @@ type lockProvider struct { volNameLocks map[string]gosync.TryLocker } -func (i *lockProvider) GetLockWithID(ctx context.Context, id string) (gosync.TryLocker, error) { +func (i *lockProvider) GetLockWithID(_ context.Context, id string) (gosync.TryLocker, error) { i.volIDLocksL.Lock() defer i.volIDLocksL.Unlock() @@ -91,7 +95,7 @@ func (i *lockProvider) GetLockWithID(ctx context.Context, id string) (gosync.Try return lock, nil } -func (i *lockProvider) GetLockWithName(ctx context.Context, name string) (gosync.TryLocker, error) { +func (i *lockProvider) GetLockWithName(_ context.Context, name string) (gosync.TryLocker, error) { i.volNameLocksL.Lock() defer i.volNameLocksL.Unlock() @@ -144,10 +148,11 @@ func NewCustomSerialLock(mode string) grpc.UnaryServerInterceptor { } func (i *interceptor) createMetadataRetrieverClient(ctx context.Context) { + log := log.WithContext(ctx) metricsManager := metrics.NewCSIMetricsManagerWithOptions("csi-metadata-retriever", metrics.WithProcessStartTime(false), metrics.WithSubsystem(metrics.SubsystemSidecar)) - if retrieverAddress, ok := csictx.LookupEnv(ctx, common.EnvMetadataRetrieverEndpoint); ok { + if retrieverAddress, ok := csictx.LookupEnv(ctx, identifiers.EnvMetadataRetrieverEndpoint); ok { rpcConn, err := connection.Connect(retrieverAddress, metricsManager, connection.OnConnectionLoss(connection.ExitOnConnectionLoss())) if err != nil { log.Error(err.Error()) @@ -160,14 +165,15 @@ func (i *interceptor) createMetadataRetrieverClient(ctx context.Context) { i.opts.MetadataSidecarClient = retrieverClient } else { - log.Warn("env var not found: ", common.EnvMetadataRetrieverEndpoint) + log.Warnf("env var not found: %s", identifiers.EnvMetadataRetrieverEndpoint) } } const pending = "pending" func (i *interceptor) nodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest, - info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (res interface{}, resErr error) { + _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler, +) (res interface{}, resErr error) { lock, err := i.opts.locker.GetLockWithID(ctx, req.VolumeId) if err != nil { return nil, err @@ -186,7 +192,8 @@ func (i *interceptor) nodeStageVolume(ctx context.Context, req *csi.NodeStageVol } func (i *interceptor) nodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest, - info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (res interface{}, resErr error) { + _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler, +) (res interface{}, resErr error) { lock, err := i.opts.locker.GetLockWithID(ctx, req.VolumeId) if err != nil { return nil, err @@ -203,8 +210,9 @@ func (i *interceptor) nodeUnstageVolume(ctx context.Context, req *csi.NodeUnstag } func (i *interceptor) createVolume(ctx context.Context, req *csi.CreateVolumeRequest, - info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (res interface{}, resErr error) { - + _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler, +) (res interface{}, resErr error) { + log := log.WithContext(ctx) lock, err := i.opts.locker.GetLockWithID(ctx, req.Name) if err != nil { return nil, err diff --git a/pkg/interceptors/interceptors_test.go b/pkg/interceptors/interceptors_test.go index 8cc603b6..17edfc86 100644 --- a/pkg/interceptors/interceptors_test.go +++ b/pkg/interceptors/interceptors_test.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2021-2023 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2021-2025 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,14 +20,21 @@ package interceptors import ( "context" + "errors" "fmt" "sync" "testing" "time" - "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/dell/csi-metadata-retriever/retriever" + controller "github.com/dell/csi-powerstore/v2/pkg/controller" + "github.com/dell/csi-powerstore/v2/pkg/identifiers" csictx "github.com/dell/gocsi/context" + lockprovider "github.com/dell/gocsi/middleware/serialvolume/lockprovider" + "github.com/akutz/gosync" + "github.com/container-storage-interface/spec/lib/go/csi" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "google.golang.org/grpc" "google.golang.org/grpc/metadata" ) @@ -39,7 +46,7 @@ const ( ) func getSleepHandler(millisec int) grpc.UnaryHandler { - return func(ctx context.Context, req interface{}) (interface{}, error) { + return func(_ context.Context, _ interface{}) (interface{}, error) { fmt.Println("start sleep") time.Sleep(time.Duration(millisec) * time.Millisecond) fmt.Println("stop sleep") @@ -80,7 +87,6 @@ func TestNewCustomSerialLock(t *testing.T) { _, err := serialLock(ctx, req2, nil, h) wg.Wait() return err - } t.Run("NodeStage for same volume concurrent call", func(t *testing.T) { err := runTest(&csi.NodeStageVolumeRequest{VolumeId: validBlockVolumeID}, @@ -134,3 +140,186 @@ func TestNewCustomSerialLock(t *testing.T) { assert.Nil(t, err) }) } + +func TestGetLockWithName(t *testing.T) { + // Test case: Requesting a lock for a name that doesn't exist + i := &lockProvider{ + volNameLocks: map[string]gosync.TryLocker{}, + } + lock, err := i.GetLockWithName(context.Background(), "test") + if err != nil { + t.Errorf("Expected no error, got %v", err) + } + if lock == nil { + t.Error("Expected a lock, got nil") + } + + // Test case: Requesting a lock for a name that already exists + lock2, err := i.GetLockWithName(context.Background(), "test") + if err != nil { + t.Errorf("Expected no error, got %v", err) + } + if lock2 != lock { + t.Error("Expected the same lock, got a different lock") + } +} + +func TestGetLockWithID(t *testing.T) { + // Test case: Get lock for an ID that doesn't exist + i := &lockProvider{ + volIDLocks: map[string]gosync.TryLocker{}, + } + lock, err := i.GetLockWithID(context.Background(), "test") + if err != nil { + t.Errorf("Expected no error, got %v", err) + } + if lock == nil { + t.Error("Expected a lock, got nil") + } + + // Test case: Get lock for an ID that already exists + lock2, err := i.GetLockWithID(context.Background(), "test") + if err != nil { + t.Errorf("Expected no error, got %v", err) + } + if lock2 != lock { + t.Error("Expected the same lock, got a different lock") + } +} + +func TestCreateMetadataRetrieverClient(t *testing.T) { + // Create a new interceptor + i := &interceptor{ + opts: opts{ + MetadataSidecarClient: nil, + }, + } + + // Create a new context with the environment variable set + ctx := context.WithValue(context.Background(), csictx.RequestIDKey, "requestID") + ctx = csictx.WithEnviron(ctx, []string{fmt.Sprintf("%s=%s", identifiers.EnvMetadataRetrieverEndpoint, "endpoint")}) + + // Call the function + i.createMetadataRetrieverClient(ctx) + + // Check if the client was created + if i.opts.MetadataSidecarClient == nil { + t.Error("Expected MetadataSidecarClient to be set, but it was nil") + } +} + +// Define the options struct +type options struct { + locker lockprovider.VolumeLockerProvider + MetadataSidecarClient MetadataSidecarClient + timeout time.Duration +} + +// Define the Locker interface +type Locker interface { + GetLockWithID(ctx context.Context, id string) (gosync.TryLocker, error) +} + +// Define the MetadataSidecarClient interface +type MetadataSidecarClient interface { + GetPVCLabels(ctx context.Context, req *retriever.GetPVCLabelsRequest) (*retriever.GetPVCLabelsResponse, error) +} + +// Mock implementations for dependencies +type MockLocker struct { + mock.Mock +} + +func (m *MockLocker) GetLockWithID(ctx context.Context, id string) (gosync.TryLocker, error) { + args := m.Called(ctx, id) + return args.Get(0).(gosync.TryLocker), args.Error(1) +} + +func (m *MockLocker) GetLockWithName(ctx context.Context, name string) (gosync.TryLocker, error) { + args := m.Called(ctx, name) + return args.Get(0).(gosync.TryLocker), args.Error(1) +} + +type MockLock struct { + mock.Mock +} + +func (m *MockLock) TryLock(timeout time.Duration) bool { + args := m.Called(timeout) + return args.Bool(0) +} + +func (m *MockLock) Unlock() { + m.Called() +} + +func (m *MockLock) Close() error { + args := m.Called() + return args.Error(0) +} + +// Add the Lock method to satisfy the gosync.TryLocker interface +func (m *MockLock) Lock() { + m.Called() +} + +type MockMetadataSidecarClient struct { + mock.Mock +} + +func (m *MockMetadataSidecarClient) GetPVCLabels(ctx context.Context, req *retriever.GetPVCLabelsRequest) (*retriever.GetPVCLabelsResponse, error) { + args := m.Called(ctx, req) + return args.Get(0).(*retriever.GetPVCLabelsResponse), args.Error(1) +} + +func TestCreateVolume(t *testing.T) { + ctx := context.Background() + req := &csi.CreateVolumeRequest{ + Name: "test-volume", + Parameters: map[string]string{ + controller.KeyCSIPVCName: "test-pvc", + controller.KeyCSIPVCNamespace: "default", + }, + } + handler := func(_ context.Context, _ interface{}) (interface{}, error) { + return "success", nil + } + + mockLocker := new(MockLocker) + mockLock := new(MockLock) + mockMetadataClient := new(MockMetadataSidecarClient) + + interceptor := &interceptor{ + opts: opts{ + locker: mockLocker, + MetadataSidecarClient: mockMetadataClient, + timeout: 5 * time.Second, + }, + } + + t.Run("successful volume creation", func(t *testing.T) { + mockLocker.On("GetLockWithID", ctx, req.Name).Return(mockLock, nil) + mockLock.On("TryLock", interceptor.opts.timeout).Return(true) + mockLock.On("Unlock").Return() + mockLock.On("Close").Return(nil) + mockMetadataClient.On("GetPVCLabels", ctx, mock.Anything).Return(&retriever.GetPVCLabelsResponse{ + Parameters: map[string]string{"label1": "value1"}, + }, nil) + + res, err := interceptor.createVolume(ctx, req, nil, handler) + assert.NoError(t, err) + assert.Equal(t, "success", res) + }) + + t.Run("metadata retrieval failure", func(t *testing.T) { + mockLocker.On("GetLockWithID", ctx, req.Name).Return(mockLock, nil) + mockLock.On("TryLock", interceptor.opts.timeout).Return(true) + mockLock.On("Unlock").Return() + mockLock.On("Close").Return(nil) + mockMetadataClient.On("GetPVCLabels", ctx, mock.Anything).Return(nil, errors.New("metadata error")) + + res, err := interceptor.createVolume(ctx, req, nil, handler) + assert.NoError(t, err) + assert.Equal(t, "success", res) + }) +} diff --git a/pkg/monitor/event.go b/pkg/monitor/event.go new file mode 100644 index 00000000..bc3da3e7 --- /dev/null +++ b/pkg/monitor/event.go @@ -0,0 +1,274 @@ +/* + * + * Copyright © 2025 Dell Inc. or its subsidiaries. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package monitor + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/dell/csi-powerstore/v2/pkg/array" + "github.com/dell/csi-powerstore/v2/pkg/identifiers" + "github.com/dell/csi-powerstore/v2/pkg/identifiers/fs" + "github.com/dell/csi-powerstore/v2/pkg/identifiers/k8sutils" + "github.com/dell/csmlog" + "github.com/dell/gopowerstore" + + csictx "github.com/dell/gocsi/context" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + typedv1core "k8s.io/client-go/kubernetes/typed/core/v1" + + "k8s.io/client-go/tools/record" +) + +type IMonitorService interface { + // Reads the array secret from the filepath and populates array.Locker + // with array info and API clients. + UpdateArrays(arrayConfigFilepath string, fs fs.Interface) error + // Starts the service, polling for PowerStore Alerts and Events + // every pollPeriod. + Start(ctx context.Context, pollPeriod time.Duration) +} + +// Service represents the volume event monitoring service +type Service struct { + EventRecorder record.EventRecorderLogger + EventBroadcaster record.EventBroadcaster + + array.Locker + + kubeclient *k8sutils.K8sClient +} + +type EventContent struct { + // LastRecord is a reference to the most recent Kubernetes + // event for a specific resource + LatestRecord *corev1.Event +} + +// PersistentVolumeEvent relates a Persistent Volume struct to its +// most recent Kubernetes event. +type PersistentVolumeEvent struct { + EventContent + + Volume corev1.PersistentVolume +} + +// Instantiate csmlog on a package level +var log = csmlog.GetLogger() + +const ( + timeFormat = "2006-01-02T15:04:05Z" + VolumeResourceType = "volume" +) + +// NewMonitorService creates a new monitor service. +// The Kubernetes client is created using the environment var, identifiers.EnvKubeConfigPath, +// or in-cluster config, and used to create the EventRecorder and EventBroadcaster. +func NewMonitorService(ctx context.Context) (IMonitorService, error) { + kubeConfigPath, _ := csictx.LookupEnv(ctx, identifiers.EnvKubeConfigPath) + kubeclient, err := k8sutils.CreateKubeClientSet(kubeConfigPath) + if err != nil { + return nil, fmt.Errorf("failed to create Kubernetes API client for the monitor service: %s", err.Error()) + } + + eventRecorder, eventBroadcaster, err := newEventRecorder(kubeclient) + if err != nil { + return nil, err + } + + return &Service{ + EventRecorder: eventRecorder, + EventBroadcaster: eventBroadcaster, + + kubeclient: kubeclient, + }, nil +} + +func newEventRecorder(kubeclient *k8sutils.K8sClient) (record.EventRecorderLogger, record.EventBroadcaster, error) { + eventBroadcaster := record.NewBroadcaster() + + eventBroadcaster.StartRecordingToSink(&typedv1core.EventSinkImpl{Interface: kubeclient.Clientset.CoreV1().Events("")}) + + scheme := runtime.NewScheme() + err := corev1.AddToScheme(scheme) + if err != nil { + return nil, nil, err + } + + eventRecorder := eventBroadcaster.NewRecorder(scheme, corev1.EventSource{Component: identifiers.Name}) + + return eventRecorder, eventBroadcaster, nil +} + +// Start starts monitoring volumes and logging kubernetes events for alerts +// associated with Persistent Volumes backed by the PowerStore array(s). +func (s *Service) Start(ctx context.Context, pollPeriod time.Duration) { + log.Infof("[Monitor] starting event monitor with poll period %s", pollPeriod) + ticker := time.NewTicker(pollPeriod).C + + defer s.EventBroadcaster.Shutdown() + // assumes events have already been recorded for pre-existing volumes + // in order to avoid recording very old events. + lastCheck := time.Now() + for { + select { + case <-ctx.Done(): + log.Debugf("[Monitor] context expired for volume event monitor: %s", ctx.Err()) + return + case now := <-ticker: + s.monitorSince(lastCheck) + lastCheck = now + } + } +} + +// monitorSince queries all PowerStore arrays for alerts that +// have occurred between now and the lastTime the function was run, and processes +// any new alerts, creating kubernetes events, as needed. +func (s *Service) monitorSince(lastTime time.Time) { + log.Debugf("[Monitor] Getting alerts and events since %s", lastTime) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + for _, arr := range s.Locker.Arrays() { + alerts := []gopowerstore.Alert{} + + log.Debugf("[Monitor] Getting latest alerts for array %q", arr.GlobalID) + pageIndex := 0 + for { + // get alerts since the last time they were read, + // reading until there are no more pages of alerts available + alertsResp, err := arr.GetClient().GetAlerts(ctx, gopowerstore.GetAlertsOpts{ + Queries: map[string]string{ + "generated_timestamp": fmt.Sprintf("gte.%s", lastTime.Format(timeFormat)), + "order": "generated_timestamp.asc", + }, + RequestPagination: gopowerstore.RequestPagination{ + PageSize: 1000, + StartIndex: pageIndex, + }, + }) + if err != nil { + log.Errorf("[Monitor] failed to get alerts for array %q: %s", arr.GlobalID, err) + return + } + alerts = append(alerts, alertsResp.Alerts...) + if alertsResp.Pagination.Next == 0 { + break + } + pageIndex = alertsResp.Pagination.Next + } + + log.Debugf("[Monitor] got alerts: %v", alerts) + s.processVolumeObjectEvents(ctx, alerts) + } +} + +// processVolumeObjectEvents steps through the provided PowerStore alerts, +// looking for alerts associated with Persistent Volumes (PVs) in the cluster. +// If alerts are found for a given PV, an event is logged with the event recorder. +// Alerts passed in should be sorted in ascending order to ensure they are recorded +// in the order they occurred on the PowerStore array. +func (s *Service) processVolumeObjectEvents(ctx context.Context, alerts gopowerstore.Alerts) { + persistentVolumes := s.createVolumeMap(ctx) + + for _, alert := range alerts { + // currently only monitoring alerts for "volume" type + if alert.ResourceType != VolumeResourceType { + continue + } + + event, found := persistentVolumes[alert.ResourceName] + if !found { + // skip recording if the PowerStore alert does not belong to any of the Persistent Volumes + continue + } + + // Do not record the same event twice. + if event.LatestRecord != nil && event.LatestRecord.Message == alert.Description { + continue + } + + eventType := corev1.EventTypeWarning + if strings.EqualFold(alert.Severity, "info") { + eventType = corev1.EventTypeNormal + } + log.Infof("[Monitor] Alert is active for volume %q. recording event: %s", event.Volume.Name, alert.Description) + s.EventRecorder.Event(&event.Volume, eventType, alert.Severity, alert.Description) + } +} + +// createVolumeMap returns a map of Persistent Volume names to their PersistentVolumeEvents, +// where a PersistentVolumeEvent contains a reference to the PersistentVolume and the most +// recently recorded event for that volume (if one exists). +func (s *Service) createVolumeMap(ctx context.Context) map[string]PersistentVolumeEvent { + volumes, err := s.kubeclient.ListPersistentVolumes(ctx) + if err != nil { + log.Errorf("[Monitor] failed to get persistent volumes: %s", err.Error()) + return nil + } + + log.Debugf("[Monitor] got persistent volumes: %v", volumes.Items) + + // Create map to easily navigate through volumes. + volumesMap := make(map[string]PersistentVolumeEvent) + for _, volume := range volumes.Items { + latestObjectEvent := s.getLatestK8sEvent(ctx, volume.Name, volume.Namespace, "PersistentVolume") + log.Debugf("[Monitor] got latest event for volume %q: %v", volume.Name, latestObjectEvent) + volumesMap[volume.Name] = PersistentVolumeEvent{ + Volume: volume, + EventContent: EventContent{ + LatestRecord: latestObjectEvent, + }, + } + } + + return volumesMap +} + +// getLatestK8sEvent queries the cluster for all events related to the resource, "kind", of the given +// name, in the namespace provided, and returns the latest event. +func (s *Service) getLatestK8sEvent(ctx context.Context, name, namespace, kind string) *corev1.Event { + events, err := s.kubeclient.GetEvents(ctx, kind, name, namespace) + if err != nil { + log.Errorf("[Monitor] failed to get kubernetes events for %s %q: %s", kind, name, err.Error()) + return nil + } + log.Debugf("[Monitor] got kubernetes events for %s %q: %v", kind, name, events.Items) + + if len(events.Items) == 0 { + return nil + } + + // Retrieval of Events from a k8sObject is not guaranteed to be sorted. + latestEvent := events.Items[0] + for _, event := range events.Items { + if event.LastTimestamp.After(latestEvent.LastTimestamp.Time) { + latestEvent = event + } + } + + log.Debugf("[Monitor] latest k8s event for %s %q: description: %s", kind, name, latestEvent.Message) + + return &latestEvent +} diff --git a/pkg/monitor/event_test.go b/pkg/monitor/event_test.go new file mode 100644 index 00000000..803e3f79 --- /dev/null +++ b/pkg/monitor/event_test.go @@ -0,0 +1,778 @@ +/* + * + * Copyright © 2025 Dell Inc. or its subsidiaries. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package monitor + +import ( + "context" + "errors" + "fmt" + "os" + "reflect" + "strings" + "testing" + "time" + + "github.com/dell/csi-powerstore/v2/pkg/array" + "github.com/dell/csi-powerstore/v2/pkg/identifiers" + "github.com/dell/csi-powerstore/v2/pkg/identifiers/k8sutils" + csictx "github.com/dell/gocsi/context" + "github.com/dell/gopowerstore" + "github.com/dell/gopowerstore/api" + "github.com/dell/gopowerstore/mocks" + "github.com/stretchr/testify/mock" + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/record" +) + +const ( + testVolName string = "csi-test-vol" + testVolGroupName string = "test-vol-group" + testNamespace string = "test" + testMessageWarning string = "this is a test of the emergency alert system" + testMessageNormal string = "everything is normal" + eventMessageTypeWarning string = "Warning" + eventMessageTypeNormal string = "Normal" + alertSeverityInfo string = "Info" + alertSeverityMinor string = "Minor" + alertSeverityMajor string = "Major" + alertSeverityCritical string = "Critical" +) + +var ( + testTime time.Time = time.Now() + + testVolumeEventOldest *corev1.Event = &corev1.Event{ + TypeMeta: v1.TypeMeta{ + Kind: "PersistentVolume", + }, + ObjectMeta: v1.ObjectMeta{ + Name: "event1", + Namespace: testNamespace, + }, + InvolvedObject: corev1.ObjectReference{ + Name: testVolName, + Namespace: testNamespace, + }, + Type: corev1.EventTypeWarning, + LastTimestamp: v1.Time{ + Time: testTime.Add(-3 * time.Minute), + }, + Message: testMessageWarning, + } + testVolumeEventLatest *corev1.Event = &corev1.Event{ + TypeMeta: v1.TypeMeta{ + Kind: "PersistentVolume", + }, + ObjectMeta: v1.ObjectMeta{ + Name: "event3", + Namespace: testNamespace, + }, + InvolvedObject: corev1.ObjectReference{ + Name: testVolName, + Namespace: testNamespace, + }, + Type: corev1.EventTypeWarning, + LastTimestamp: v1.Time{ + Time: testTime.Add(-1 * time.Minute), + }, + Message: testMessageWarning, + } + testVolumeEventMiddle *corev1.Event = &corev1.Event{ + TypeMeta: v1.TypeMeta{ + Kind: "PersistentVolume", + }, + ObjectMeta: v1.ObjectMeta{ + Name: "event2", + Namespace: testNamespace, + }, + InvolvedObject: corev1.ObjectReference{ + Name: testVolName, + Namespace: testNamespace, + }, + Type: corev1.EventTypeWarning, + LastTimestamp: v1.Time{ + Time: testTime.Add(-2 * time.Minute), + }, + Message: testMessageWarning, + } + testVolumeEventNormal *corev1.Event = &corev1.Event{ + TypeMeta: v1.TypeMeta{ + Kind: "PersistentVolume", + }, + ObjectMeta: v1.ObjectMeta{ + Name: "event2", + Namespace: testNamespace, + }, + InvolvedObject: corev1.ObjectReference{ + Name: testVolName, + Namespace: testNamespace, + }, + Type: corev1.EventTypeNormal, + LastTimestamp: v1.Time{ + Time: testTime.Add(-2 * time.Minute), + }, + Message: testMessageNormal, + } + + testVolume *corev1.PersistentVolume = &corev1.PersistentVolume{ + ObjectMeta: v1.ObjectMeta{ + Name: testVolName, + Namespace: testNamespace, + }, + TypeMeta: v1.TypeMeta{ + Kind: "PersistentVolume", + }, + Status: corev1.PersistentVolumeStatus{ + Phase: corev1.VolumeBound, + }, + } + + testEvents []runtime.Object = []runtime.Object{testVolumeEventMiddle, testVolumeEventOldest, testVolumeEventLatest} +) + +func TestNewService(t *testing.T) { + tests := []struct { + name string // description of this test case + init func(context.Context, *testing.T) + cleanup func() + want *Service + wantErr bool + }{ + { + name: "fail to create kubeclient", + init: func(ctx context.Context, tt *testing.T) { + k8sutils.Kubeclient = nil + tt.Setenv(identifiers.EnvKubeConfigPath, "") + err := csictx.Setenv(ctx, identifiers.EnvKubeConfigPath, "") + if err != nil { + tt.Fatalf("failed to overwrite kubeconfig path: %v", err) + } + + tempNewForConfigFunc := k8sutils.NewForConfigFunc + k8sutils.NewForConfigFunc = func(_ *rest.Config) (kubernetes.Interface, error) { + return nil, errors.New("new for config error") + } + tt.Cleanup(func() { + k8sutils.NewForConfigFunc = tempNewForConfigFunc + }) + }, + want: nil, + wantErr: true, + }, + { + name: "success", + init: func(ctx context.Context, tt *testing.T) { + kubeconfigPath, err := createFakeKubeconfig(t) + if err != nil { + tt.Fatalf("failed to create fake kubeconfig for test: %s", err.Error()) + return + } + if err := csictx.Setenv(ctx, identifiers.EnvKubeConfigPath, kubeconfigPath); err != nil { + tt.Fatalf("failed to add kubeconfig variable to context: %s", err.Error()) + return + } + }, + want: &Service{}, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + tt.init(ctx, t) + got, gotErr := NewMonitorService(ctx) + if gotErr != nil { + if !tt.wantErr { + t.Errorf("NewService() failed: %v", gotErr) + } + return + } + if tt.wantErr { + t.Fatal("NewService() succeeded unexpectedly") + } + + if tt.want == nil && got != nil { + t.Errorf("NewService() = %v, want %v", got, tt.want) + } + }) + } +} + +func createFakeKubeconfig(t *testing.T) (kubeconfigPath string, err error) { + fakeConfig := ` +apiVersion: v1 +clusters: +- cluster: + server: https://localhost:8080 + name: foo-cluster +contexts: +- context: + cluster: foo-cluster + user: foo-user + namespace: bar + name: foo-context +current-context: foo-context +kind: Config +users: +- name: foo-user +` + tmpKubeconfigDir := t.TempDir() + tmpfile, err := os.CreateTemp(tmpKubeconfigDir, "kubeconfig") + if err != nil { + return "", fmt.Errorf("failed to create temp kubeconfig file: %s", err.Error()) + } + if err := os.WriteFile(tmpfile.Name(), []byte(fakeConfig), 0o600); err != nil { + return "", fmt.Errorf("failed to write config to the kubeconfig file: %s", err.Error()) + } + return tmpfile.Name(), nil +} + +func TestService_Start(t *testing.T) { + defaultArray := &array.PowerStoreArray{ + Endpoint: "my-powerstore.com/api/rest", + GlobalID: "gid1", + Username: "user", + Password: "password", + BlockProtocol: identifiers.ISCSITransport, + Insecure: true, + IsDefault: true, + } + type params struct { + pollPeriod time.Duration + ctxTimeout time.Duration + } + type fields struct { + service *Service + } + tests := []struct { + name string + fields fields + params params + getArrays func() map[string]*array.PowerStoreArray + }{ + { + name: "context timeout", + fields: fields{ + service: &Service{ + kubeclient: &k8sutils.K8sClient{ + Clientset: fake.NewClientset(), + }, + EventRecorder: record.NewFakeRecorder(0), + EventBroadcaster: record.NewBroadcasterForTests(0 * time.Second), + }, + }, + params: params{ + pollPeriod: 10 * time.Second, + // timeout being less than polling period will ensure + // the context is canceled before the initial poll. + ctxTimeout: 100 * time.Millisecond, + }, + getArrays: func() map[string]*array.PowerStoreArray { + return map[string]*array.PowerStoreArray{ + defaultArray.GlobalID: defaultArray, + } + }, + }, + { + name: "a single monitor loop execution", + fields: fields{ + service: &Service{ + kubeclient: &k8sutils.K8sClient{ + Clientset: fake.NewClientset(), + }, + EventRecorder: record.NewFakeRecorder(0), + EventBroadcaster: record.NewBroadcasterForTests(0 * time.Second), + }, + }, + params: params{ + pollPeriod: 10 * time.Millisecond, + // give enough time to run the request + // but ensure the context is canceled after the first run + ctxTimeout: 11 * time.Millisecond, + }, + getArrays: func() map[string]*array.PowerStoreArray { + client := mocks.NewClient(t) + client.On("GetAlerts", mock.Anything, mock.Anything).Return(&gopowerstore.GetAlertsResponse{}, nil) + + defaultArray.Client = client + return map[string]*array.PowerStoreArray{ + defaultArray.GlobalID: defaultArray, + } + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(_ *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), tt.params.ctxTimeout) + defer cancel() + + s := tt.fields.service + s.SetArrays(tt.getArrays()) + s.SetDefaultArray(defaultArray) + + // it may appear nothing is being tested here, but + // the tests will pass or fail based on whether the mocked + // functions are called. + s.Start(ctx, tt.params.pollPeriod) + }) + } +} + +func TestService_monitorSince(t *testing.T) { + defaultArray := &array.PowerStoreArray{ + Endpoint: "primary.my-powerstore.com/api/rest", + GlobalID: "gid1", + Username: "user", + Password: "password", + BlockProtocol: identifiers.ISCSITransport, + Insecure: true, + IsDefault: true, + } + secondaryArray := &array.PowerStoreArray{ + Endpoint: "secondary.my-powerstore.com/api/rest", + GlobalID: "gid2", + Username: "user", + Password: "password", + BlockProtocol: identifiers.ISCSITransport, + Insecure: true, + IsDefault: false, + } + tests := []struct { + name string + getArrays func() map[string]*array.PowerStoreArray + getClient func(time.Time) gopowerstore.Client + }{ + { + name: "query array without issues", + getArrays: func() map[string]*array.PowerStoreArray { + return map[string]*array.PowerStoreArray{ + defaultArray.GlobalID: defaultArray, + } + }, + getClient: func(timeStamp time.Time) gopowerstore.Client { + client := mocks.NewClient(t) + + client.On("GetAlerts", mock.Anything, gopowerstore.GetAlertsOpts{ + RequestPagination: gopowerstore.RequestPagination{ + PageSize: 1000, + StartIndex: 0, + }, + Queries: map[string]string{ + "generated_timestamp": fmt.Sprintf("gte.%s", timeStamp.Format(timeFormat)), + "order": "generated_timestamp.asc", + }, + }).Return(&gopowerstore.GetAlertsResponse{}, nil) + + return client + }, + }, + { + name: "query all arrays", + getArrays: func() map[string]*array.PowerStoreArray { + return map[string]*array.PowerStoreArray{ + defaultArray.GlobalID: defaultArray, + secondaryArray.GlobalID: secondaryArray, + } + }, + getClient: func(timestamp time.Time) gopowerstore.Client { + client := mocks.NewClient(t) + client.On("GetAlerts", mock.Anything, gopowerstore.GetAlertsOpts{ + RequestPagination: gopowerstore.RequestPagination{ + PageSize: 1000, + StartIndex: 0, + }, + Queries: map[string]string{ + "generated_timestamp": fmt.Sprintf("gte.%s", timestamp.Format(timeFormat)), + "order": "generated_timestamp.asc", + }, + }).Return(&gopowerstore.GetAlertsResponse{}, nil) + + return client + }, + }, + { + name: "query paginated data", + getArrays: func() map[string]*array.PowerStoreArray { + return map[string]*array.PowerStoreArray{ + defaultArray.GlobalID: defaultArray, + secondaryArray.GlobalID: secondaryArray, + } + }, + getClient: func(timestamp time.Time) gopowerstore.Client { + client := mocks.NewClient(t) + client.On("GetAlerts", mock.Anything, gopowerstore.GetAlertsOpts{ + RequestPagination: gopowerstore.RequestPagination{ + PageSize: 1000, + StartIndex: 0, + }, + Queries: map[string]string{ + "generated_timestamp": fmt.Sprintf("gte.%s", timestamp.Format(timeFormat)), + "order": "generated_timestamp.asc", + }, + }).Return(&gopowerstore.GetAlertsResponse{ + AlertsResponseMeta: gopowerstore.AlertsResponseMeta{ + RespMeta: api.RespMeta{ + Pagination: api.PaginationInfo{ + First: 0, + Last: 999, + Next: 1000, + Total: 2000, + IsPaginate: true, + }, + }, + }, + Alerts: gopowerstore.Alerts{}, + }, nil) + client.On("GetAlerts", mock.Anything, gopowerstore.GetAlertsOpts{ + RequestPagination: gopowerstore.RequestPagination{ + PageSize: 1000, + StartIndex: 1000, + }, + Queries: map[string]string{ + "generated_timestamp": fmt.Sprintf("gte.%s", timestamp.Format(timeFormat)), + "order": "generated_timestamp.asc", + }, + }).Return(&gopowerstore.GetAlertsResponse{ + AlertsResponseMeta: gopowerstore.AlertsResponseMeta{ + RespMeta: api.RespMeta{ + Pagination: api.PaginationInfo{ + First: 1000, + Last: 1999, + Next: 0, + Total: 2000, + IsPaginate: true, + }, + }, + }, + Alerts: gopowerstore.Alerts{}, + }, nil) + + return client + }, + }, + { + name: "error when querying for alerts", + getArrays: func() map[string]*array.PowerStoreArray { + return map[string]*array.PowerStoreArray{ + defaultArray.GlobalID: defaultArray, + } + }, + getClient: func(timeStamp time.Time) gopowerstore.Client { + client := mocks.NewClient(t) + client.On("GetAlerts", mock.Anything, gopowerstore.GetAlertsOpts{ + RequestPagination: gopowerstore.RequestPagination{ + PageSize: 1000, + StartIndex: 0, + }, + Queries: map[string]string{ + "generated_timestamp": fmt.Sprintf("gte.%s", timeStamp.Format(timeFormat)), + "order": "generated_timestamp.asc", + }, + }).Return(nil, errors.New("foo error")) + + return client + }, + }, + { + name: "error when querying for events", + getArrays: func() map[string]*array.PowerStoreArray { + return map[string]*array.PowerStoreArray{ + defaultArray.GlobalID: defaultArray, + } + }, + getClient: func(timeStamp time.Time) gopowerstore.Client { + client := mocks.NewClient(t) + + client.On("GetAlerts", mock.Anything, gopowerstore.GetAlertsOpts{ + RequestPagination: gopowerstore.RequestPagination{ + PageSize: 1000, + StartIndex: 0, + }, + Queries: map[string]string{ + "generated_timestamp": fmt.Sprintf("gte.%s", timeStamp.Format(timeFormat)), + "order": "generated_timestamp.asc", + }, + }).Return(&gopowerstore.GetAlertsResponse{}, nil) + + return client + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(_ *testing.T) { + s := &Service{ + kubeclient: &k8sutils.K8sClient{ + Clientset: fake.NewClientset(), + }, + EventRecorder: record.NewFakeRecorder(0), + EventBroadcaster: record.NewBroadcasterForTests(0 * time.Second), + } + lastTime := time.Now().Add(1 * time.Minute) + + arrays := tt.getArrays() + for _, arr := range arrays { + arr.Client = tt.getClient(lastTime) + } + s.SetArrays(arrays) + s.SetDefaultArray(defaultArray) + + // it may appear nothing is being tested here, but + // the test will confirm the expected mock functions have + // been called. + s.monitorSince(lastTime) + }) + } +} + +func TestService_processVolumeObjectEvents(t *testing.T) { + tests := []struct { + name string + alerts gopowerstore.Alerts + clusterResources []runtime.Object + want string + }{ + { + name: "record a new event from an alert", + alerts: gopowerstore.Alerts{ + { + ResourceType: VolumeResourceType, + ResourceName: testVolume.Name, + Description: testMessageWarning, + Severity: alertSeverityMinor, + }, + }, + clusterResources: []runtime.Object{testVolume}, + want: strings.Join([]string{eventMessageTypeWarning, alertSeverityMinor, testMessageWarning}, " "), + }, + { + name: "alert resource type is not a volume", + alerts: gopowerstore.Alerts{ + { + ResourceType: "volume_group", + Description: testMessageWarning, + }, + }, + clusterResources: []runtime.Object{testVolume}, + want: "", + }, + { + name: "volume is not being monitored by the driver", + alerts: gopowerstore.Alerts{ + { + ResourceName: "csivol-foo", // should not be known to the fake kubeclient + ResourceType: VolumeResourceType, + Description: testMessageWarning, + }, + }, + clusterResources: []runtime.Object{testVolume}, + want: "", + }, + { + name: "volume alert has already been submitted to event recorder", + alerts: gopowerstore.Alerts{ + { + ResourceName: testVolume.Name, + ResourceType: VolumeResourceType, + Description: testMessageWarning, + }, + }, + clusterResources: []runtime.Object{testVolume, testVolumeEventLatest}, + want: "", + }, + { + name: "volume has a new alert", + alerts: gopowerstore.Alerts{ + { + ResourceName: testVolume.Name, + ResourceType: VolumeResourceType, + Description: "this is the second test", + Severity: alertSeverityInfo, + }, + }, + clusterResources: []runtime.Object{testVolume, testVolumeEventLatest}, + want: strings.Join([]string{eventMessageTypeNormal, alertSeverityInfo, "this is the second test"}, " "), + }, + { + name: "no event updates from the array", + alerts: gopowerstore.Alerts{}, + clusterResources: []runtime.Object{testVolume, testVolumeEventLatest}, + want: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fakeRecorder := record.NewFakeRecorder(10) + s := &Service{ + kubeclient: &k8sutils.K8sClient{ + Clientset: fake.NewClientset(tt.clusterResources...), + }, + EventBroadcaster: record.NewBroadcaster(), + EventRecorder: fakeRecorder, + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + go func() { + defer cancel() + s.processVolumeObjectEvents(ctx, tt.alerts) + }() + + for { + select { + case <-ctx.Done(): + return + case event := <-fakeRecorder.Events: + if tt.want != event { + t.Errorf("processVolumeObjectEvents() = %v, want: %v", event, tt.want) + } + } + } + }) + } +} + +func TestService_CreateVolumeMap(t *testing.T) { + type fields struct { + s *Service + } + tests := []struct { + name string + fields fields + want map[string]PersistentVolumeEvent + }{ + { + name: "fail to list volumes", + fields: fields{ + s: &Service{ + // client is nil and will error + kubeclient: &k8sutils.K8sClient{}, + }, + }, + want: nil, + }, + { + name: "get volume map", + fields: fields{ + s: &Service{ + kubeclient: &k8sutils.K8sClient{ + Clientset: fake.NewClientset(append(testEvents, testVolume)...), + }, + }, + }, + want: map[string]PersistentVolumeEvent{ + testVolName: { + Volume: *testVolume, + EventContent: EventContent{ + LatestRecord: testVolumeEventLatest, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := tt.fields.s + + got := s.createVolumeMap(context.Background()) + + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("CreateVolumeMap() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestService_GetLastK8sEvents(t *testing.T) { + type params struct { + name string + namespace string + kind string + } + type fields struct { + s *Service + } + tests := []struct { + name string + fields fields + params params + want *corev1.Event + }{ + { + name: "no kubeclient", + fields: fields{ + s: &Service{ + kubeclient: &k8sutils.K8sClient{}, + }, + }, + params: params{ + name: "", + namespace: "", + kind: "", + }, + want: nil, + }, + { + name: "no events", + fields: fields{ + s: &Service{ + kubeclient: &k8sutils.K8sClient{ + // init client without any resources + Clientset: fake.NewClientset(), + }, + }, + }, + params: params{ + name: "", + namespace: "", + kind: "", + }, + want: nil, + }, + { + name: "get the most recent event", + fields: fields{ + s: &Service{ + kubeclient: &k8sutils.K8sClient{ + Clientset: fake.NewClientset(testEvents...), + }, + }, + }, + params: params{ + name: testVolName, + namespace: "test", + kind: "PersistentVolume", + }, + want: testVolumeEventLatest, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := tt.fields.s + got := s.getLatestK8sEvent(context.Background(), tt.params.name, tt.params.namespace, tt.params.kind) + + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("GetLastK8sEvents() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/node/acl.go b/pkg/node/acl.go index 34703bd6..f647c59b 100644 --- a/pkg/node/acl.go +++ b/pkg/node/acl.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2022 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2022-2025 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,7 +26,6 @@ import ( "strings" "github.com/dell/gopowerstore" - log "github.com/sirupsen/logrus" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -41,6 +40,7 @@ type NFSv4ACLsInterface interface { type NFSv4ACLs struct{} func validateAndSetACLs(ctx context.Context, s NFSv4ACLsInterface, nasName string, client gopowerstore.Client, acls string, dir string) (bool, error) { + log := log.WithContext(ctx) aclsConfigured := false if nfsv4ACLs(acls) { if isNfsv4Enabled(ctx, client, nasName) { @@ -89,10 +89,11 @@ func (n *NFSv4ACLs) SetNfsv4Acls(acls string, dir string) error { } func isNfsv4Enabled(ctx context.Context, client gopowerstore.Client, nasName string) bool { + log := log.WithContext(ctx) nfsv4Enabled := false nas, err := gopowerstore.Client.GetNASByName(client, ctx, nasName) if err == nil { - nfsServer, err := gopowerstore.Client.GetNfsServer(client, ctx, nas.NfsServers[0].Id) + nfsServer, err := gopowerstore.Client.GetNfsServer(client, ctx, nas.NfsServers[0].ID) if err == nil { if nfsServer.IsNFSv4Enabled { nfsv4Enabled = true @@ -100,7 +101,7 @@ func isNfsv4Enabled(ctx context.Context, client gopowerstore.Client, nasName str log.Error(fmt.Sprintf("NFS v4 not enabled on NAS server: %s\n", nasName)) } } else { - log.Error(fmt.Sprintf("can't fetch nfs server with id %s: %s", nas.NfsServers[0].Id, err.Error())) + log.Error(fmt.Sprintf("can't fetch nfs server with id %s: %s", nas.NfsServers[0].ID, err.Error())) } } else { log.Error(fmt.Sprintf("can't determine nfsv4 enabled: %s", err.Error())) diff --git a/pkg/node/acl_test.go b/pkg/node/acl_test.go index f2cedd9e..0ae72afa 100644 --- a/pkg/node/acl_test.go +++ b/pkg/node/acl_test.go @@ -19,7 +19,6 @@ package node import ( "context" "errors" - "fmt" "testing" "github.com/dell/csi-powerstore/v2/mocks" @@ -34,7 +33,7 @@ func TestPosixMode_Success(t *testing.T) { isPosixMode := posixMode("0755") expected := true if isPosixMode != expected { - t.Errorf(fmt.Sprintf("expected: %v, actual: %v", expected, isPosixMode)) + t.Errorf("expected: %v, actual: %v", expected, isPosixMode) } } @@ -42,7 +41,7 @@ func TestPosixMode_Fail(t *testing.T) { isPosixMode := posixMode("abcd") expected := false if isPosixMode != expected { - t.Errorf(fmt.Sprintf("expected: %v, actual: %v", expected, isPosixMode)) + t.Errorf("expected: %v, actual: %v", expected, isPosixMode) } } @@ -50,7 +49,7 @@ func TestNfsv4Acl_Success(t *testing.T) { isNfsv4ACLs := nfsv4ACLs("A::OWNER@:RWX") expected := true if isNfsv4ACLs != expected { - t.Errorf(fmt.Sprintf("expected: %v, actual: %v", expected, isNfsv4ACLs)) + t.Errorf("expected: %v, actual: %v", expected, isNfsv4ACLs) } } @@ -58,7 +57,7 @@ func TestNfsv4Acl_Fail(t *testing.T) { isNfsv4ACLs := nfsv4ACLs("abcd") expected := false if isNfsv4ACLs != expected { - t.Errorf(fmt.Sprintf("expected: %v, actual: %v", expected, isNfsv4ACLs)) + t.Errorf("expected: %v, actual: %v", expected, isNfsv4ACLs) } } @@ -67,18 +66,18 @@ func TestNfsv4NasServer_Success(t *testing.T) { nfsServers := []gopowerstore.NFSServerInstance{ { - Id: validNfsServerID, + ID: validNfsServerID, IsNFSv4Enabled: true, }, } clientMock.On("GetNASByName", mock.Anything, validNasName).Return(gopowerstore.NAS{ID: validNasID, NfsServers: nfsServers}, nil) - clientMock.On("GetNfsServer", mock.Anything, mock.Anything).Return(gopowerstore.NFSServerInstance{Id: validNfsServerID, IsNFSv4Enabled: true}, nil) + clientMock.On("GetNfsServer", mock.Anything, mock.Anything).Return(gopowerstore.NFSServerInstance{ID: validNfsServerID, IsNFSv4Enabled: true}, nil) isNFSv4Enabled := isNfsv4Enabled(context.Background(), clientMock, validNasName) expected := true if isNFSv4Enabled != expected { - t.Errorf(fmt.Sprintf("expected: %v, actual: %v", expected, isNFSv4Enabled)) + t.Errorf("expected: %v, actual: %v", expected, isNFSv4Enabled) } } @@ -86,12 +85,12 @@ func TestNfsv4NasServer_Err_GetNASByName(t *testing.T) { clientMock = new(gopowerstoremock.Client) clientMock.On("GetNASByName", mock.Anything, validNasName).Return(gopowerstore.NAS{ID: validNasID}, errors.New("GetNASByName_fail")) - clientMock.On("GetNfsServer", mock.Anything, mock.Anything).Return(gopowerstore.NFSServerInstance{Id: validNfsServerID, IsNFSv4Enabled: true}, nil) + clientMock.On("GetNfsServer", mock.Anything, mock.Anything).Return(gopowerstore.NFSServerInstance{ID: validNfsServerID, IsNFSv4Enabled: true}, nil) isNFSv4Enabled := isNfsv4Enabled(context.Background(), clientMock, validNasName) expected := false if isNFSv4Enabled != expected { - t.Errorf(fmt.Sprintf("expected: %v, actual: %v", expected, isNFSv4Enabled)) + t.Errorf("expected: %v, actual: %v", expected, isNFSv4Enabled) } } @@ -100,18 +99,18 @@ func TestNfsv4NasServer_Err_GetNfsServer(t *testing.T) { nfsServers := []gopowerstore.NFSServerInstance{ { - Id: validNfsServerID, + ID: validNfsServerID, IsNFSv4Enabled: true, }, } clientMock.On("GetNASByName", mock.Anything, validNasName).Return(gopowerstore.NAS{ID: validNasID, NfsServers: nfsServers}, nil) - clientMock.On("GetNfsServer", mock.Anything, mock.Anything).Return(gopowerstore.NFSServerInstance{Id: validNfsServerID, IsNFSv4Enabled: true}, errors.New("GetNfsServer_fail")) + clientMock.On("GetNfsServer", mock.Anything, mock.Anything).Return(gopowerstore.NFSServerInstance{ID: validNfsServerID, IsNFSv4Enabled: true}, errors.New("GetNfsServer_fail")) isNFSv4Enabled := isNfsv4Enabled(context.Background(), clientMock, validNasName) expected := false if isNFSv4Enabled != expected { - t.Errorf(fmt.Sprintf("expected: %v, actual: %v", expected, isNFSv4Enabled)) + t.Errorf("expected: %v, actual: %v", expected, isNFSv4Enabled) } } @@ -120,18 +119,18 @@ func TestNfsv4NasServer_Fail(t *testing.T) { nfsServers := []gopowerstore.NFSServerInstance{ { - Id: validNfsServerID, + ID: validNfsServerID, IsNFSv4Enabled: true, }, } clientMock.On("GetNASByName", mock.Anything, validNasName).Return(gopowerstore.NAS{ID: validNasID, NfsServers: nfsServers}, nil) - clientMock.On("GetNfsServer", mock.Anything, mock.Anything).Return(gopowerstore.NFSServerInstance{Id: validNfsServerID, IsNFSv4Enabled: false}, nil) + clientMock.On("GetNfsServer", mock.Anything, mock.Anything).Return(gopowerstore.NFSServerInstance{ID: validNfsServerID, IsNFSv4Enabled: false}, nil) isNFSv4Enabled := isNfsv4Enabled(context.Background(), clientMock, validNasName) expected := false if isNFSv4Enabled != expected { - t.Errorf(fmt.Sprintf("expected: %v, actual: %v", expected, isNFSv4Enabled)) + t.Errorf("expected: %v, actual: %v", expected, isNFSv4Enabled) } } @@ -141,19 +140,19 @@ func TestValidateAndSetNfsACLs_Success_nfsv4Acls(t *testing.T) { nfsServers := []gopowerstore.NFSServerInstance{ { - Id: validNfsServerID, + ID: validNfsServerID, IsNFSv4Enabled: true, }, } nfsv4ACLsMock.On("SetNfsv4Acls", mock.Anything, mock.Anything).Return(nil) clientMock.On("GetNASByName", mock.Anything, validNasName).Return(gopowerstore.NAS{ID: validNasID, NfsServers: nfsServers}, nil) - clientMock.On("GetNfsServer", mock.Anything, mock.Anything).Return(gopowerstore.NFSServerInstance{Id: validNfsServerID, IsNFSv4Enabled: true}, nil) + clientMock.On("GetNfsServer", mock.Anything, mock.Anything).Return(gopowerstore.NFSServerInstance{ID: validNfsServerID, IsNFSv4Enabled: true}, nil) aclConfigured, err := validateAndSetACLs(context.Background(), nfsv4ACLsMock, validNasName, clientMock, "A::OWNER@:RWX", "dir2") if err != nil || aclConfigured == false { - t.Errorf(fmt.Sprintf("expected: true, actual: %v err: %s", aclConfigured, err.Error())) + t.Errorf("expected: true, actual: %v err: %s", aclConfigured, err.Error()) } } @@ -163,19 +162,19 @@ func TestValidateAndSetNfsACLs_Fail_InvalidAcls(t *testing.T) { nfsServers := []gopowerstore.NFSServerInstance{ { - Id: validNfsServerID, + ID: validNfsServerID, IsNFSv4Enabled: true, }, } clientMock.On("GetNASByName", mock.Anything, validNasName).Return(gopowerstore.NAS{ID: validNasID, NfsServers: nfsServers}, nil) - clientMock.On("GetNfsServer", mock.Anything, mock.Anything).Return(gopowerstore.NFSServerInstance{Id: validNfsServerID, IsNFSv4Enabled: true}, nil) + clientMock.On("GetNfsServer", mock.Anything, mock.Anything).Return(gopowerstore.NFSServerInstance{ID: validNfsServerID, IsNFSv4Enabled: true}, nil) nfsv4ACLsMock.On("SetNfsv4Acls", mock.Anything, mock.Anything).Return(nil) aclConfigured, err := validateAndSetACLs(context.Background(), nfsv4ACLsMock, validNasName, clientMock, "abcd", "dir1") if err == nil || aclConfigured != false { - t.Errorf(fmt.Sprintf("expected: false, actual: %v err: %s", aclConfigured, err.Error())) + t.Errorf("expected: false, actual: %v err: %s", aclConfigured, err.Error()) } } @@ -185,18 +184,18 @@ func TestValidateAndSetNfsACLs_Fail_GetNfsServerFail(t *testing.T) { nfsServers := []gopowerstore.NFSServerInstance{ { - Id: validNfsServerID, + ID: validNfsServerID, IsNFSv4Enabled: true, }, } clientMock.On("GetNASByName", mock.Anything, validNasName).Return(gopowerstore.NAS{ID: validNasID, NfsServers: nfsServers}, nil) - clientMock.On("GetNfsServer", mock.Anything, mock.Anything).Return(gopowerstore.NFSServerInstance{Id: validNfsServerID, IsNFSv4Enabled: true}, errors.New("GetNfsServer_fail")) + clientMock.On("GetNfsServer", mock.Anything, mock.Anything).Return(gopowerstore.NFSServerInstance{ID: validNfsServerID, IsNFSv4Enabled: true}, errors.New("GetNfsServer_fail")) nfsv4ACLsMock.On("SetNfsv4Acls", mock.Anything, mock.Anything).Return(nil) aclConfigured, err := validateAndSetACLs(context.Background(), nfsv4ACLsMock, validNasName, clientMock, "A::OWNER@:RWX", "dir1") if err == nil || aclConfigured != false { - t.Errorf(fmt.Sprintf("expected: false, actual: %v err: %s", aclConfigured, err.Error())) + t.Errorf("expected: false, actual: %v err: %s", aclConfigured, err.Error()) } } diff --git a/pkg/node/base.go b/pkg/node/base.go index a171185f..c4ebe4c1 100644 --- a/pkg/node/base.go +++ b/pkg/node/base.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2021-2023 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2021-2025 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,13 +29,13 @@ import ( "strconv" "strings" - "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/dell/csi-powerstore/v2/pkg/common" - "github.com/dell/csi-powerstore/v2/pkg/common/fs" + "github.com/dell/csi-powerstore/v2/pkg/identifiers" + "github.com/dell/csi-powerstore/v2/pkg/identifiers/fs" + "github.com/dell/csmlog" "github.com/dell/gobrick" csictx "github.com/dell/gocsi/context" "github.com/dell/gofsutil" - log "github.com/sirupsen/logrus" + "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -44,6 +44,7 @@ const ( powerStoreMaxNodeNameLength = 64 blockVolumePathMarker = "/csi/volumeDevices/publish/" sysBlock = "/sys/block/" + dev = "/dev/" defaultNodeNamePrefix = "csi-node" defaultNodeChrootPath = "/noderoot" @@ -76,6 +77,7 @@ type NVMEConnector interface { type FcConnector interface { ConnectVolume(ctx context.Context, info gobrick.FCVolumeInfo) (gobrick.Device, error) DisconnectVolumeByDeviceName(ctx context.Context, name string) error + DisconnectVolumeByWWN(ctx context.Context, wwn string) error GetInitiatorPorts(ctx context.Context) ([]string, error) } @@ -83,15 +85,15 @@ func getNodeOptions() Opts { var opts Opts ctx := context.Background() - if path, ok := csictx.LookupEnv(ctx, common.EnvNodeIDFilePath); ok { + if path, ok := csictx.LookupEnv(ctx, identifiers.EnvNodeIDFilePath); ok { opts.NodeIDFilePath = path } - if kubeConfigPath, ok := csictx.LookupEnv(ctx, common.EnvKubeConfigPath); ok { + if kubeConfigPath, ok := csictx.LookupEnv(ctx, identifiers.EnvKubeConfigPath); ok { opts.KubeConfigPath = kubeConfigPath } - if prefix, ok := csictx.LookupEnv(ctx, common.EnvNodeNamePrefix); ok { + if prefix, ok := csictx.LookupEnv(ctx, identifiers.EnvNodeNamePrefix); ok { opts.NodeNamePrefix = prefix } @@ -99,11 +101,11 @@ func getNodeOptions() Opts { opts.NodeNamePrefix = defaultNodeNamePrefix } - if kubeNodeName, ok := csictx.LookupEnv(ctx, common.EnvKubeNodeName); ok { + if kubeNodeName, ok := csictx.LookupEnv(ctx, identifiers.EnvKubeNodeName); ok { opts.KubeNodeName = kubeNodeName } - if nodeChrootPath, ok := csictx.LookupEnv(ctx, common.EnvNodeChrootPath); ok { + if nodeChrootPath, ok := csictx.LookupEnv(ctx, identifiers.EnvNodeChrootPath); ok { opts.NodeChrootPath = nodeChrootPath } @@ -111,7 +113,7 @@ func getNodeOptions() Opts { opts.NodeChrootPath = defaultNodeChrootPath } - if maxVolumesPerNodeStr, ok := csictx.LookupEnv(ctx, common.EnvMaxVolumesPerNode); ok { + if maxVolumesPerNodeStr, ok := csictx.LookupEnv(ctx, identifiers.EnvMaxVolumesPerNode); ok { maxVolumesPerNode, err := strconv.ParseInt(maxVolumesPerNodeStr, 10, 64) if err != nil { log.Warn("error while parsing the value of maxPowerstoreVolumesPerNode, using default value 0") @@ -121,7 +123,7 @@ func getNodeOptions() Opts { } } - if tmpDir, ok := csictx.LookupEnv(ctx, common.EnvTmpDir); ok { + if tmpDir, ok := csictx.LookupEnv(ctx, identifiers.EnvTmpDir); ok { opts.TmpDir = tmpDir } @@ -129,7 +131,7 @@ func getNodeOptions() Opts { opts.TmpDir = defaultTmpDir } - if fcPortsFilterFilePath, ok := csictx.LookupEnv(ctx, common.EnvFCPortsFilterFilePath); ok { + if fcPortsFilterFilePath, ok := csictx.LookupEnv(ctx, identifiers.EnvFCPortsFilterFilePath); ok { opts.FCPortsFilterFilePath = fcPortsFilterFilePath } @@ -139,7 +141,7 @@ func getNodeOptions() Opts { if v, ok := csictx.LookupEnv(ctx, n); ok { b, err := strconv.ParseBool(v) if err != nil { - log.WithField(n, v).Debug("invalid boolean value. defaulting to false") + log.WithFields(csmlog.Fields{n: v}).Debug("invalid boolean value. defaulting to false") return false } return b @@ -147,11 +149,11 @@ func getNodeOptions() Opts { return false } - opts.EnableCHAP = pb(common.EnvEnableCHAP) + opts.EnableCHAP = pb(identifiers.EnvEnableCHAP) if opts.EnableCHAP { opts.CHAPUsername = "admin" - opts.CHAPPassword = common.RandomString(12) + opts.CHAPPassword = identifiers.RandomString(12) } return opts @@ -175,8 +177,13 @@ func formatWWPN(data string) (string, error) { } // Get preferred outbound ip of this machine -func getOutboundIP(endpoint string, fs fs.Interface) (net.IP, error) { - conn, err := fs.NetDial(endpoint) +func getOutboundIP(endpoint string, port string, fs fs.Interface) (net.IP, error) { + finalEndpoint := endpoint + if port != "" { + // this means the port is set in the URL and should be used (In case of Auth v2 enablement) + finalEndpoint = endpoint + ":" + port + } + conn, err := fs.NetDial(finalEndpoint) if err != nil { return nil, err } @@ -204,18 +211,18 @@ func getStagedDev(ctx context.Context, stagePath string, fs fs.Interface) (strin return sourceDev, nil } -func getStagingPath(ctx context.Context, sp string, volID string) string { - logFields := common.GetLogFields(ctx) +func getStagingPath(ctx context.Context, sp string, volID string) (string, string) { + log := log.WithContext(ctx) if sp == "" || volID == "" { - return "" + return volID, sp } stagingPath := path.Join(sp, volID) - log.WithFields(logFields).Infof("staging path is: %s", stagingPath) - return path.Join(sp, volID) + log.Infof("staging path is: %s", stagingPath) + return volID, path.Join(sp, volID) } func getRemnantTargetMounts(ctx context.Context, target string, fs fs.Interface) ([]gofsutil.Info, bool, error) { - logFields := common.GetLogFields(ctx) + log := log.WithContext(ctx) var targetMounts []gofsutil.Info var found bool mounts, err := getMounts(ctx, fs) @@ -226,7 +233,7 @@ func getRemnantTargetMounts(ctx context.Context, target string, fs fs.Interface) for _, mount := range mounts { if strings.Contains(mount.Path, target) { targetMounts = append(targetMounts, mount) - log.WithFields(logFields).Infof("matching remnantTargetMount %s target %s", target, mount.Path) + log.Infof("matching remnantTargetMount %s target %s", target, mount.Path) found = true } } @@ -234,7 +241,7 @@ func getRemnantTargetMounts(ctx context.Context, target string, fs fs.Interface) } func getTargetMount(ctx context.Context, target string, fs fs.Interface) (gofsutil.Info, bool, error) { - logFields := common.GetLogFields(ctx) + log := log.WithContext(ctx) var targetMount gofsutil.Info var found bool mounts, err := getMounts(ctx, fs) @@ -246,7 +253,7 @@ func getTargetMount(ctx context.Context, target string, fs fs.Interface) (gofsut for _, mount := range mounts { if mount.Path == target { targetMount = mount - log.WithFields(logFields).Infof("matching targetMount %s target %s", + log.Infof("matching targetMount %s target %s", target, mount.Path) found = true break @@ -255,7 +262,7 @@ func getTargetMount(ctx context.Context, target string, fs fs.Interface) (gofsut return targetMount, found, nil } -func getMounts(ctx context.Context, fs fs.Interface) ([]gofsutil.Info, error) { +func getMounts(_ context.Context, fs fs.Interface) ([]gofsutil.Info, error) { data, err := consistentRead(procMountsPath, procMountsRetries, fs) if err != nil { return []gofsutil.Info{}, err @@ -289,7 +296,7 @@ func consistentRead(filename string, retry int, fs fs.Interface) ([]byte, error) } func createMapping(volID, deviceName, tmpDir string, fs fs.Interface) error { - return fs.WriteFile(path.Join(tmpDir, volID), []byte(deviceName), 0640) + return fs.WriteFile(path.Join(tmpDir, volID), []byte(deviceName), 0o640) } func getMapping(volID, tmpDir string, fs fs.Interface) (string, error) { @@ -311,8 +318,8 @@ func deleteMapping(volID, tmpDir string, fs fs.Interface) error { return err } -func isBlock(cap *csi.VolumeCapability) bool { - _, isBlock := cap.GetAccessType().(*csi.VolumeCapability_Block) +func isBlock(vc *csi.VolumeCapability) bool { + _, isBlock := vc.GetAccessType().(*csi.VolumeCapability_Block) return isBlock } @@ -350,11 +357,12 @@ func getRWModeString(isRO bool) string { } func format(ctx context.Context, source, fsType string, fs fs.Interface, opts ...string) error { - f := log.Fields{ + f := csmlog.Fields{ "source": source, "fsType": fsType, "options": opts, } + log := log.WithContext(ctx).WithFields(f) // Use 'ext4' as the default if fsType == "" { @@ -369,10 +377,10 @@ func format(ctx context.Context, source, fsType string, fs fs.Interface, opts .. } mkfsArgs = append(mkfsArgs, opts...) - log.WithFields(f).Infof("formatting with command: %s %v", mkfsCmd, mkfsArgs) + log.Infof("formatting with command: %s %v", mkfsCmd, mkfsArgs) out, err := fs.ExecCommand(mkfsCmd, mkfsArgs...) if err != nil { - log.WithFields(f).WithError(err).Errorf("formatting disk failed, output: %q", string(out)) + log.Errorf("formatting disk failed with error: %s, output: %q", err.Error(), string(out)) return errors.New(string(out)) } diff --git a/pkg/node/ephemeral.go b/pkg/node/ephemeral.go index f9a51503..cb0a3672 100644 --- a/pkg/node/ephemeral.go +++ b/pkg/node/ephemeral.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2021-2023 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2021-2025 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,7 +27,6 @@ import ( "strconv" "github.com/container-storage-interface/spec/lib/go/csi" - log "github.com/sirupsen/logrus" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -52,10 +51,12 @@ func parseSize(size string) (int64, error) { func (s *Service) ephemeralNodePublish( ctx context.Context, req *csi.NodePublishVolumeRequest) ( - *csi.NodePublishVolumeResponse, error) { + *csi.NodePublishVolumeResponse, error, +) { + log := log.WithContext(ctx) if _, err := s.Fs.Stat(ephemeralStagingMountPath); os.IsNotExist(err) { log.Debug("path does not exists") - err = s.Fs.MkdirAll(ephemeralStagingMountPath, 0750) + err = s.Fs.MkdirAll(ephemeralStagingMountPath, 0o750) if err != nil { log.Errorf("NodestageErrorEph %s", err.Error()) return nil, status.Error(codes.Internal, "Unable to create directory for mounting ephemeral volumes") @@ -85,7 +86,7 @@ func (s *Service) ephemeralNodePublish( return nil, status.Error(codes.Internal, "inline ephemeral create volume failed") } - errLock := s.Fs.MkdirAll(ephemeralStagingMountPath+volID, 0750) + errLock := s.Fs.MkdirAll(ephemeralStagingMountPath+volID, 0o750) if errLock != nil { return nil, errLock } @@ -155,12 +156,13 @@ func (s *Service) ephemeralNodePublish( } return &csi.NodePublishVolumeResponse{}, nil - } func (s *Service) ephemeralNodeUnpublish( ctx context.Context, - req *csi.NodeUnpublishVolumeRequest) error { + req *csi.NodeUnpublishVolumeRequest, +) error { + log := log.WithContext(ctx) volID := req.GetVolumeId() if volID == "" { return status.Error(codes.InvalidArgument, "volume ID is required") @@ -178,7 +180,7 @@ func (s *Service) ephemeralNodeUnpublish( StagingTargetPath: stagingPath, }) if err != nil { - log.Info(err) + log.Info(err.Error()) return status.Error(codes.Internal, "Inline ephemeral node unstage unpublish failed") } log.Info("Calling unpublish") diff --git a/pkg/node/node.go b/pkg/node/node.go index a01594bc..a3b98f58 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2021-2023 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2021-2026 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,31 +23,51 @@ import ( "context" "errors" "fmt" - "net/http" + "maps" + "net/url" "os" "path" "path/filepath" "regexp" + "slices" "strconv" "strings" + "time" "github.com/dell/gonvme" + "github.com/dell/gopowerstore/api" - "github.com/container-storage-interface/spec/lib/go/csi" "github.com/dell/csi-powerstore/v2/pkg/array" - "github.com/dell/csi-powerstore/v2/pkg/common" - "github.com/dell/csi-powerstore/v2/pkg/common/fs" - "github.com/dell/csi-powerstore/v2/pkg/common/k8sutils" "github.com/dell/csi-powerstore/v2/pkg/controller" + "github.com/dell/csi-powerstore/v2/pkg/helpers" + "github.com/dell/csi-powerstore/v2/pkg/identifiers" + "github.com/dell/csi-powerstore/v2/pkg/identifiers/fs" + "github.com/dell/csi-powerstore/v2/pkg/identifiers/k8sutils" + "github.com/dell/csmlog" "github.com/dell/gobrick" csictx "github.com/dell/gocsi/context" "github.com/dell/gofsutil" "github.com/dell/goiscsi" "github.com/dell/gopowerstore" - log "github.com/sirupsen/logrus" + "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/golang/protobuf/proto" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" + corev1 "k8s.io/api/core/v1" + + "k8s.io/apimachinery/pkg/labels" + "k8s.io/component-helpers/scheduling/corev1/nodeaffinity" +) + +// Instantiate csmlog on a package level +var log = csmlog.GetLogger() + +// For unit testing +var ( + createOrUpdateJournalEntryFunc = array.CreateOrUpdateJournalEntry + isNodeConnectedToArrayFunc = isNodeConnectedToArray + checkMetroStateFunc = array.CheckMetroState ) // Opts defines service configuration options. @@ -80,11 +100,10 @@ type Service struct { opts Opts nodeID string - useFC bool - useNVME bool + useFC map[string]bool + useNVME map[string]bool useNFS bool initialized bool - reusedHost bool isHealthMonitorEnabled bool isPodmonEnabled bool @@ -99,22 +118,30 @@ const ( // Will init ISCSIConnector, FcConnector and ControllerService if they are nil. func (s *Service) Init() error { ctx := context.Background() + log := log.WithContext(ctx) s.opts = getNodeOptions() + _, err := k8sutils.CreateKubeClientSet(s.opts.KubeConfigPath) + if err != nil { + return fmt.Errorf("failed to create Kubernetes client: %s", err.Error()) + } + s.initConnectors() - err := s.updateNodeID() + err = s.updateNodeID() if err != nil { return fmt.Errorf("can't update node id: %s", err.Error()) } s.iscsiTargets = make(map[string][]string) s.nvmeTargets = make(map[string][]string) + s.useFC = make(map[string]bool) + s.useNVME = make(map[string]bool) iscsiInitiators, fcInitiators, nvmeInitiators, err := s.getInitiators() if err != nil { return fmt.Errorf("can't get initiators of the node: %s", err.Error()) } - if isPodmonEnabled, ok := csictx.LookupEnv(ctx, common.EnvPodmonEnabled); ok { + if isPodmonEnabled, ok := csictx.LookupEnv(ctx, identifiers.EnvPodmonEnabled); ok { // in case of any error in reading/parsing the env variable default value will be false s.isPodmonEnabled, _ = strconv.ParseBool(isPodmonEnabled) } @@ -125,51 +152,59 @@ func (s *Service) Init() error { return nil } + if len(nvmeInitiators) != 0 { + err = k8sutils.Kubeclient.AddNVMeLabels(ctx, s.opts.KubeNodeName, "hostnqn-uuid", nvmeInitiators) + if err != nil { + log.Warnf("Unable to add hostnqn uuid label for node %s: %v", s.opts.KubeNodeName, err.Error()) + } + } + // Setup host on each of available arrays for _, arr := range s.Arrays() { - if arr.BlockProtocol == common.NoneTransport { + if arr.BlockProtocol == identifiers.NoneTransport { continue } var initiators []string + var useNVME, useFC bool switch arr.BlockProtocol { - case common.NVMETCPTransport: + case identifiers.NVMETCPTransport: if len(nvmeInitiators) == 0 { log.Errorf("NVMeTCP transport was requested but NVMe initiator is not available") } - s.useNVME = true - s.useFC = false - case common.NVMEFCTransport: + useNVME = true + useFC = false + case identifiers.NVMEFCTransport: if len(nvmeInitiators) == 0 { log.Errorf("NVMeFC transport was requested but NVMe initiator is not available") } - s.useNVME = true - s.useFC = true - case common.ISCSITransport: + useNVME = true + useFC = true + case identifiers.ISCSITransport: if len(iscsiInitiators) == 0 { log.Errorf("iSCSI transport was requested but iSCSI initiator is not available") } - s.useNVME = false - s.useFC = false - case common.FcTransport: + useNVME = false + useFC = false + case identifiers.FcTransport: if len(fcInitiators) == 0 { log.Errorf("FC transport was requested but FC initiator is not available") } - s.useNVME = false - s.useFC = true + useNVME = false + useFC = true default: - s.useNVME = len(nvmeInitiators) > 0 - s.useFC = len(fcInitiators) > 0 + useNVME = len(nvmeInitiators) > 0 + useFC = len(fcInitiators) > 0 } - if s.useNVME { + if useNVME { initiators = nvmeInitiators - if s.useFC { + if useFC { log.Infof("NVMeFC Protocol is requested") } else { log.Infof("NVMeTCP Protocol is requested") } - } else if s.useFC { + } else if useFC { initiators = fcInitiators log.Infof("FC Protocol is requested") } else { @@ -177,13 +212,17 @@ func (s *Service) Init() error { log.Infof("iSCSI Protocol is requested") } - err = s.setupHost(initiators, arr.GetClient(), arr.GetIP()) + // store the values in the array list for later use + s.useNVME[arr.GlobalID] = useNVME + s.useFC[arr.GlobalID] = useFC + + err = s.setupHost(initiators, arr.GetClient(), arr.GetIP(), arr.GetGlobalID()) if err != nil { log.Errorf("can't setup host on %s: %s", arr.Endpoint, err.Error()) } } - if isHealthMonitorEnabled, ok := csictx.LookupEnv(ctx, common.EnvIsHealthMonitorEnabled); ok { + if isHealthMonitorEnabled, ok := csictx.LookupEnv(ctx, identifiers.EnvIsHealthMonitorEnabled); ok { s.isHealthMonitorEnabled, _ = strconv.ParseBool(isHealthMonitorEnabled) } @@ -192,8 +231,7 @@ func (s *Service) Init() error { } func (s *Service) initConnectors() { - gobrick.SetLogger(&common.CustomLogger{}) - + gobrick.SetLogger(&identifiers.CustomLogger{}) if s.iscsiConnector == nil { s.iscsiConnector = gobrick.NewISCSIConnector( gobrick.ISCSIConnectorParams{ @@ -236,10 +274,31 @@ func (s *Service) initConnectors() { } } +// Check for duplicate hostnqn uuids +func (s *Service) checkForDuplicateUUIDs() { + duplicateUUIDs := make(map[string]string) + + var err error + nodeUUIDs, err := k8sutils.Kubeclient.GetNVMeUUIDs(context.Background()) + if err != nil { + log.Errorf("Unable to check uuids") + return + } + + // Iterate over all nodes to check their uuid + for node, uuid := range nodeUUIDs { + if existingNode, found := duplicateUUIDs[uuid]; found { + log.Errorf("Duplicate hostnqn uuid %s found on nodes: %s and %s", uuid, existingNode, node) + } else { + duplicateUUIDs[uuid] = node + } + } +} + // NodeStageVolume prepares volume to be consumed by node publish by connecting volume to the node func (s *Service) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) { - logFields := common.GetLogFields(ctx) - + log := log.WithContext(ctx) + logFields := csmlog.ExtractFieldsFromContext(ctx) if req.GetVolumeCapability() == nil { return nil, status.Error(codes.InvalidArgument, "volume capability is required") } @@ -253,43 +312,170 @@ func (s *Service) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeR return nil, status.Error(codes.InvalidArgument, "staging target path is required") } - nvmeIP := strings.Split(req.PublishContext["PORTAL0"], ":") - nvmeTargets, _ := s.nvmeLib.DiscoverNVMeTCPTargets(nvmeIP[0], false) - for i, t := range nvmeTargets { - req.PublishContext[fmt.Sprintf("%s%d", common.PublishContextNVMETCPTargetsPrefix, i)] = t.TargetNqn - req.PublishContext[fmt.Sprintf("%s%d", common.PublishContextNVMETCPPortalsPrefix, i)] = t.Portal + ":4420" + volumeHandle, err := array.ParseVolumeID(ctx, id, s.DefaultArray(), req.VolumeCapability) + if err != nil { + return nil, err } - id, arrayID, protocol, _ := array.ParseVolumeID(ctx, id, s.DefaultArray(), req.VolumeCapability) - - var stager VolumeStager + id = volumeHandle.LocalUUID + arrayID := volumeHandle.LocalArrayGlobalID + protocol := volumeHandle.Protocol + remoteVolumeID := volumeHandle.RemoteUUID + remoteArrayID := volumeHandle.RemoteArrayGlobalID + _, stagingPath := getStagingPath(ctx, req.GetStagingTargetPath(), id) arr, ok := s.Arrays()[arrayID] if !ok { - return nil, status.Errorf(codes.Internal, "can't find array with provided arrayID %s", arrayID) + return nil, status.Errorf(codes.Internal, "can't find array with ID %s", arrayID) + } + + client := arr.GetClient() + + var remoteArray *array.PowerStoreArray + isMetroFractured := false + metroSession := &array.MetroFracturedResponse{ + IsFractured: false, + } + localVolumeDemoted := false + + if volumeHandle.IsMetro() { + remoteArray, ok = s.Arrays()[remoteArrayID] + if !ok { + return nil, status.Errorf(codes.InvalidArgument, "failed to find remote array with ID %s", remoteArrayID) + } + + metroSession, localVolumeDemoted, err = checkMetroStateFunc(ctx, volumeHandle, arr.GetClient(), remoteArray.GetClient()) + if err != nil { + return nil, err + } + + isMetroFractured = metroSession.IsFractured + if isMetroFractured { + log.Warnf("[METRO] metro volume %s is in a fractured state", req.GetVolumeId()) + } + if localVolumeDemoted { + log.Warnf("[METRO] metro volume %s has been demoted", req.GetVolumeId()) + } } + var stager VolumeStager if protocol == "nfs" { stager = &NFSStager{ array: arr, } } else { stager = &SCSIStager{ - useFC: s.useFC, - useNVME: s.useNVME, + useFC: s.useFC[arr.GlobalID], + useNVME: s.useNVME[arr.GlobalID], iscsiConnector: s.iscsiConnector, nvmeConnector: s.nvmeConnector, fcConnector: s.fcConnector, } } - return stager.Stage(ctx, req, logFields, s.Fs, id) + localStaged := false + remoteStaged := false + + var response *csi.NodeStageVolumeResponse + + // For NFS , no need to check for connectivity before attempting staging. + if protocol == "nfs" { + response, err = stager.Stage(ctx, req, stagingPath, s.nodeID, logFields, s.Fs, id, false, client) + if err != nil { + return nil, err + } + return response, nil + } + + // For block volumes, stage only if array has connectivity to this node. + // This supports non-uniform metro configuration and will support Multi-az for powerstore non-metro volumes (future) + nodeConnectedToLocalArray := isNodeConnectedToArrayFunc(ctx, s.nodeID, arr) + if nodeConnectedToLocalArray { + resp, err := stager.Stage(ctx, req, stagingPath, s.nodeID, logFields, s.Fs, id, false, client) + if err != nil { + if isMetroFractured && localVolumeDemoted { + // expected failure if Metro is Fractured and local array is down + log.Infof("[METRO] Could not stage volume %s on node %s for array %s due to Metro Session Fracture", id, s.opts.KubeNodeName, arr.Endpoint) + } else { + log.Errorf("Failed to stage volume %s for array %s: %s", id, arr.Endpoint, err) + return nil, err + } + } else { + log.Infof("Staged volume %s for array %s", id, arr.Endpoint) + localStaged = true + response = resp + } + } else { + log.Warnf("local volume %s has no connectivity to node %s. skipping staging.", id, s.opts.KubeNodeName) + } + + nodeConnectedToRemoteArray := false + if volumeHandle.IsMetro() { // For Remote Metro volume + nodeConnectedToRemoteArray = isNodeConnectedToArrayFunc(ctx, s.nodeID, remoteArray) + if nodeConnectedToRemoteArray { + log.Infof("Staging remote metro volume %s for volume %s", remoteVolumeID, id) + resp, err := stager.Stage(ctx, req, stagingPath, s.nodeID, logFields, s.Fs, remoteVolumeID, true, remoteArray.GetClient()) + if err != nil { + if isMetroFractured && !localVolumeDemoted { + // expected failure if Metro is Fractured and remote array is down + log.Infof("[METRO] Could not stage volume %s on node %s for array %s due to Metro Session Fracture", id, s.opts.KubeNodeName, remoteArray.Endpoint) + } else { + log.Errorf("Failed to stage volume %s for array %s: %s", id, remoteArray.Endpoint, err) + return nil, err + } + } else { + log.Infof("Remote volume %s staged", remoteVolumeID) + remoteStaged = true + response = resp + } + } else { + log.Debugf("skipping staging remote metro %s, node has not been registered with the remote array %s", remoteVolumeID, remoteArrayID) + } + } + + // at least one stage should succeed for non-metro, non-uniform metro, and uniform metro + // if a staging fails for uniform metro, the failed request will be deferred by adding to the volume journal + if !localStaged && !remoteStaged { + return nil, status.Error(codes.Internal, "failed to stage volume") + } + + if volumeHandle.IsMetro() && nodeConnectedToLocalArray && nodeConnectedToRemoteArray { + if (localStaged && !remoteStaged) || (!localStaged && remoteStaged) { + deferredRequest, err := proto.Marshal(req) + if err != nil { + log.Errorf("[METRO] Error marshalling req: %s", err.Error()) + return nil, err + } + + deferredArrayID := arrayID + if !remoteStaged { + deferredArrayID = remoteArrayID + } + + err = createOrUpdateJournalEntryFunc(ctx, metroSession.VolumeName, volumeHandle, deferredArrayID, s.opts.KubeNodeName, "NodeStageVolume", deferredRequest) + if err != nil { + log.Errorf("Could not create journal entry for operation %s for volume %s node %s array %s", "NodeStageVolume", id, s.opts.KubeNodeName, arrayID) + return nil, err + } + + log.Infof("[METRO] Metro volume %s created journal entry for operation %s for volume %s node %s array %s", id, "NodeStageVolume", id, s.opts.KubeNodeName, arrayID) + } + } + return response, nil } // NodeUnstageVolume reverses steps done in NodeStage by disconnecting volume from the node func (s *Service) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) { - logFields := common.GetLogFields(ctx) + log := log.WithContext(ctx) var err error + var reqID string + logFields := csmlog.ExtractFieldsFromContext(ctx) + headers, ok := metadata.FromIncomingContext(ctx) + if ok { + if req, ok := headers["csi.requestid"]; ok && len(req) > 0 && req[0] != "" { + reqID = req[0] + } + } id := req.GetVolumeId() if id == "" { @@ -300,7 +486,7 @@ func (s *Service) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVol return nil, status.Error(codes.InvalidArgument, "staging target path is required") } - id, _, protocol, err := array.ParseVolumeID(ctx, id, s.DefaultArray(), nil) + volumeHandle, err := array.ParseVolumeID(ctx, id, s.DefaultArray(), nil) if err != nil { if apiError, ok := err.(gopowerstore.APIError); ok && apiError.NotFound() { return &csi.NodeUnstageVolumeResponse{}, nil @@ -309,14 +495,45 @@ func (s *Service) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVol "failure checking volume status for volume node unstage: %s", err.Error()) } + id = volumeHandle.LocalUUID + arrayID := volumeHandle.LocalArrayGlobalID + protocol := volumeHandle.Protocol + remoteVolumeID := volumeHandle.RemoteUUID + + arr, ok := s.Arrays()[arrayID] + if !ok { + return nil, status.Errorf(codes.Internal, "can't find array with ID %s", arrayID) + } + + stagingPath := req.GetStagingTargetPath() + + id, stagingPath = getStagingPath(ctx, stagingPath, id) + + vol, err := arr.Client.GetVolume(ctx, id) + if err != nil { + if apiError, ok := err.(gopowerstore.APIError); ok { + if !apiError.NotFound() { + return nil, status.Errorf(codes.Internal, "issue getting volume %s, error %s", id, err.Error()) + } - // append additional path to be able to do bind mounts - stagingPath := getStagingPath(ctx, req.GetStagingTargetPath(), id) + // Not found due to potentially deleted volume through UI. Still need to unstage. + log.Infof("Volume with ID %s not found", id) + } + } device, err := unstageVolume(ctx, stagingPath, id, logFields, err, s.Fs) if err != nil { return nil, err } + if remoteVolumeID != "" { // For Remote Metro volume + log.Info("Unstaging remote metro volume") + _, remoteStagingPath := getStagingPath(ctx, req.GetStagingTargetPath(), remoteVolumeID) + + _, err = unstageVolume(ctx, remoteStagingPath, remoteVolumeID, logFields, err, s.Fs) + if err != nil { + return nil, err + } + } if protocol == "nfs" { return &csi.NodeUnstageVolumeResponse{}, nil @@ -325,47 +542,129 @@ func (s *Service) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVol if device != "" { err := createMapping(id, device, s.opts.TmpDir, s.Fs) if err != nil { - log.WithFields(logFields).Warningf("failed to create vol to device mapping: %s", err.Error()) + log.Warnf("failed to create vol to device mapping : %s", err.Error()) } } else { device, err = getMapping(id, s.opts.TmpDir, s.Fs) if err != nil { - log.WithFields(logFields).Info("no device found. skip device removal") + log.Info("no device found. skip device removal") return &csi.NodeUnstageVolumeResponse{}, nil } } - f := log.Fields{"Device": device} + f := csmlog.Fields{"Device": device} - connectorCtx := common.SetLogFields(context.Background(), logFields) + connectorCtx := csmlog.SetLogFields(context.Background(), logFields) - if s.useNVME { + if s.useNVME[arr.GlobalID] { err = s.nvmeConnector.DisconnectVolumeByDeviceName(connectorCtx, device) - } else if s.useFC { - err = s.fcConnector.DisconnectVolumeByDeviceName(connectorCtx, device) + } else if s.useFC[arr.GlobalID] { + log.Infof("WWN of Volume for unstaging: %s", vol.Wwn) + + volumeWWN := vol.Wwn + err = s.disconnectFCVolume(ctx, reqID, id, arrayID, device, strings.Split(volumeWWN, ".")[1], logFields) } else { err = s.iscsiConnector.DisconnectVolumeByDeviceName(connectorCtx, device) } if err != nil { - log.WithFields(logFields).Error(err) + log.WithFields(logFields).Errorf("failed to disconnect volume: %s", err.Error()) return nil, err } - log.WithFields(logFields).WithFields(f).Info("block device removal complete") + + log.WithFields(logFields).WithFields(f).Infof("block device %s removal completed :", device) err = deleteMapping(id, s.opts.TmpDir, s.Fs) if err != nil { - log.WithFields(logFields).Warningf("failed to remove vol to Dev mapping: %s", err.Error()) + log.WithFields(logFields).Warnf("failed to delete vol to device mapping : %s", err.Error()) } return &csi.NodeUnstageVolumeResponse{}, nil } -func unstageVolume(ctx context.Context, stagingPath, id string, logFields log.Fields, err error, fs fs.Interface) (string, error) { +// New method implementing FC volume disconnection with retry logic similar to PowerMax +func (s *Service) disconnectFCVolume(ctx context.Context, reqID, volumeID, arrayID, device, volumeWWN string, logFields map[string]interface{}) error { + log := log.WithContext(ctx) + var err error + maxDisconnectRetries := identifiers.GetVolumeDisconnectMaxRetries() + timeout := identifiers.GetVolumeDisconnectTimeout() + retryInterval := identifiers.GetVolumeDisconnectRetryInterval() + + f := csmlog.Fields{ + "CSIRequestID": reqID, + "VolumeID": volumeID, + "ArrayID": arrayID, + "Device": device, + "WWN": volumeWWN, + } + log.Infof("WWN of Volume for disconnectingFCVolume: %s", volumeWWN) + + for i := 1; i <= maxDisconnectRetries; i++ { + f["Retry"] = i + log.WithFields(f).Info("NodeUnstageVolume disconnect volume FC") + + // Create context with timeout for disconnection + disconnectCtx, cancel := context.WithTimeout(ctx, timeout) + disconnectCtx = csmlog.SetLogFields(disconnectCtx, logFields) + + if volumeWWN != "" { + // Preferred: Use WWN-based disconnection (more reliable) + err = s.fcConnector.DisconnectVolumeByWWN(disconnectCtx, volumeWWN) + } else { + // Fallback: Use device name based disconnection + err = s.fcConnector.DisconnectVolumeByDeviceName(disconnectCtx, device) + } + + cancel() + + if err == nil { + log.WithFields(f).Debug("FC disconnect volume complete") + + // Clean up symlink if WWN was available + if volumeWWN != "" { + symlinkPath, _, err := gofsutil.WWNToDevicePathX(ctx, volumeWWN) + if err != nil { + log.WithFields(f).Warnf("failed to resolve symlink path for WWN %s: %s", volumeWWN, err) + } else if symlinkPath != "" { + if removeErr := os.Remove(symlinkPath); removeErr != nil && !os.IsNotExist(removeErr) { + log.WithFields(f).Warnf("failed to remove symlink at path %s: %s", symlinkPath, removeErr.Error()) + } + } + } + return nil + } + + log.WithFields(f).Errorf("error disconnecting volume for retry %d: %s", i, err.Error()) + + if i < maxDisconnectRetries { + time.Sleep(retryInterval) + + // Additional check: verify if device still exists before retrying + if volumeWWN != "" { + devPath, err := gofsutil.WWNToDevicePath(ctx, volumeWWN) + if err != nil { + log.WithFields(f).Warnf("failed to resolve device path for WWN %s: %v", volumeWWN, err) + return nil + } + if devPath == "" { + log.WithFields(f).Info("device no longer exists, considering disconnect successful") + return nil + } + } + } + } + + return status.Errorf(codes.Internal, + "FC disconnectVolume exceeded retry limit %d for volume %s, device %s, WWN %s", + maxDisconnectRetries, volumeID, device, volumeWWN) +} + +func unstageVolume(ctx context.Context, stagingPath, id string, logFields csmlog.Fields, err error, fs fs.Interface) (string, error) { logFields["ID"] = id logFields["StagingPath"] = stagingPath - ctx = common.SetLogFields(ctx, logFields) + ctx = csmlog.SetLogFields(ctx, logFields) + log := log.WithContext(ctx).WithFields(logFields) - log.WithFields(logFields).Info("calling unstage") + log.Info("calling unstage") device, err := getStagedDev(ctx, stagingPath, fs) if err != nil { @@ -375,16 +674,16 @@ func unstageVolume(ctx context.Context, stagingPath, id string, logFields log.Fi if device != "" { _, device = path.Split(device) - log.WithFields(logFields).Infof("active mount exist") + log.Info("active mount exist") err = fs.GetUtil().Unmount(ctx, stagingPath) if err != nil { return "", status.Errorf(codes.Internal, "could not unmount dev %s: %s", device, err.Error()) } - log.WithFields(logFields).Infof("unmount without error") + log.Info("unmount without error") } else { // no mounts - log.WithFields(logFields).Infof("no mounts found") + log.Info("no active mounts found") } err = fs.Remove(stagingPath) @@ -400,18 +699,22 @@ func unstageVolume(ctx context.Context, stagingPath, id string, logFields log.Fi return "", status.Errorf(codes.Internal, "failed to delete mount path %s: %s", stagingPath, err.Error()) } - log.WithFields(logFields).Infof("target mount file deleted") + log.Info("target mount file deleted") return device, nil } -func removeRemnantMounts(ctx context.Context, stagingPath string, fs fs.Interface, logFields log.Fields) (string, error) { - log.WithFields(logFields).Infof("getting remnant mount") +func removeRemnantMounts(ctx context.Context, stagingPath string, fs fs.Interface, logFields csmlog.Fields) (string, error) { + log := log.WithContext(ctx).WithFields(logFields) + log.Info("finding remnant mount") mounts, found, err := getRemnantTargetMounts(ctx, stagingPath, fs) - if !found || err != nil { + if err != nil { return "", fmt.Errorf("could not reliably determine remnant mounts for path %s: %s", stagingPath, err.Error()) } + if !found { + return "", fmt.Errorf("no remnant mounts for %s", stagingPath) + } - log.WithFields(logFields).Infof("%d remnant mount exist", len(mounts)) + log.Infof("%d remnant mount exist", len(mounts)) for _, mount := range mounts { delete(logFields, "StagingPath") logFields["RemnantPath"] = mount.Path @@ -419,7 +722,7 @@ func removeRemnantMounts(ctx context.Context, stagingPath string, fs fs.Interfac if err != nil { return "", fmt.Errorf("could not unmount dev %s: %s", mount.Path, err.Error()) } - log.WithFields(logFields).Infof("unmount without error") + log.Info("unmount without error") } delete(logFields, "RemnantPath") @@ -432,8 +735,10 @@ func removeRemnantMounts(ctx context.Context, stagingPath string, fs fs.Interfac // NodePublishVolume publishes volume to the node by mounting it to the target path func (s *Service) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { - logFields := common.GetLogFields(ctx) + log := log.WithContext(ctx) + logFields := csmlog.ExtractFieldsFromContext(ctx) var ephemeralVolume bool + ephemeral, ok := req.VolumeContext["csi.storage.k8s.io/ephemeral"] if ok { ephemeralVolume = strings.ToLower(ephemeral) == "true" @@ -461,10 +766,11 @@ func (s *Service) NodePublishVolume(ctx context.Context, req *csi.NodePublishVol return nil, status.Error(codes.InvalidArgument, "stagingPath is required") } - id, _, protocol, _ := array.ParseVolumeID(ctx, id, s.DefaultArray(), req.VolumeCapability) + volumeHandle, _ := array.ParseVolumeID(ctx, id, s.DefaultArray(), req.VolumeCapability) + id = volumeHandle.LocalUUID + protocol := volumeHandle.Protocol - // append additional path to be able to do bind mounts - stagingPath := getStagingPath(ctx, req.GetStagingTargetPath(), id) + id, stagingPath := getStagingPath(ctx, req.GetStagingTargetPath(), id) isRO := req.GetReadonly() volumeCapability := req.GetVolumeCapability() @@ -473,9 +779,9 @@ func (s *Service) NodePublishVolume(ctx context.Context, req *csi.NodePublishVol logFields["TargetPath"] = targetPath logFields["StagingPath"] = stagingPath logFields["ReadOnly"] = req.GetReadonly() - ctx = common.SetLogFields(ctx, logFields) + ctx = csmlog.SetLogFields(ctx, logFields) - log.WithFields(logFields).Info("calling publish") + log.WithFields(logFields).Info("calling node publish volume") var publisher VolumePublisher @@ -497,7 +803,8 @@ func (s *Service) NodePublishVolume(ctx context.Context, req *csi.NodePublishVol // NodeUnpublishVolume unpublishes volume from the node by unmounting it from the target path func (s *Service) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) { - logFields := common.GetLogFields(ctx) + logFields := csmlog.ExtractFieldsFromContext(ctx) + log := log.WithFields(logFields) var err error targetPath := req.GetTargetPath() @@ -518,8 +825,8 @@ func (s *Service) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublis } logFields["ID"] = volID logFields["TargetPath"] = targetPath - ctx = common.SetLogFields(ctx, logFields) - log.WithFields(logFields).Info("calling unpublish") + ctx = csmlog.SetLogFields(ctx, logFields) + log.Info("calling unpublish") _, found, err := getTargetMount(ctx, targetPath, s.Fs) if err != nil { @@ -530,11 +837,11 @@ func (s *Service) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublis if !found { // no mounts - log.WithFields(logFields).Infof("no mounts found") + log.Info("no mounts found") return &csi.NodeUnpublishVolumeResponse{}, nil } - log.WithFields(logFields).Infof("active mount exist") + log.Info("active mount exist") err = s.Fs.GetUtil().Unmount(ctx, targetPath) if err != nil { return nil, status.Errorf(codes.Internal, @@ -542,7 +849,13 @@ func (s *Service) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublis targetPath, err.Error()) } - log.WithFields(logFields).Info("unpublish complete") + // remove target path + err = s.Fs.Remove(targetPath) + if err != nil { + return nil, status.Errorf(codes.Internal, "Failed to remove target path: %s as part of NodeUnpublish: %s", targetPath, err.Error()) + } + + log.Info("unpublish complete") log.Debug("Checking for ephemeral after node unpublish") if ephemeralVolume { @@ -569,20 +882,35 @@ func (s *Service) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGetVolume return nil, status.Error(codes.InvalidArgument, "no volume Path provided") } + if !filepath.IsAbs(volumePath) { + return nil, status.Error(codes.NotFound, "no volume Path provided") + } + // parse volume Id - id, arrayID, protocol, err := array.ParseVolumeID(ctx, volumeID, s.DefaultArray(), nil) + volumeHandle, err := array.ParseVolumeID(ctx, volumeID, s.DefaultArray(), nil) if err != nil { if apiError, ok := err.(gopowerstore.APIError); ok && apiError.NotFound() { return nil, err } return nil, err } + id := volumeHandle.LocalUUID + arrayID := volumeHandle.LocalArrayGlobalID + protocol := volumeHandle.Protocol arr, ok := s.Arrays()[arrayID] if !ok { return nil, status.Error(codes.InvalidArgument, "failed to find array with given ID") } - + // default empty usage + usage := []*csi.VolumeUsage{ + { + Available: 0, + Total: 0, + Used: 0, + Unit: csi.VolumeUsage_BYTES, + }, + } // Validate if volume exists if protocol == "nfs" { fs, err := arr.Client.GetFS(ctx, id) @@ -591,6 +919,7 @@ func (s *Service) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGetVolume return nil, status.Errorf(codes.NotFound, "failed to find filesystem %s with error: %v", id, err.Error()) } resp := &csi.NodeGetVolumeStatsResponse{ + Usage: usage, VolumeCondition: &csi.VolumeCondition{ Abnormal: true, Message: fmt.Sprintf("Filesystem %s is not found", id), @@ -604,6 +933,7 @@ func (s *Service) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGetVolume return nil, status.Errorf(codes.NotFound, "failed to find nfs export for filesystem with error: %v", err.Error()) } resp := &csi.NodeGetVolumeStatsResponse{ + Usage: usage, VolumeCondition: &csi.VolumeCondition{ Abnormal: true, Message: fmt.Sprintf("NFS export for volume %s is not found", id), @@ -616,13 +946,23 @@ func (s *Service) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGetVolume hosts = append(hosts, nfsExport.RWHosts...) hosts = append(hosts, nfsExport.RWRootHosts...) attached := false + // Extract the IP address from the node ID + ipList := identifiers.GetIPListFromString(s.nodeID) + if len(ipList) == 0 { + return nil, status.Errorf(codes.NotFound, "failed to find IP in nodeID %s", s.nodeID) + } + nodeIP := ipList[0] for _, host := range hosts { - if s.nodeID == host { + // Extract the IP address from the host (IP/netmask) + hostIP := strings.Split(host, "/")[0] + if nodeIP == hostIP { attached = true + break } } if !attached { resp := &csi.NodeGetVolumeStatsResponse{ + Usage: usage, VolumeCondition: &csi.VolumeCondition{ Abnormal: true, Message: fmt.Sprintf("host %s is not attached to NFS export for filesystem %s", s.nodeID, id), @@ -637,6 +977,7 @@ func (s *Service) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGetVolume return nil, status.Errorf(codes.NotFound, "failed to find volume %s with error: %v", id, err.Error()) } resp := &csi.NodeGetVolumeStatsResponse{ + Usage: usage, VolumeCondition: &csi.VolumeCondition{ Abnormal: true, Message: fmt.Sprintf("Volume %s is not found", id), @@ -657,6 +998,7 @@ func (s *Service) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGetVolume return nil, status.Errorf(codes.NotFound, "failed to get host: %s with error: %v", hostMapping.HostID, err.Error()) } resp := &csi.NodeGetVolumeStatsResponse{ + Usage: usage, VolumeCondition: &csi.VolumeCondition{ Abnormal: true, Message: fmt.Sprintf("host %s is not attached to volume %s", s.nodeID, id), @@ -687,6 +1029,7 @@ func (s *Service) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGetVolume } if !hostMapped { resp := &csi.NodeGetVolumeStatsResponse{ + Usage: usage, VolumeCondition: &csi.VolumeCondition{ Abnormal: true, Message: fmt.Sprintf("host %s is not attached to volume %s", s.nodeID, id), @@ -701,10 +1044,11 @@ func (s *Service) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGetVolume // Check if staging target path is mounted _, found, err := getTargetMount(ctx, stagingPath, s.Fs) if err != nil { - return nil, status.Errorf(codes.Internal, "can't check mounts for path %s: %s", volumePath, err.Error()) + return nil, status.Errorf(codes.Internal, "can't check mounts for path %s: %s", stagingPath, err.Error()) } if !found { resp := &csi.NodeGetVolumeStatsResponse{ + Usage: usage, VolumeCondition: &csi.VolumeCondition{ Abnormal: true, Message: fmt.Sprintf("staging target path %s not mounted for volume %s", stagingPath, id), @@ -721,6 +1065,7 @@ func (s *Service) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGetVolume } if !found { resp := &csi.NodeGetVolumeStatsResponse{ + Usage: usage, VolumeCondition: &csi.VolumeCondition{ Abnormal: true, Message: fmt.Sprintf("volume path %s not mounted for volume %s", volumePath, id), @@ -733,6 +1078,7 @@ func (s *Service) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGetVolume _, err = os.ReadDir(volumePath) if err != nil { resp := &csi.NodeGetVolumeStatsResponse{ + Usage: usage, VolumeCondition: &csi.VolumeCondition{ Abnormal: true, Message: fmt.Sprintf("volume path %s not accessible for volume %s", volumePath, id), @@ -773,6 +1119,7 @@ func (s *Service) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGetVolume // NodeExpandVolume expands the volume by re-scanning and resizes filesystem if needed func (s *Service) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandVolumeRequest) (*csi.NodeExpandVolumeResponse, error) { + log := log.WithContext(ctx) var reqID string var err error headers, ok := metadata.FromIncomingContext(ctx) @@ -783,37 +1130,81 @@ func (s *Service) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandVolum } // Get the VolumeID and validate against the volume - id, arrayID, _, err := array.ParseVolumeID(ctx, req.VolumeId, s.DefaultArray(), nil) + volumeHandle, err := array.ParseVolumeID(ctx, req.VolumeId, s.DefaultArray(), nil) if err != nil { if apiError, ok := err.(gopowerstore.APIError); ok && apiError.NotFound() { - return nil, err + // Return error code csi-sanity test expects + return nil, status.Error(codes.NotFound, err.Error()) } return nil, err } + targetPath := req.GetVolumePath() + if targetPath == "" { + return nil, status.Error(codes.InvalidArgument, "targetPath is required") + } + + if volumeHandle.Protocol == "nfs" { + // workaround for https://github.com/kubernetes/kubernetes/issues/131419 + return &csi.NodeExpandVolumeResponse{}, nil + } + + id := volumeHandle.LocalUUID + arrayID := volumeHandle.LocalArrayGlobalID + arr, ok := s.Arrays()[arrayID] if !ok { return nil, status.Error(codes.InvalidArgument, "failed to find array with given ID") } - targetPath := req.GetVolumePath() - if targetPath == "" { - return nil, status.Error(codes.InvalidArgument, "targetPath is required") - } isBlock := strings.Contains(targetPath, blockVolumePathMarker) - // Parse the CSI VolumeId and validate against the volume vol, err := arr.Client.GetVolume(ctx, id) if err != nil { // If the volume isn't found, we cannot stage it return nil, status.Error(codes.NotFound, "Volume not found") } + + isAuthEnabled := os.Getenv("X_CSM_AUTH_ENABLED") + if isAuthEnabled == "true" { + // If the volume is created from Auth v2 which has tenant prefix then we need to remove that while publishing, otherwise mount will fail - THIS IS A TEMPORARY FIX + splittedVolName := strings.Split(vol.Name, "-") + if len(splittedVolName) > 2 { + vol.Name = strings.Join(splittedVolName[1:], "-") // we will just discard first part which is tenant prefix - Ex: tn1-csivol-12345 + } + } + + log.Debugf("Volume name: %s", vol.Name) + volumeWWN := vol.Wwn // Locate and fetch all (multipath/regular) mounted paths using this volume var devMnt *gofsutil.DeviceMountInfo var targetmount string devMnt, err = s.Fs.GetUtil().GetMountInfoFromDevice(ctx, vol.Name) + + // Stop block volume expansion if metro session is paused + // User needs to resume it first. + remoteVolumeID := volumeHandle.RemoteUUID // metro indicator + if remoteVolumeID != "" { + if vol.MetroReplicationSessionID == "" { + return nil, status.Errorf(codes.Internal, + "cannot expand volume %s: missing metro replication session ID", vol.Name) + } + + state, err := controller.GetMetroSessionState(ctx, vol.MetroReplicationSessionID, arr) + if err != nil { + return nil, status.Errorf(codes.Internal, + "cannot expand volume %s: failed to get metro session state: %v", vol.Name, err) + } + + if state != gopowerstore.RsStateOk { + return nil, status.Errorf(codes.Aborted, + "cannot expand volume %s: metro session %s is not active, its in %s state", + vol.Name, vol.MetroReplicationSessionID, state) + } + } + if err != nil { if isBlock { return s.nodeExpandRawBlockVolume(ctx, volumeWWN) @@ -826,12 +1217,14 @@ func (s *Service) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandVolum log.Infof("DisklLocation: %s", disklocation) targetmount = fmt.Sprintf("tmp/%s/%s", vol.ID, vol.Name) log.Infof("TargetMount: %s", targetmount) - err = s.Fs.MkdirAll(targetmount, 0750) + err = s.Fs.MkdirAll(targetmount, 0o750) if err != nil { return nil, status.Error(codes.Internal, fmt.Sprintf("Failed to find mount info for (%s) with error (%s)", vol.Name, err.Error())) } - err = s.Fs.GetUtil().Mount(ctx, disklocation, targetmount, "") + + mntFlags := identifiers.GetMountFlags(req.GetVolumeCapability()) + err = s.Fs.GetUtil().Mount(ctx, disklocation, targetmount, "", mntFlags...) if err != nil { return nil, status.Error(codes.Internal, fmt.Sprintf("Failed to find mount info for (%s) with error (%s)", vol.Name, err.Error())) @@ -863,7 +1256,7 @@ func (s *Service) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandVolum size := req.GetCapacityRange().GetRequiredBytes() - f := log.Fields{ + f := csmlog.Fields{ "CSIRequestID": reqID, "VolumeName": vol.Name, "VolumePath": targetPath, @@ -871,8 +1264,7 @@ func (s *Service) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandVolum "VolumeWWN": volumeWWN, } log.WithFields(f).Info("Calling resize the file system") - - if !s.useNVME { + if !s.useNVME[arr.GlobalID] { // Rescan the device for the volume expanded on the array for _, device := range devMnt.DeviceNames { devicePath := sysBlock + device @@ -958,6 +1350,7 @@ func (s *Service) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandVolum } func (s *Service) nodeExpandRawBlockVolume(ctx context.Context, volumeWWN string) (*csi.NodeExpandVolumeResponse, error) { + log := log.WithContext(ctx) log.Info(" Block volume expansion. Will try to perform a rescan...") wwnNum := strings.Replace(volumeWWN, "naa.", "", 1) deviceNames, err := s.Fs.GetUtil().GetSysBlockDevicesForVolumeWWN(context.Background(), wwnNum) @@ -968,12 +1361,29 @@ func (s *Service) nodeExpandRawBlockVolume(ctx context.Context, volumeWWN string if len(deviceNames) > 0 { var devName string for _, deviceName := range deviceNames { - devicePath := sysBlock + deviceName - log.Infof("Rescanning unmounted (raw block) device %s to expand size", deviceName) - err = s.Fs.GetUtil().DeviceRescan(context.Background(), devicePath) - if err != nil { - log.Errorf("Failed to rescan device (%s) with error (%s)", devicePath, err.Error()) - return nil, status.Error(codes.Internal, err.Error()) + if strings.HasPrefix(deviceName, "nvme") { + nvmeControllerDevice, err := s.Fs.GetUtil().GetNVMeController(deviceName) + if err != nil { + log.Errorf("Failed to rescan device (%s) with error (%s)", deviceName, err.Error()) + return nil, status.Error(codes.Internal, err.Error()) + } + if nvmeControllerDevice != "" { + devicePath := dev + nvmeControllerDevice + log.Infof("Rescanning unmounted (raw block) device %s to expand size", devicePath) + err = s.nvmeLib.DeviceRescan(devicePath) + if err != nil { + log.Errorf("Failed to rescan device (%s) with error (%s)", devicePath, err.Error()) + return nil, status.Error(codes.Internal, err.Error()) + } + } + } else { + devicePath := sysBlock + deviceName + log.Infof("Rescanning unmounted (raw block) device %s to expand size", deviceName) + err = s.Fs.GetUtil().DeviceRescan(context.Background(), devicePath) + if err != nil { + log.Errorf("Failed to rescan device (%s) with error (%s)", devicePath, err.Error()) + return nil, status.Error(codes.Internal, err.Error()) + } } devName = deviceName } @@ -1000,12 +1410,12 @@ func (s *Service) nodeExpandRawBlockVolume(ctx context.Context, volumeWWN string } // NodeGetCapabilities returns supported features by the node service -func (s *Service) NodeGetCapabilities(context context.Context, request *csi.NodeGetCapabilitiesRequest) (*csi.NodeGetCapabilitiesResponse, error) { - newCap := func(cap csi.NodeServiceCapability_RPC_Type) *csi.NodeServiceCapability { +func (s *Service) NodeGetCapabilities(_ context.Context, _ *csi.NodeGetCapabilitiesRequest) (*csi.NodeGetCapabilitiesResponse, error) { + newCap := func(capability csi.NodeServiceCapability_RPC_Type) *csi.NodeServiceCapability { return &csi.NodeServiceCapability{ Type: &csi.NodeServiceCapability_Rpc{ Rpc: &csi.NodeServiceCapability_RPC{ - Type: cap, + Type: capability, }, }, } @@ -1034,9 +1444,10 @@ func (s *Service) NodeGetCapabilities(context context.Context, request *csi.Node } // NodeGetInfo returns id of the node and topology constraints -func (s *Service) NodeGetInfo(ctx context.Context, req *csi.NodeGetInfoRequest) (*csi.NodeGetInfoResponse, error) { +func (s *Service) NodeGetInfo(ctx context.Context, _ *csi.NodeGetInfoRequest) (*csi.NodeGetInfoResponse, error) { // Create the topology keys // /-: true + log := log.WithContext(ctx) resp := &csi.NodeGetInfoResponse{ NodeId: s.nodeID, AccessibleTopology: &csi.Topology{ @@ -1044,16 +1455,29 @@ func (s *Service) NodeGetInfo(ctx context.Context, req *csi.NodeGetInfoRequest) }, } + nodeLabels, err := k8sutils.Kubeclient.GetNodeLabels(ctx, s.opts.KubeNodeName) + if err != nil { + log.Warnf("failed to get Node Labels with error: %s", err.Error()) + } + for _, arr := range s.Arrays() { - _, err := getOutboundIP(arr.GetIP(), s.Fs) - if err == nil { - resp.AccessibleTopology.Segments[common.Name+"/"+arr.GetIP()+"-nfs"] = "true" + if isNFSEnabled, err := identifiers.IsNFSServiceEnabled(ctx, arr.GetClient()); err != nil { + log.Errorf("failed to validate NFS service for the array: %s", err.Error()) + } else if isNFSEnabled { + log.Infof("NFS service is enabled on the array %s ", arr.GetGlobalID()) + // we will chop off port from the host if present. + port, err := ExtractPort(arr.Endpoint) + _, err = getOutboundIP(arr.GetIP(), port, s.Fs) + if err == nil { + resp.AccessibleTopology.Segments[identifiers.Name+"/"+arr.GetIP()+"-nfs"] = "true" + } else { + log.Errorf("Error: failed to get ip details: %s\n", err.Error()) + } } - - if arr.BlockProtocol != common.NoneTransport { - if s.useNVME { - if s.useFC { - nvmefcInfo, err := common.GetNVMEFCTargetInfoFromStorage(arr.GetClient(), "") + if arr.BlockProtocol != identifiers.NoneTransport { + if s.useNVME[arr.GlobalID] { + if s.useFC[arr.GlobalID] { + nvmefcInfo, err := identifiers.GetNVMEFCTargetInfoFromStorage(arr.GetClient(), "") if err != nil { log.Errorf("couldn't get targets from the array: %s", err.Error()) continue @@ -1066,43 +1490,57 @@ func (s *Service) NodeGetInfo(ctx context.Context, req *csi.NodeGetInfoRequest) if err != nil { log.Errorf("couldn't discover NVMeFC targets") continue - } else { - for _, target := range NVMeFCTargets { - err = s.nvmeLib.NVMeFCConnect(target, false) - if err != nil { - log.Errorf("couldn't connect to NVMeFC target") - } else { - nvmefcConnectCount = nvmefcConnectCount + 1 - otherTargets := s.nvmeTargets[arr.GlobalID] - s.nvmeTargets[arr.GlobalID] = append(otherTargets, target.TargetNqn) - } + } + for _, target := range NVMeFCTargets { + err = s.nvmeLib.NVMeFCConnect(target, false) + if err != nil { + log.Errorf("couldn't connect to NVMeFC target") + } else { + nvmefcConnectCount = nvmefcConnectCount + 1 + otherTargets := s.nvmeTargets[arr.GlobalID] + s.nvmeTargets[arr.GlobalID] = append(otherTargets, target.TargetNqn) } } } if nvmefcConnectCount != 0 { - resp.AccessibleTopology.Segments[common.Name+"/"+arr.GetIP()+"-nvmefc"] = "true" + resp.AccessibleTopology.Segments[identifiers.Name+"/"+arr.GetIP()+"-nvmefc"] = "true" } } else { - infoList, err := common.GetISCSITargetsInfoFromStorage(arr.GetClient(), "") + // useNVME/TCP + infoList, err := identifiers.GetNVMETCPTargetsInfoFromStorage(arr.GetClient(), "") if err != nil { log.Errorf("couldn't get targets from array: %s", err.Error()) continue } + var nvmeTargets []gonvme.NVMeTarget + networkIDs := map[string]struct{}{} for _, address := range infoList { + // discovering with one portal returns all targets in the network + // so if we already discovered an address with this network ID, continue + if _, ok := networkIDs[address.NetworkID]; ok { + continue + } + + // discover the target // doesn't matter how many portals are present, discovering from any one will list out all targets - nvmeIP := strings.Split(address.Portal, ":") - log.Info("Trying to discover NVMe target from portal ", nvmeIP[0]) - nvmeTargets, err = s.nvmeLib.DiscoverNVMeTCPTargets(nvmeIP[0], false) + nvmeIP := strings.Split(address.Portal, ":")[0] + log.Infof("Trying to discover NVMe targets from portal %s on network %s", nvmeIP, address.NetworkID) + discoveredTargets, err := s.nvmeLib.DiscoverNVMeTCPTargets(nvmeIP, false) if err != nil { - log.Error("couldn't discover targets") + log.Errorf("discovering portal: %s: %v", nvmeIP, err) continue } - break + + nvmeTargets = append(nvmeTargets, discoveredTargets...) + + // mark this network ID as discovered so we don't discover another portal in the same network + // since it will return all the same target information already seen + networkIDs[address.NetworkID] = struct{}{} } loginToAtleastOneTarget := false for _, target := range nvmeTargets { - log.Info("Logging to NVMe target ", target) + log.Infof("Logging to NVMe target %v", target) err = s.nvmeLib.NVMeTCPConnect(target, false) if err != nil { log.Errorf("couldn't connect to the nvme target") @@ -1113,29 +1551,17 @@ func (s *Service) NodeGetInfo(ctx context.Context, req *csi.NodeGetInfoRequest) loginToAtleastOneTarget = true } if loginToAtleastOneTarget { - resp.AccessibleTopology.Segments[common.Name+"/"+arr.GetIP()+"-nvmetcp"] = "true" + resp.AccessibleTopology.Segments[identifiers.Name+"/"+arr.GetIP()+"-nvmetcp"] = "true" } else { s.useNFS = true } } - - } else if s.useFC { + } else if s.useFC[arr.GlobalID] { // Check node initiators connection to array - nodeID := s.nodeID - if s.reusedHost { - ipList := common.GetIPListFromString(nodeID) - if ipList == nil || len(ipList) == 0 { - log.Errorf("can't find ip in nodeID %s", nodeID) - continue - } - ip := ipList[len(ipList)-1] - nodeID = nodeID[:len(nodeID)-len(ip)-1] - } - - host, err := arr.GetClient().GetHostByName(ctx, nodeID) + host, err := arr.GetClient().GetHostByName(ctx, s.nodeID) if err != nil { - log.WithFields(log.Fields{ - "hostName": nodeID, + log.WithFields(csmlog.Fields{ + "hostName": s.nodeID, "error": err, }).Error("could not find host on PowerStore array") continue @@ -1146,46 +1572,66 @@ func (s *Service) NodeGetInfo(ctx context.Context, req *csi.NodeGetInfoRequest) continue } - if len(host.Initiators[0].ActiveSessions) != 0 { - resp.AccessibleTopology.Segments[common.Name+"/"+arr.GetIP()+"-fc"] = "true" + fcInitiatorsWithActiveSessionCount := countActiveSessionsInitiators(host) + if fcInitiatorsWithActiveSessionCount > 0 { + resp.AccessibleTopology.Segments[identifiers.Name+"/"+arr.GetIP()+"-fc"] = "true" } else { - log.WithFields(log.Fields{ + log.WithFields(csmlog.Fields{ "hostName": host.Name, "initiator": host.Initiators[0].PortName, }).Error("there is no active FC sessions") continue } } else { - infoList, err := common.GetISCSITargetsInfoFromStorage(arr.GetClient(), "") + infoList, err := identifiers.GetISCSITargetsInfoFromStorage(arr.GetClient(), "") if err != nil { log.Errorf("couldn't get targets from array: %s", err.Error()) continue } - + var ipAddress string var iscsiTargets []goiscsi.ISCSITarget + networkIDs := map[string]struct{}{} for _, address := range infoList { + // discovering with one portal returns all targets in the network + // so if we already discovered an address with this network ID, continue + if _, ok := networkIDs[address.NetworkID]; ok { + continue + } + // first check if this portal is reachable from this machine or not if ReachableEndPoint(address.Portal) { + ipAddressList := splitIPAddress(address.Portal) + ipAddress = ipAddressList[0] // doesn't matter how many portals are present, discovering from any one will list out all targets - log.Info("Trying to discover iSCSI target from portal ", address.Portal) - iscsiTargets, err = s.iscsiLib.DiscoverTargets(address.Portal, false) + log.Infof("Trying to discover iSCSI target from portal %s", ipAddress) + + ipInterface, err := s.iscsiLib.GetInterfaceForTargetIP(ipAddress) if err != nil { - log.Error("couldn't discover targets") + log.Errorf("couldn't get interface: %s", err.Error()) continue } - break - } else { - log.Debugf("Portal %s is not rechable from the node", address.Portal) + discoveredTargets, err := s.iscsiLib.DiscoverTargetsWithInterface(address.Portal, ipInterface[ipAddress], false) + if err != nil { + log.Errorf("couldn't discover targets: %s", err.Error()) + continue + } + + iscsiTargets = append(iscsiTargets, discoveredTargets...) + + // mark this network ID as discovered so we don't discover another portal in the same network + // since it will return all the same target information already seen + networkIDs[address.NetworkID] = struct{}{} } + log.Debugf("Portal is not rechable from the node") } // login is also performed as a part of ConnectVolume by using dynamically created chap credentials, In case if it fails here if len(iscsiTargets) > 0 { - resp.AccessibleTopology.Segments[common.Name+"/"+arr.GetIP()+"-iscsi"] = "true" + resp.AccessibleTopology.Segments[identifiers.Name+"/"+arr.GetIP()+"-iscsi"] = "true" } loginToAtleastOneTarget := false for _, target := range iscsiTargets { if ReachableEndPoint(target.Portal) { - log.Info("Logging to Iscsi target ", target) + log.Infof("Logging to Iscsi target %v", target) if s.opts.EnableCHAP { log.Debug("Setting CHAP Credentials before login") err = s.iscsiLib.SetCHAPCredentials(target, s.opts.CHAPUsername, s.opts.CHAPPassword) @@ -1211,9 +1657,11 @@ func (s *Service) NodeGetInfo(ctx context.Context, req *csi.NodeGetInfoRequest) } } } + + updateMetroToplogy(arr, nodeLabels, resp) } - var maxVolumesPerNode int64 = 0 + var maxVolumesPerNode int64 // Setting maxVolumesPerNode using the value of field maxPowerstoreVolumesPerNode specified in values.yaml if s.opts.MaxVolumesPerNode > 0 { @@ -1221,11 +1669,8 @@ func (s *Service) NodeGetInfo(ctx context.Context, req *csi.NodeGetInfoRequest) } // Check for node label 'max-powerstore-volumes-per-node'. If present set 'maxVolumesPerNode' to this value. - labels, err := k8sutils.GetNodeLabels(ctx, s.opts.KubeConfigPath, s.opts.KubeNodeName) - if err != nil { - log.Warnf("failed to get Node Labels with error: %s", err.Error()) - } else if labels != nil { - if val, ok := labels[maxPowerstoreVolumesPerNodeLabel]; ok { + if nodeLabels != nil { + if val, ok := nodeLabels[maxPowerstoreVolumesPerNodeLabel]; ok { maxVols, err := strconv.ParseInt(val, 10, 64) if err != nil { log.Warnf("invalid value '%s' specified for 'max-powerstore-volumes-per-node' node label", val) @@ -1245,11 +1690,22 @@ func (s *Service) NodeGetInfo(ctx context.Context, req *csi.NodeGetInfoRequest) return resp, nil } +// Count the FC initiators with active sessions +func countActiveSessionsInitiators(host gopowerstore.Host) int { + fcInitiatorsWithActiveSessionCount := 0 + for _, initiator := range host.Initiators { + if len(initiator.ActiveSessions) != 0 { + fcInitiatorsWithActiveSessionCount++ + } + } + return fcInitiatorsWithActiveSessionCount +} + func (s *Service) updateNodeID() error { if s.nodeID == "" { hostID, err := s.Fs.ReadFile(s.opts.NodeIDFilePath) if err != nil { - log.WithFields(log.Fields{ + log.WithFields(csmlog.Fields{ "path": s.opts.NodeIDFilePath, "error": err, }).Error("Could not read Node ID file") @@ -1261,9 +1717,23 @@ func (s *Service) updateNodeID() error { if defaultArray == nil { return status.Errorf(codes.FailedPrecondition, "Could not fetch default PowerStore array") } - ip, err := getOutboundIP(s.DefaultArray().GetIP(), s.Fs) + // we will chop off port from the host if present. + port, err := ExtractPort(defaultArray.Endpoint) + ip, err := getOutboundIP(defaultArray.GetIP(), port, s.Fs) + log.Debugf("Outbound IP address: %s", ip.String()) + + // When Authorization v2 is enabled the host IP address will be localhost. We should get the actual IP else volume will not mount + if ip.String() == "127.0.0.1" || ip.String() == "localhost" { + log.Debug("Detected localhost IP address, trying to get node IP address") + ip, err = helpers.GetNodeIP() + if err != nil { + return status.Errorf(codes.FailedPrecondition, "Could not get node IP address: %s", err.Error()) + } + } + + log.Debugf("Outbound IP address after check: %s", ip.String()) if err != nil { - log.WithFields(log.Fields{ + log.WithFields(csmlog.Fields{ "endpoint": s.DefaultArray().GetIP(), "error": err, }).Error("Could not connect to PowerStore array") @@ -1276,7 +1746,7 @@ func (s *Service) updateNodeID() error { if len(nodeID) > powerStoreMaxNodeNameLength { err := errors.New("node name prefix is too long") - log.WithFields(log.Fields{ + log.WithFields(csmlog.Fields{ "value": s.opts.NodeNamePrefix, "error": err, }).Error("Invalid Node ID") @@ -1288,11 +1758,10 @@ func (s *Service) updateNodeID() error { } func (s *Service) getInitiators() ([]string, []string, []string, error) { - ctx := context.Background() - var iscsiAvailable bool var fcAvailable bool var nvmeAvailable bool + ctx := context.Background() iscsiInitiators, err := s.iscsiConnector.GetInitiatorName(ctx) if err != nil { @@ -1335,6 +1804,7 @@ func (s *Service) getInitiators() ([]string, []string, []string, error) { func (s *Service) getNodeFCPorts(ctx context.Context) ([]string, error) { var err error var initiators []string + log := log.WithContext(ctx) defer func() { initiators := initiators @@ -1402,7 +1872,7 @@ func (s *Service) readFCPortsFilterFile() ([]string, error) { return result, nil } -func (s *Service) setupHost(initiators []string, client gopowerstore.Client, arrayIP string) error { +func (s *Service) setupHost(initiators []string, client gopowerstore.Client, arrayIP, arrayID string) error { log.Infof("setting up host on %s", arrayIP) defer log.Infof("finished setting up host on %s", arrayIP) @@ -1410,94 +1880,79 @@ func (s *Service) setupHost(initiators []string, client gopowerstore.Client, arr return fmt.Errorf("nodeID not set") } - reqInitiators := s.buildInitiatorsArray(initiators) - var host *gopowerstore.Host - updateCHAP := false - - h, err := client.GetHostByName(context.Background(), s.nodeID) - if err == nil { - err := s.updateHost(context.Background(), initiators, client, h) - if err != nil { - return err - } - if s.opts.EnableCHAP && len(h.Initiators) > 0 && (h.Initiators[0].ChapSingleUsername == "" || h.Initiators[0].ChapSingleUsername == "admin") { - log.Debug("CHAP was enabled earlier, modifying credentials") - err := s.modifyHostInitiators(context.Background(), h.ID, client, nil, nil, initiators) - if err != nil { - return fmt.Errorf("can't modify initiators CHAP credentials %s", err.Error()) - } - } - - s.initialized = true - return nil + if s.useNVME[arrayID] { + s.checkForDuplicateUUIDs() } + reqInitiators := s.buildInitiatorsArray(initiators, arrayID) + var existingHost *gopowerstore.Host + hosts, err := client.GetHosts(context.Background()) if err != nil { - log.Error(err.Error()) - return err + return fmt.Errorf("failed getting hosts on %s", arrayIP) } - for i, h := range hosts { - found := false - for _, hI := range h.Initiators { + for i := range hosts { + for _, hI := range hosts[i].Initiators { for _, rI := range reqInitiators { if hI.PortName == *rI.PortName && hI.PortType == *rI.PortType { - log.Info("Found existing host ", h.Name, hI.PortName, hI.PortType) - updateCHAP = s.opts.EnableCHAP && (hI.ChapSingleUsername == "" || hI.ChapSingleUsername == "admin") - found = true + existingHost = &hosts[i] break } } - if found { + if existingHost != nil { break } } - if found { - host = &hosts[i] + if existingHost != nil { break } } - if host == nil { - // register node on PowerStore - _, err := s.createHost(context.Background(), initiators, client) + if existingHost == nil { + log.Infof("Creating host %s on array %s", s.nodeID, arrayID) + _, err := s.createHost(context.Background(), initiators) if err != nil { - log.Error(err.Error()) return err } } else { - // node already registered - if updateCHAP { // add CHAP credentials if they aren't available - err := s.modifyHostInitiators(context.Background(), host.ID, client, nil, nil, initiators) + log.Infof("Host with initiator already exists. Updating metadata or CHAP if needed.") + if s.opts.EnableCHAP { + err := s.modifyHostInitiators(context.Background(), existingHost.ID, client, nil, nil, initiators, arrayID, &existingHost.HostConnectivity) if err != nil { - return fmt.Errorf("can't modify initiators CHAP credentials %s", err.Error()) + return fmt.Errorf("failed to update CHAP: %v", err) } } - ip, err := getOutboundIP(arrayIP, s.Fs) - if err != nil { - log.WithFields(log.Fields{ - "endpoint": arrayIP, - "error": err, - }).Error("Could not connect to PowerStore array") - return status.Errorf(codes.FailedPrecondition, "couldn't connect to PowerStore array: %s", err.Error()) + if s.nodeID != existingHost.ID { + err := s.modifyHostName(context.Background(), client, s.nodeID, existingHost.ID) + if err != nil { + return fmt.Errorf("failed to update host name: %v", err) + } } - - s.nodeID = host.Name + "-" + ip.String() - s.reusedHost = true } s.initialized = true + return nil +} +func (s *Service) modifyHostName(ctx context.Context, client gopowerstore.Client, nodeID string, hostID string) error { + log := log.WithContext(ctx) + modifyParams := gopowerstore.HostModify{} + modifyParams.Name = &nodeID + _, err := client.ModifyHost(ctx, &modifyParams, hostID) + if err != nil { + return err + } + log.Infof("Updated nodeID %s", nodeID) return nil } -func (s *Service) buildInitiatorsArray(initiators []string) []gopowerstore.InitiatorCreateModify { +func (s *Service) buildInitiatorsArray(initiators []string, arrayID string) []gopowerstore.InitiatorCreateModify { var portType gopowerstore.InitiatorProtocolTypeEnum - if s.useNVME { + if s.useNVME[arrayID] { portType = gopowerstore.InitiatorProtocolTypeEnumNVME - } else if s.useFC { + } else if s.useFC[arrayID] { portType = gopowerstore.InitiatorProtocolTypeEnumFC } else { portType = gopowerstore.InitiatorProtocolTypeEnumISCSI @@ -1505,7 +1960,7 @@ func (s *Service) buildInitiatorsArray(initiators []string) []gopowerstore.Initi initiatorsReq := make([]gopowerstore.InitiatorCreateModify, len(initiators)) for i, iqn := range initiators { iqn := iqn - if !s.useFC && s.opts.EnableCHAP { + if !s.useFC[arrayID] && s.opts.EnableCHAP { initiatorsReq[i] = gopowerstore.InitiatorCreateModify{ ChapSinglePassword: &s.opts.CHAPPassword, ChapSingleUsername: &s.opts.CHAPUsername, @@ -1523,82 +1978,628 @@ func (s *Service) buildInitiatorsArray(initiators []string) []gopowerstore.Initi } // create or update host on PowerStore array -func (s *Service) updateHost(ctx context.Context, initiators []string, client gopowerstore.Client, host gopowerstore.Host) (err error) { +func (s *Service) updateHost(ctx context.Context, initiators []string, client gopowerstore.Client, host gopowerstore.Host, arrayID string, connectivity *gopowerstore.HostConnectivityEnum) error { initiatorsToAdd, initiatorsToDelete := checkIQNS(initiators, host) - return s.modifyHostInitiators(ctx, host.ID, client, initiatorsToAdd, initiatorsToDelete, nil) + return s.modifyHostInitiators(ctx, host.ID, client, initiatorsToAdd, initiatorsToDelete, nil, arrayID, connectivity) +} + +var ( + getArrayfn = func(s *Service) map[string]*array.PowerStoreArray { + return s.Arrays() + } + + getIsHostAlreadyRegistered = func(s *Service, ctx context.Context, client gopowerstore.Client, initiators []string) bool { + return s.isHostAlreadyRegistered(ctx, client, initiators) + } + + getAllRemoteSystemsFunc = func(arr *array.PowerStoreArray, ctx context.Context) ([]gopowerstore.RemoteSystem, error) { + return arr.GetClient().GetAllRemoteSystems(ctx) + } + + getIsRemoteToOtherArray = func(s *Service, ctx context.Context, arr, remoteArr *array.PowerStoreArray) bool { + return s.isRemoteToOtherArray(ctx, arr, remoteArr) + } + + registerHostFunc = func(s *Service, ctx context.Context, client gopowerstore.Client, arrayID string, initiators []string, connType gopowerstore.HostConnectivityEnum) error { + return s.registerHost(ctx, client, arrayID, initiators, connType) + } +) + +func (s *Service) createHost(ctx context.Context, initiators []string) (string, error) { + hostConnectivity := false + metroTopology := false + for _, arr := range getArrayfn(s) { + if arr.MetroTopology != "" { + if hostConnectivity { + return "", fmt.Errorf("host connectivity and metro topology cannot be set at the same time") + } + metroTopology = true + } + if arr.HostConnectivity != nil { + if metroTopology { + return "", fmt.Errorf("host connectivity and metro topology cannot be set at the same time") + } + hostConnectivity = true + } + } + if hostConnectivity { + return s.createHostHostConnectivity(ctx, initiators) + } + return s.createHostMetroTopologyAndLocal(ctx, initiators) } // register host -func (s *Service) createHost(ctx context.Context, initiators []string, client gopowerstore.Client) (id string, err error) { - osType := gopowerstore.OSTypeEnumLinux - reqInitiators := s.buildInitiatorsArray(initiators) - description := fmt.Sprintf("k8s node: %s", s.opts.KubeNodeName) +func (s *Service) createHostHostConnectivity(ctx context.Context, initiators []string) (string, error) { + log := log.WithContext(ctx) + node, err := k8sutils.Kubeclient.GetNode(context.Background(), s.opts.KubeNodeName) + if err != nil { + return "", fmt.Errorf("[createHost] Failed to get node %s: %v", s.opts.KubeNodeName, err) + } + var primaryArrayID string + + for _, arr := range getArrayfn(s) { + var conn gopowerstore.HostConnectivityEnum + log.Infof("[createHost] Processing array %s (%s)", arr.GlobalID, arr.IP) + // 1) Skip if already registered + if getIsHostAlreadyRegistered(s, ctx, arr.GetClient(), initiators) { + log.Infof("[createHost] Already registered on %s, skipping", arr.GlobalID) + if primaryArrayID == "" { + primaryArrayID = arr.GlobalID + } + continue + } + // 2) Metro vs Non‑Metro + log.Debugf("[createHost] Processing array %s (%d)(%v)", arr.GlobalID, arr.HostConnectivity.Local.Size(), &arr.HostConnectivity.Metro) + if arr.HostConnectivity.Local.Size() > 0 { + match, err := nodeMatchSelector(node, &arr.HostConnectivity.Local, conn) + if err != nil { + return "", fmt.Errorf("[createHost] Error matching host connectivity selector for array %s: %v", arr.GlobalID, err) + } + if match { + log.Infof("[createHost] Zone match on %s, registering host locally", arr.GlobalID) + conn = gopowerstore.HostConnectivityEnumLocalOnly + } + } + + // 3) Metro: match zone with array topology + match, err := nodeMatchSelector(node, &arr.HostConnectivity.Metro.ColocatedLocal, conn) + if err != nil { + return "", fmt.Errorf("[createHost] Error matching host connectivity selector for array %s: %v", arr.GlobalID, err) + } + if match { + log.Infof("[createHost] Metro & label match on %s, registering as ColocatedLocal", arr.GlobalID) + conn = gopowerstore.HostConnectivityEnumMetroOptimizeLocal + } + + match, err = nodeMatchSelector(node, &arr.HostConnectivity.Metro.ColocatedRemote, conn) + if err != nil { + return "", fmt.Errorf("[createHost] Error matching host connectivity selector for array %s: %v", arr.GlobalID, err) + } + if match { + log.Infof("[createHost] Metro & label match on %s, registering as ColocatedRemote", arr.GlobalID) + conn = gopowerstore.HostConnectivityEnumMetroOptimizeRemote + } + + match, err = nodeMatchSelector(node, &arr.HostConnectivity.Metro.ColocatedBoth, conn) + if err != nil { + return "", fmt.Errorf("[createHost] Error matching host connectivity selector for array %s: %v", arr.GlobalID, err) + } + if match { + log.Infof("[createHost] Metro & label match on %s, registering as ColocatedBoth", arr.GlobalID) + conn = gopowerstore.HostConnectivityEnumMetroOptimizeBoth + } + if conn == "" { + log.Infof("[createHost] Metro & label mismatch on %s, skip registration for this host", arr.GlobalID) + continue + } + if err := registerHostFunc(s, ctx, arr.GetClient(), arr.GlobalID, initiators, conn); err != nil { + return "", fmt.Errorf("[createHost] Failed to register host with connectivity %s on %s: %v", conn, arr.GlobalID, err) + } + if primaryArrayID == "" { + primaryArrayID = arr.GlobalID + } + } + + if primaryArrayID != "" { + log.Infof("[createHost] Success. Primary array: %s", primaryArrayID) + return primaryArrayID, nil + } + return "", fmt.Errorf("[createHost] Failed to register host on any array") +} + +func nodeMatchSelector(node *corev1.Node, selector *corev1.NodeSelector, conn gopowerstore.HostConnectivityEnum) (bool, error) { + runtimeSelector, err := nodeaffinity.NewNodeSelector(selector) + if err != nil { + return false, err + } + if runtimeSelector.Match(node) { + if conn != "" { + return false, fmt.Errorf("match expressions should be mutual exclusive, a duplicated match found") + } + return true, nil + } + return false, nil +} + +// register host +// For backwards compatibility to use array metro topology +func (s *Service) createHostMetroTopologyAndLocal( + ctx context.Context, + initiators []string, +) (string, error) { + log := log.WithContext(ctx) + nodeLabels, err := k8sutils.Kubeclient.GetNodeLabels(context.Background(), s.opts.KubeNodeName) + if err != nil { + return "", fmt.Errorf("failed to get node labels for node %s: %v", s.opts.KubeNodeName, err) + } + var primaryArrayID string + + // Step 1: Check if this node matches at least one labeled Metro array + anyLabelMatch := false + for _, arr := range getArrayfn(s) { + if strings.ToLower(arr.MetroTopology) == "uniform" && len(arr.Labels) == 1 { + if labelsMatch(arr.Labels, nodeLabels) { + anyLabelMatch = true + break + } + } + } + + for _, arr := range getArrayfn(s) { + log.Infof("[createHost] Processing array %s (%s)", arr.GlobalID, arr.IP) + // 1) Skip if already registered + if getIsHostAlreadyRegistered(s, ctx, arr.GetClient(), initiators) { + log.Infof("[createHost] Already registered on %s, skipping", arr.GlobalID) + if primaryArrayID == "" { + primaryArrayID = arr.GlobalID + } + continue + } + + // 2) Metro vs Non‑Metro + if strings.ToLower(arr.MetroTopology) != "uniform" { + log.Infof("[createHost] Non‑Metro array %s → registering LocalOnly", arr.GlobalID) + if err := s.registerHost( + ctx, arr.GetClient(), arr.GlobalID, initiators, + gopowerstore.HostConnectivityEnumLocalOnly, + ); err != nil { + return "", fmt.Errorf("failed LocalOnly on %s: %v", arr.GlobalID, err) + } + if primaryArrayID == "" { + primaryArrayID = arr.GlobalID + } + continue + } - var createParams gopowerstore.HostCreate - defaultHeaders := client.GetCustomHTTPHeaders() - if defaultHeaders == nil { - defaultHeaders = make(http.Header) - } - customHeaders := defaultHeaders - k8sMetadataSupported := common.IsK8sMetadataSupported(client) - if k8sMetadataSupported { - customHeaders.Add("DELL-VISIBILITY", "internal") - client.SetCustomHTTPHeaders(customHeaders) - - if s.opts.KubeNodeName == "" { - log.Warnf("KubeNodeName value is not set") - createParams = gopowerstore.HostCreate{Name: &s.nodeID, OsType: &osType, Initiators: &reqInitiators, - Description: &description} + if len(arr.Labels) > 1 { + log.Warnf("[createHost] Skipping Metro array %s: more than one label", arr.GlobalID) + continue + } + + // 4) Skip Metro arrays if this node doesn’t match any Metro array label + if !anyLabelMatch { + log.Warnf("[createHost] Node does not match any Metro array labels — skipping Metro registration for %s", arr.GlobalID) + continue + } + + // 5) Metro: now dispatch to label-match vs no-label-match + arrayAddedList := make(map[string]bool) + + if labelsMatch(arr.Labels, nodeLabels) { + // 4a) Labels match + log.Infof("[createHost] Metro & label match on %s", arr.GlobalID) + coLocated, err := s.handleLabelMatchRegistration(ctx, arr, initiators, nodeLabels, arrayAddedList) + if err != nil { + return "", err + } + conn := gopowerstore.HostConnectivityEnumMetroOptimizeLocal + if coLocated { + conn = gopowerstore.HostConnectivityEnumMetroOptimizeBoth + } + log.Infof("[createHost] Registering %s as %s", arr.GlobalID, conn) + if err := registerHostFunc(s, ctx, arr.GetClient(), arr.GlobalID, initiators, conn); err != nil { + return "", fmt.Errorf("failed on %s: %v", arr.GlobalID, err) + } + if primaryArrayID == "" { + primaryArrayID = arr.GlobalID + } } else { - metadata := map[string]string{ - "k8s_node_name": s.opts.KubeNodeName, + // 4b) Labels don’t match + log.Infof("[createHost] Metro & no label match on %s", arr.GlobalID) + coLocated, err := s.handleNoLabelMatchRegistration( + ctx, arr, initiators, nodeLabels, arrayAddedList, + ) + if err != nil { + return "", err + } + conn := gopowerstore.HostConnectivityEnumMetroOptimizeRemote + if coLocated { + conn = gopowerstore.HostConnectivityEnumMetroOptimizeBoth + } + + log.Infof("[createHost] Registering %s as %s", arr.GlobalID, conn) + if err := s.registerHost( + ctx, arr.GetClient(), arr.GlobalID, initiators, conn, + ); err != nil { + return "", fmt.Errorf("failed on %s: %v", arr.GlobalID, err) + } + if primaryArrayID == "" { + primaryArrayID = arr.GlobalID + } + } + } + + if primaryArrayID != "" { + log.Infof("[createHost] Success. Primary array: %s", primaryArrayID) + return primaryArrayID, nil + } + return "", fmt.Errorf("[createHost] Failed to register host on any array") +} + +func (s *Service) handleLabelMatchRegistration(ctx context.Context, arr *array.PowerStoreArray, initiators []string, nodeLabels map[string]string, arrayAddedList map[string]bool, +) (bool, error) { + log := log.WithContext(ctx) + // Early exit if no array labels match the node labels + anyLabelMatch := false + for _, configuredArr := range getArrayfn(s) { + if labelsMatch(configuredArr.Labels, nodeLabels) { + anyLabelMatch = true + break + } + } + if !anyLabelMatch { + log.Infof("[handleLabelMatch] No arrays match node labels — skipping registration") + return false, nil + } + + remoteSystems, err := getAllRemoteSystemsFunc(arr, ctx) + if err != nil { + log.Warnf("[handleLabelMatch] failed to get remotes for %s: %v", arr.GlobalID, err) + return false, err + } + + coLocated := false + + // 1) Determine this array's connectivity based on its label vs. the node's labels + var arrayConn gopowerstore.HostConnectivityEnum + if labelsMatch(arr.Labels, nodeLabels) { + arrayConn = gopowerstore.HostConnectivityEnumMetroOptimizeLocal + } else { + arrayConn = gopowerstore.HostConnectivityEnumMetroOptimizeRemote + } + + for _, remote := range remoteSystems { + if remote.Name == "" || arrayAddedList[remote.SerialNumber] { + continue + } + + for _, remoteArr := range getArrayfn(s) { + if remoteArr.GlobalID != remote.SerialNumber { + continue + } + if len(remoteArr.Labels) > 1 { + return false, fmt.Errorf("skipping remote array %s – more than one label", remoteArr.GlobalID) + } + + // 2) Mutual-remote check + if !getIsRemoteToOtherArray(s, ctx, arr, remoteArr) { + log.Infof("[handleLabelMatch] skipping %s↔%s: not mutually remote", + arr.GlobalID, remoteArr.GlobalID) + continue } - createParams = gopowerstore.HostCreate{Name: &s.nodeID, OsType: &osType, Initiators: &reqInitiators, - Description: &description, Metadata: &metadata} + + clientB := remoteArr.GetClient() + if getIsHostAlreadyRegistered(s, ctx, clientB, initiators) { + arrayAddedList[remoteArr.GlobalID] = true + continue + } + + if !labelsMatch(remoteArr.Labels, arr.Labels) && labelsMatch(remoteArr.Labels, nodeLabels) && labelsMatch(arr.Labels, nodeLabels) { + log.Info("skipping registration as the node is having all the array labels") + return false, fmt.Errorf("skipping registration as the node matching all the array labels node label: %s, arr label: %s, remote label: %s", nodeLabels, arr.Labels, remoteArr.Labels) + } + + // 3) Determine remote's connectivity + var remoteConn gopowerstore.HostConnectivityEnum + if labelsMatch(remoteArr.Labels, arr.Labels) { + remoteConn = gopowerstore.HostConnectivityEnumMetroOptimizeBoth + } else if labelsMatch(remoteArr.Labels, nodeLabels) { + remoteConn = gopowerstore.HostConnectivityEnumMetroOptimizeLocal + } else { + remoteConn = gopowerstore.HostConnectivityEnumMetroOptimizeRemote + } + + // 4) Guard: skip if both would end up with the same non‑Both connectivity + if arrayConn == remoteConn && remoteConn != gopowerstore.HostConnectivityEnumMetroOptimizeBoth { + log.Infof("[handleLabelMatch] skipping %s: both arrays would be %s", + arr.GlobalID, arrayConn) + continue + } + + // 5) Register + if remoteConn == gopowerstore.HostConnectivityEnumMetroOptimizeBoth { + log.Infof("[handleLabelMatch] Full match → MetroOptimizeBoth on %s", remoteArr.GlobalID) + } else { + log.Infof("[handleLabelMatch] Partial match → MetroOptimizeRemote on %s", remoteArr.GlobalID) + } + if err := registerHostFunc(s, ctx, clientB, remoteArr.GlobalID, initiators, remoteConn); err != nil { + return false, err + } + arrayAddedList[remoteArr.GlobalID] = true + + if remoteConn == gopowerstore.HostConnectivityEnumMetroOptimizeBoth { + coLocated = true + } + } + } + return coLocated, nil +} + +func (s *Service) handleNoLabelMatchRegistration( + ctx context.Context, + arr *array.PowerStoreArray, + initiators []string, + nodeLabels map[string]string, + arrayAddedList map[string]bool, +) (bool, error) { + log := log.WithContext(ctx) + // Early exit if no array labels match the node labels + anyLabelMatch := false + for _, configuredArr := range getArrayfn(s) { + if labelsMatch(configuredArr.Labels, nodeLabels) { + anyLabelMatch = true + break } + } + if !anyLabelMatch { + log.Infof("[handleNoLabelMatch] No arrays match node labels — skipping registration for %s", arr.GlobalID) + return false, nil + } + + remoteSystems, err := getAllRemoteSystemsFunc(arr, ctx) + if err != nil { + log.Warnf("[handleNoLabelMatch] failed to get remotes for %s: %v", arr.GlobalID, err) + return false, err + } + + coLocated := false + // 1) Determine this array's connectivity based on its label vs. the node's labels + var arrayConn gopowerstore.HostConnectivityEnum + if labelsMatch(arr.Labels, nodeLabels) { + arrayConn = gopowerstore.HostConnectivityEnumMetroOptimizeLocal } else { - createParams = gopowerstore.HostCreate{Name: &s.nodeID, OsType: &osType, Initiators: &reqInitiators, - Description: &description} + arrayConn = gopowerstore.HostConnectivityEnumMetroOptimizeRemote + } + + for _, remote := range remoteSystems { + if remote.Name == "" || arrayAddedList[remote.SerialNumber] { + continue + } + + for _, remoteArr := range getArrayfn(s) { + if remoteArr.GlobalID != remote.SerialNumber { + continue + } + // Mutual remote check + if !getIsRemoteToOtherArray(s, ctx, arr, remoteArr) { + log.Infof("[handleNoLabelMatch] skipping %s↔%s: not mutually remote", + arr.GlobalID, remoteArr.GlobalID) + continue + } + + clientB := remoteArr.GetClient() + if getIsHostAlreadyRegistered(s, ctx, clientB, initiators) { + arrayAddedList[remoteArr.GlobalID] = true + continue + } + + if !labelsMatch(remoteArr.Labels, arr.Labels) && labelsMatch(remoteArr.Labels, nodeLabels) && labelsMatch(arr.Labels, nodeLabels) { + log.Info("skipping registration as the node is having all the array labels") + return false, fmt.Errorf("skipping registration as the node matching all the array labels node label: %s, arr label: %s, remote label: %s", nodeLabels, arr.Labels, remoteArr.Labels) + } + + // Determine remote's connectivity + var remoteConn gopowerstore.HostConnectivityEnum + if labelsMatch(remoteArr.Labels, arr.Labels) { + remoteConn = gopowerstore.HostConnectivityEnumMetroOptimizeBoth + } else if labelsMatch(remoteArr.Labels, nodeLabels) { + remoteConn = gopowerstore.HostConnectivityEnumMetroOptimizeLocal + } else { + remoteConn = gopowerstore.HostConnectivityEnumMetroOptimizeRemote + } + + // Guard: skip if both would end up with the same non-Both connectivity + if arrayConn == remoteConn && remoteConn != gopowerstore.HostConnectivityEnumMetroOptimizeBoth { + log.Infof("[handleNoLabelMatch] skipping %s: both arrays would be %s", + arr.GlobalID, arrayConn) + continue + } + + // Register + if remoteConn == gopowerstore.HostConnectivityEnumMetroOptimizeBoth { + log.Infof("[handleNoLabelMatch] Full match → MetroOptimizeBoth on %s", remoteArr.GlobalID) + } else { + log.Infof("[handleNoLabelMatch] Partial match → MetroOptimizeRemote on %s", remoteArr.GlobalID) + } + if err := registerHostFunc(s, ctx, clientB, remoteArr.GlobalID, initiators, remoteConn); err != nil { + return false, err + } + arrayAddedList[remoteArr.GlobalID] = true + + if remoteConn == gopowerstore.HostConnectivityEnumMetroOptimizeBoth { + coLocated = true + } + } + } + return coLocated, nil +} + +// isRemoteToOtherArray returns true if arrA and arrB are mutually remote to each other. +func (s *Service) isRemoteToOtherArray( + ctx context.Context, + arrA, arrB *array.PowerStoreArray, +) bool { + log := log.WithContext(ctx) + // fetch arrA’s remotes + remotesA, err := getAllRemoteSystemsFunc(arrA, ctx) + if err != nil { + log.Warnf("[isRemoteToOtherArray] failed to get remotes for %s: %v", arrA.GlobalID, err) + return false + } + // fetch arrB’s remotes + remotesB, err := getAllRemoteSystemsFunc(arrB, ctx) + if err != nil { + log.Warnf("[isRemoteToOtherArray] failed to get remotes for %s: %v", arrB.GlobalID, err) + return false + } + + foundAtoB := false + for _, r := range remotesA { + if r.SerialNumber == arrB.GlobalID { + foundAtoB = true + break + } + } + if !foundAtoB { + return false + } + + for _, r := range remotesB { + if r.SerialNumber == arrA.GlobalID { + return true + } + } + return false +} + +// Checks if host with given initiators already exists +func (s *Service) isHostAlreadyRegistered(ctx context.Context, client gopowerstore.Client, initiators []string) bool { + log := log.WithContext(ctx) + existingHosts, err := client.GetHosts(ctx) + if err != nil { + log.Warnf("[isHostAlreadyRegistered] Failed to get hosts: %v", err) + return false + } + + for _, host := range existingHosts { + for _, hInit := range host.Initiators { + for _, i := range initiators { + if hInit.PortName == i { + log.Infof("[isHostAlreadyRegistered] Found existing host with initiator %s", i) + return true + } + } + } + } + return false +} + +func (s *Service) registerHost( + ctx context.Context, + client gopowerstore.Client, + arrayID string, + initiators []string, + connType gopowerstore.HostConnectivityEnum, +) error { + log := log.WithContext(ctx) + description := fmt.Sprintf("k8s node: %s", s.opts.KubeNodeName) + reqInitiators := s.buildInitiatorsArray(initiators, arrayID) + osType := gopowerstore.OSTypeEnumLinux + + createParams := gopowerstore.HostCreate{ + Name: &s.nodeID, + OsType: &osType, + Initiators: &reqInitiators, + Description: &description, + HostConnectivity: connType, + } + + if s.opts.KubeNodeName != "" && identifiers.IsK8sMetadataSupported(client) { + metadata := map[string]string{"k8s_node_name": s.opts.KubeNodeName} + createParams.Metadata = &metadata + + headers := client.GetCustomHTTPHeaders() + if headers == nil { + headers = api.NewSafeHeader().GetHeader() + } + headers.Add("DELL-VISIBILITY", "internal") + client.SetCustomHTTPHeaders(headers) } + + log.Infof("[registerHost] Creating host on array %s with connectivity: %s", arrayID, connType) resp, err := client.CreateHost(ctx, &createParams) - // reset custom header - customHeaders.Del("DELL-VISIBILITY") - client.SetCustomHTTPHeaders(customHeaders) + client.SetCustomHTTPHeaders(nil) + if err != nil { - return id, err + if connType == gopowerstore.HostConnectivityEnumMetroOptimizeRemote && + strings.Contains(err.Error(), "already registered with another host") { + log.Warnf("[registerHost] Skipping array %s due to duplicate initiator error (remote host): %v", arrayID, err) + return nil + } + log.Errorf("[registerHost] Failed to create host on array %s: %v", arrayID, err) + return err + } + log.Infof("[registerHost] Host successfully registered on array %s with ID: %s", arrayID, resp.ID) + return nil +} + +func labelsMatch(arrayLabels, nodeLabels map[string]string) bool { + for key, val := range arrayLabels { + if nodeVal, exists := nodeLabels[key]; !exists || nodeVal != val { + return false + } } - return resp.ID, err + return true } // add or remove initiators from host func (s *Service) modifyHostInitiators(ctx context.Context, hostID string, client gopowerstore.Client, - initiatorsToAdd []string, initiatorsToDelete []string, initiatorsToModify []string) error { + initiatorsToAdd []string, initiatorsToDelete []string, initiatorsToModify []string, arrayID string, connectivity *gopowerstore.HostConnectivityEnum, +) error { + log := log.WithContext(ctx) if len(initiatorsToDelete) > 0 { - modifyParams := gopowerstore.HostModify{} - modifyParams.RemoveInitiators = &initiatorsToDelete + modifyParams := gopowerstore.HostModify{RemoveInitiators: &initiatorsToDelete} _, err := client.ModifyHost(ctx, &modifyParams, hostID) if err != nil { - return err + return fmt.Errorf("failed to remove initiators: %w", err) } } + if len(initiatorsToAdd) > 0 { modifyParams := gopowerstore.HostModify{} - initiators := s.buildInitiatorsArray(initiatorsToAdd) + initiators := s.buildInitiatorsArray(initiatorsToAdd, arrayID) modifyParams.AddInitiators = &initiators + _, err := client.ModifyHost(ctx, &modifyParams, hostID) if err != nil { - return err + if strings.Contains(err.Error(), "already registered with another host") { + log.Warnf("Skipping duplicate initiator registration: %v", err) + } else { + return fmt.Errorf("failed to add initiators: %w", err) + } } } + if len(initiatorsToModify) > 0 { modifyParams := gopowerstore.HostModify{} - initiators := s.buildInitiatorsArrayModify(initiatorsToModify) + initiators := s.buildInitiatorsArrayModify(initiatorsToModify, arrayID) modifyParams.ModifyInitiators = &initiators + _, err := client.ModifyHost(ctx, &modifyParams, hostID) if err != nil { - return err + return fmt.Errorf("failed to modify initiators: %w", err) } } + + // Ensure HostConnectivity is updated only if needed + if connectivity != nil { + modifyParams := gopowerstore.HostModify{HostConnectivity: *connectivity} + _, err := client.ModifyHost(ctx, &modifyParams, hostID) + if err != nil { + return fmt.Errorf("failed to update host connectivity for %s: %w", hostID, err) + } + } + return nil } @@ -1626,14 +2627,14 @@ func checkIQNS(IQNs []string, host gopowerstore.Host) (iqnToAdd, iqnToDelete []s iqnToDelete = append(iqnToDelete, iqn) } } - return + return iqnToAdd, iqnToDelete } -func (s *Service) buildInitiatorsArrayModify(initiators []string) []gopowerstore.UpdateInitiatorInHost { +func (s *Service) buildInitiatorsArrayModify(initiators []string, arrayID string) []gopowerstore.UpdateInitiatorInHost { initiatorsReq := make([]gopowerstore.UpdateInitiatorInHost, len(initiators)) for i, iqn := range initiators { iqn := iqn - if !s.useFC && s.opts.EnableCHAP { + if !s.useFC[arrayID] && s.opts.EnableCHAP { initiatorsReq[i] = gopowerstore.UpdateInitiatorInHost{ ChapSinglePassword: &s.opts.CHAPPassword, ChapSingleUsername: &s.opts.CHAPUsername, @@ -1650,19 +2651,124 @@ func (s *Service) buildInitiatorsArrayModify(initiators []string) []gopowerstore func (s *Service) fileExists(filename string) bool { _, err := s.Fs.Stat(filename) + logFields := csmlog.Fields{ + "filename": filename, + "error": err, + } + log := log.WithFields(logFields) if err == nil { return true } if os.IsNotExist(err) { - log.WithFields(log.Fields{ - "filename": filename, - "error": err, - }).Error("File doesn't exist") + log.Error("File does not exist") } else { - log.WithFields(log.Fields{ - "filename": filename, - "error": err, - }).Error("Error while checking stat of file") + log.Error("Error while checking stat of the file") } return false } + +// splitIPAddress function takes a string in the format "hostname:port" +// and returns a slice containing the hostname and port. +func splitIPAddress(address string) []string { + return strings.Split(address, ":") +} + +// ExtractPort extracts the port from a URL. +func ExtractPort(urlString string) (string, error) { + u, err := url.Parse(urlString) + if err != nil { + return "", err + } + + port := u.Port() + if port == "" { + return "", errors.New("port not specified in URL") + } + + return port, nil +} + +// updateMetroToplogy updates the metro topology in the response +func updateMetroToplogy(arr *array.PowerStoreArray, nodeLabels map[string]string, resp *csi.NodeGetInfoResponse) { + if arr.HostConnectivity != nil { + labelSet := labels.Set(nodeLabels) + if ok, matchingLabels := metroMatchNodeSelectorTerms(arr.HostConnectivity.Local.NodeSelectorTerms, labelSet); ok { + maps.Copy(resp.AccessibleTopology.Segments, matchingLabels) + } + + if ok, matchingLabels := metroMatchNodeSelectorTerms(arr.HostConnectivity.Metro.ColocatedBoth.NodeSelectorTerms, labelSet); ok { + maps.Copy(resp.AccessibleTopology.Segments, matchingLabels) + } + + if ok, matchingLabels := metroMatchNodeSelectorTerms(arr.HostConnectivity.Metro.ColocatedLocal.NodeSelectorTerms, labelSet); ok { + maps.Copy(resp.AccessibleTopology.Segments, matchingLabels) + } + + if ok, matchingLabels := metroMatchNodeSelectorTerms(arr.HostConnectivity.Metro.ColocatedRemote.NodeSelectorTerms, labelSet); ok { + maps.Copy(resp.AccessibleTopology.Segments, matchingLabels) + } + } +} + +// metroMatchNodeSelectorTerms checks if the metro labels from the secret match the node selector terms +// Returns the matched labels if bool is true, else empty map +func metroMatchNodeSelectorTerms(terms []corev1.NodeSelectorTerm, nodeLabels map[string]string) (bool, map[string]string) { + for _, term := range terms { + matched := true + matchedLabels := make(map[string]string) + + for _, expr := range term.MatchExpressions { + nodeVal, exists := nodeLabels[expr.Key] + switch expr.Operator { + case corev1.NodeSelectorOpIn: + if !exists { + matched = false + break + } + if slices.Contains(expr.Values, nodeVal) { + matchedLabels[expr.Key] = nodeVal + } else { + matched = false + } + + case corev1.NodeSelectorOpNotIn: + if !exists { + continue + } + if slices.Contains(expr.Values, nodeVal) { + matched = false + } + + case corev1.NodeSelectorOpExists: + if exists { + matchedLabels[expr.Key] = nodeVal + } else { + matched = false + } + + case corev1.NodeSelectorOpDoesNotExist: + if exists { + matched = false + } + + default: + matched = false + } + + if !matched { + break + } + } + + // Kubernetes treats nodeSelectorTerms as OR — if one term matches, overall match is true. + if matched { + return true, matchedLabels + } + } + + return false, nil +} + +func isNodeConnectedToArray(ctx context.Context, kubeNodeID string, arr *array.PowerStoreArray) bool { + return arr.CheckConnectivity(ctx, kubeNodeID) +} diff --git a/pkg/node/node_connectivity_checker.go b/pkg/node/node_connectivity_checker.go index 9b32851a..a51a42c0 100644 --- a/pkg/node/node_connectivity_checker.go +++ b/pkg/node/node_connectivity_checker.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2022-2023 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2022-2026 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,12 +28,11 @@ import ( "time" "github.com/dell/csi-powerstore/v2/pkg/array" - "github.com/dell/csi-powerstore/v2/pkg/common" + "github.com/dell/csi-powerstore/v2/pkg/identifiers" "github.com/dell/goiscsi" "github.com/dell/gonvme" "github.com/dell/gopowerstore" "github.com/gorilla/mux" - log "github.com/sirupsen/logrus" ) // pollingFrequency in seconds @@ -44,30 +43,32 @@ var probeStatus *sync.Map // startAPIService reads nodes to array status periodically func (s *Service) startAPIService(ctx context.Context) { + log := log.WithContext(ctx) if !s.isPodmonEnabled { log.Info("podmon is not enabled") return } - pollingFrequencyInSeconds = common.SetPollingFrequency(ctx) + pollingFrequencyInSeconds = identifiers.SetPollingFrequency(ctx) s.startNodeToArrayConnectivityCheck(ctx) s.apiRouter(ctx) } // apiRouter serves http requests func (s *Service) apiRouter(ctx context.Context) { - log.Infof("starting http server on port %s", common.APIPort) + log := log.WithContext(ctx) + log.Infof("starting http server on port %s", identifiers.APIPort) // create a new mux router router := mux.NewRouter() // route to connectivity status // connectivityStatus is the handlers - router.HandleFunc(common.ArrayStatus, connectivityStatus).Methods("GET") - router.HandleFunc(common.ArrayStatus+"/"+"{arrayId}", getArrayConnectivityStatus).Methods("GET") + router.HandleFunc(identifiers.ArrayStatus, connectivityStatus).Methods("GET") + router.HandleFunc(identifiers.ArrayStatus+"/"+"{arrayId}", getArrayConnectivityStatus).Methods("GET") // start http server to serve requests server := &http.Server{ - Addr: common.APIPort, + Addr: identifiers.APIPort, Handler: router, - ReadTimeout: common.Timeout, - WriteTimeout: common.Timeout, + ReadTimeout: identifiers.PodmonArrayConnectivityTimeout, + WriteTimeout: identifiers.PodmonArrayConnectivityTimeout, } err := server.ListenAndServe() if err != nil { @@ -76,7 +77,7 @@ func (s *Service) apiRouter(ctx context.Context) { } // connectivityStatus handler returns array connectivity status -func connectivityStatus(w http.ResponseWriter, r *http.Request) { +func connectivityStatus(w http.ResponseWriter, _ *http.Request) { log.Infof("connectivityStatus called, status is %v \n", probeStatus) // w.Header().Set("Content-Type", "application/json") if probeStatus == nil { @@ -106,12 +107,12 @@ func connectivityStatus(w http.ResponseWriter, r *http.Request) { // MarshalSyncMapToJSON marshal the sync Map to Json func MarshalSyncMapToJSON(m *sync.Map) ([]byte, error) { - tmpMap := make(map[string]common.ArrayConnectivityStatus) + tmpMap := make(map[string]identifiers.ArrayConnectivityStatus) m.Range(func(k, value interface{}) bool { // this check is not necessary but just in case is someone in future play around this switch value.(type) { - case common.ArrayConnectivityStatus: - tmpMap[k.(string)] = value.(common.ArrayConnectivityStatus) + case identifiers.ArrayConnectivityStatus: + tmpMap[k.(string)] = value.(identifiers.ArrayConnectivityStatus) return true default: log.Errorf("invalid data is stored in cache") @@ -138,7 +139,7 @@ func getArrayConnectivityStatus(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, "array %s not found \n", arrayID) return } - //convert status struct to JSON + // convert status struct to JSON jsonResponse, err := json.Marshal(status) if err != nil { log.Errorf("error %s during marshaling to json", err) @@ -147,7 +148,7 @@ func getArrayConnectivityStatus(w http.ResponseWriter, r *http.Request) { return } log.Infof("sending response %+v for array %s \n", status, arrayID) - //update response + // update response _, err = w.Write(jsonResponse) if err != nil { log.Errorf("unable to write response %s", err) @@ -156,6 +157,7 @@ func getArrayConnectivityStatus(w http.ResponseWriter, r *http.Request) { // startNodeToArrayConnectivityCheck starts connectivityTest as one goroutine for each array func (s *Service) startNodeToArrayConnectivityCheck(ctx context.Context) { + log := log.WithContext(ctx) log.Debug("startNodeToArrayConnectivityCheck called") probeStatus = new(sync.Map) // in case if we want to store the status of default array, uncomment below line @@ -164,7 +166,7 @@ func (s *Service) startNodeToArrayConnectivityCheck(ctx context.Context) { for _, array := range powerStoreArray { // start one goroutine for each array, so each array's nodeProbe run concurrently // should we really store the status of all array instead of default one, currently podman query only default array? - go s.testConnectivityAndUpdateStatus(ctx, array, common.Timeout) + go s.testConnectivityAndUpdateStatus(ctx, array, identifiers.PodmonArrayConnectivityTimeout) } log.Infof("startNodeToArrayConnectivityCheck is running probes at pollingFrequency %d ", pollingFrequencyInSeconds/2) } @@ -172,6 +174,7 @@ func (s *Service) startNodeToArrayConnectivityCheck(ctx context.Context) { // testConnectivityAndUpdateStatus runs probe to test connectivity from node to array // updates probeStatus map[array]ArrayConnectivityStatus func (s *Service) testConnectivityAndUpdateStatus(ctx context.Context, array *array.PowerStoreArray, timeout time.Duration) { + log := log.WithContext(ctx) defer func() { if err := recover(); err != nil { log.Errorf("panic occurred in testConnectivityAndUpdateStatus: %s for array having %s", err, array.GlobalID) @@ -179,7 +182,7 @@ func (s *Service) testConnectivityAndUpdateStatus(ctx context.Context, array *ar // if panic occurs restart new goroutine go s.testConnectivityAndUpdateStatus(ctx, array, timeout) }() - var status common.ArrayConnectivityStatus + var status identifiers.ArrayConnectivityStatus for { // add timeout to context timeOutCtx, cancel := context.WithTimeout(ctx, timeout) @@ -187,7 +190,7 @@ func (s *Service) testConnectivityAndUpdateStatus(ctx context.Context, array *ar if existingStatus, ok := probeStatus.Load(array.GlobalID); !ok { log.Debugf("%s not in probeStatus ", array.GlobalID) } else { - if status, ok = existingStatus.(common.ArrayConnectivityStatus); !ok { + if status, ok = existingStatus.(identifiers.ArrayConnectivityStatus); !ok { log.Errorf("failed to extract ArrayConnectivityStatus for array '%s'", array.GlobalID) } } @@ -211,7 +214,8 @@ func (s *Service) testConnectivityAndUpdateStatus(ctx context.Context, array *ar } // nodeProbe function used to store the status of array -func (s *Service) nodeProbe(timeOutCtx context.Context, array *array.PowerStoreArray) error { +func (s *Service) nodeProbe(ctx context.Context, array *array.PowerStoreArray) error { + log := log.WithContext(ctx) // try to get the host host, err := array.Client.GetHostByName(context.Background(), s.nodeID) // possibly NFS could be there. @@ -228,7 +232,7 @@ func (s *Service) nodeProbe(timeOutCtx context.Context, array *array.PowerStoreA log.Debugf("Successfully got Host on %s", array.GlobalID) s.populateTargetsInCache(array) // check if nvme sessions are active - if s.useNVME { + if s.useNVME[array.GlobalID] { log.Debugf("Checking if nvme sessions are active on node or not") sessions, _ := s.nvmeLib.GetSessions() for _, target := range s.nvmeTargets[array.GlobalID] { @@ -247,7 +251,7 @@ func (s *Service) nodeProbe(timeOutCtx context.Context, array *array.PowerStoreA return nil } return fmt.Errorf("no active nvme sessions") - } else if s.useFC { + } else if s.useFC[array.GlobalID] { log.Debugf("Checking if FC sessions are active on node or not") for _, initiator := range host.Initiators { if len(initiator.ActiveSessions) > 0 { @@ -255,41 +259,40 @@ func (s *Service) nodeProbe(timeOutCtx context.Context, array *array.PowerStoreA } } return fmt.Errorf("no active fc sessions") - } else { - // check if iscsi sessions are active - // if !s.useNVME && !s.useFC { - log.Debugf("Checking if iscsi sessions are active on node or not") - sessions, _ := s.iscsiLib.GetSessions() - for _, target := range s.iscsiTargets[array.GlobalID] { - for _, session := range sessions { - log.Debugf("matching %v with %v", target, session) - if session.Target == target && session.ISCSISessionState == goiscsi.ISCSISessionStateLOGGEDIN { - if s.useNFS { - s.useNFS = false - } - return nil + } + // check if iscsi sessions are active + // if !s.useNVME && !s.useFC { + log.Debugf("Checking if iscsi sessions are active on node or not") + sessions, _ := s.iscsiLib.GetSessions() + for _, target := range s.iscsiTargets[array.GlobalID] { + for _, session := range sessions { + log.Debugf("matching %v with %v", target, session) + if session.Target == target && session.ISCSISessionState == goiscsi.ISCSISessionStateLOGGEDIN { + if s.useNFS { + s.useNFS = false } + return nil } } - if s.useNFS { - log.Infof("Host Entry found but failed to login to iscsi target, seems to be this worker has only NFS") - return nil - } - return fmt.Errorf("no active iscsi sessions") } + if s.useNFS { + log.Infof("Host Entry found but failed to login to iscsi target, seems to be this worker has only NFS") + return nil + } + return fmt.Errorf("no active iscsi sessions") } // populateTargetsInCache checks if nvmeTargets or iscsiTargets in cache is empty, try to fetch the targets from array and populate the cache func (s *Service) populateTargetsInCache(array *array.PowerStoreArray) { // if nvmeTargets in cache is empty // this could be empty in 2 cases: Either container is getting restarted or discovery & login has failed in NodeGetInfo - if s.useNVME { + if s.useNVME[array.GlobalID] { if len(s.nvmeTargets[array.GlobalID]) != 0 { return } // for NVMeFC - if s.useFC { - nvmefcInfo, err := common.GetNVMEFCTargetInfoFromStorage(array.GetClient(), "") + if s.useFC[array.GlobalID] { + nvmefcInfo, err := identifiers.GetNVMEFCTargetInfoFromStorage(array.GetClient(), "") if err != nil { log.Errorf("couldn't get targets from the array: %s", err.Error()) return @@ -307,45 +310,70 @@ func (s *Service) populateTargetsInCache(array *array.PowerStoreArray) { break } } else { - infoList, err := common.GetISCSITargetsInfoFromStorage(array.GetClient(), "") + // for NVMeTCP + infoList, err := identifiers.GetNVMETCPTargetsInfoFromStorage(array.GetClient(), "") if err != nil { log.Errorf("couldn't get targets from array: %s", err.Error()) return } + networkIDs := map[string]struct{}{} for _, address := range infoList { - nvmeIP := strings.Split(address.Portal, ":") - log.Info("Trying to discover NVMe target from portal ", nvmeIP[0]) - nvmeTargets, err := s.nvmeLib.DiscoverNVMeTCPTargets(nvmeIP[0], false) + // discovering with one portal returns all targets in the network + // so if we already discovered an address with this network ID, continue + if _, ok := networkIDs[address.NetworkID]; ok { + continue + } + + nvmeIP := strings.Split(address.Portal, ":")[0] + log.Infof("Trying to discover NVMe targets from portal %s on network %s", nvmeIP, address.NetworkID) + nvmeTargets, err := s.nvmeLib.DiscoverNVMeTCPTargets(nvmeIP, false) if err != nil { - log.Error("couldn't discover targets") + log.Errorf("discovering portal: %s: %v", nvmeIP, err) continue } for _, target := range nvmeTargets { otherTargets := s.nvmeTargets[array.GlobalID] s.nvmeTargets[array.GlobalID] = append(otherTargets, target.TargetNqn) } - break + + // mark this network ID as discovered so we don't discover another portal in the same network + // since it will return all the same target information already seen + networkIDs[address.NetworkID] = struct{}{} } } - } else if !s.useFC && !s.useNFS { + } else if !s.useFC[array.GlobalID] && !s.useNFS { // if iscsiTargets in cache is empty if len(s.iscsiTargets[array.GlobalID]) != 0 { return } - infoList, err := common.GetISCSITargetsInfoFromStorage(array.GetClient(), "") + infoList, err := identifiers.GetISCSITargetsInfoFromStorage(array.GetClient(), "") if err != nil { log.Errorf("couldn't get targets from array: %s", err.Error()) return } - + var ipAddress string var iscsiTargets []goiscsi.ISCSITarget + networkIDs := map[string]struct{}{} for _, address := range infoList { + // discovering with one portal returns all targets in the network + // so if we already discovered an address with this network ID, continue + if _, ok := networkIDs[address.NetworkID]; ok { + continue + } + // first check if this portal is reachable from this machine or not if ReachableEndPoint(address.Portal) { + ipAddressList := splitIPAddress(address.Portal) + ipAddress = ipAddressList[0] // doesn't matter how many portals are present, discovering from any one will list out all targets - log.Info("Trying to discover iSCSI target from portal ", address.Portal) - iscsiTargets, err = s.iscsiLib.DiscoverTargets(address.Portal, false) + log.Infof("Trying to discover iSCSI target from portal %s ", ipAddress) + ipInterface, err := s.iscsiLib.GetInterfaceForTargetIP(ipAddress) + if err != nil { + log.Errorf("couldn't get interface: %s", err.Error()) + continue + } + iscsiTargets, err = s.iscsiLib.DiscoverTargetsWithInterface(address.Portal, ipInterface[ipAddress], false) if err != nil { log.Error("couldn't discover targets") continue @@ -354,10 +382,12 @@ func (s *Service) populateTargetsInCache(array *array.PowerStoreArray) { otherTargets := s.iscsiTargets[array.GlobalID] s.iscsiTargets[array.GlobalID] = append(otherTargets, target.Target) } - break - } else { - log.Debugf("Portal %s is not rechable from the node", address.Portal) + + // mark this network ID as discovered so we don't discover another portal in the same network + // since it will return all the same target information already seen + networkIDs[address.NetworkID] = struct{}{} } + log.Debugf("Portal is not rechable from the node") } } diff --git a/pkg/node/node_connectivity_checker_test.go b/pkg/node/node_connectivity_checker_test.go index 11a18ae4..f7e480dc 100644 --- a/pkg/node/node_connectivity_checker_test.go +++ b/pkg/node/node_connectivity_checker_test.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2022-2023 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2022-2026 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,17 +20,20 @@ package node import ( "context" + "errors" "net/http" "sync" "testing" "time" - "github.com/dell/csi-powerstore/v2/pkg/common" + "github.com/dell/csi-powerstore/v2/pkg/identifiers" + "github.com/dell/gopowerstore" + "github.com/stretchr/testify/mock" ) func TestApiRouter2(t *testing.T) { // server should not be up and running - common.APIPort = "abc" + identifiers.APIPort = "abc" setVariables() nodeSvc.apiRouter(context.Background()) @@ -41,7 +44,7 @@ func TestApiRouter2(t *testing.T) { } func TestApiRouter(t *testing.T) { - common.SetAPIPort(context.Background()) + identifiers.SetAPIPort(context.Background()) setVariables() go nodeSvc.apiRouter(context.Background()) time.Sleep(2 * time.Second) @@ -60,7 +63,7 @@ func TestApiRouter(t *testing.T) { } // fill some dummy data in the cache and try to fetch - var status common.ArrayConnectivityStatus + var status identifiers.ArrayConnectivityStatus status.LastSuccess = time.Now().Unix() status.LastAttempt = time.Now().Unix() probeStatus = new(sync.Map) @@ -94,7 +97,7 @@ func TestMarshalSyncMapToJSON(t *testing.T) { } sample := new(sync.Map) sample2 := new(sync.Map) - var status common.ArrayConnectivityStatus + var status identifiers.ArrayConnectivityStatus status.LastSuccess = time.Now().Unix() status.LastAttempt = time.Now().Unix() @@ -118,3 +121,184 @@ func TestMarshalSyncMapToJSON(t *testing.T) { }) } } + +func TestPopulateTargetsInCache(t *testing.T) { + t.Run("PopulateTargetsInCache - iscsiTargets should be populated [iSCSI]", func(t *testing.T) { + setVariables() + + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn"}, + }, + }, nil) + + nodeSvc.populateTargetsInCache(nodeSvc.Arrays()[firstValidIP]) + + if len(nodeSvc.iscsiTargets[firstGlobalID]) != 1 { + t.Errorf("Expected iscsiTargets to be populated") + } + }) + + t.Run("PopulateTargetsInCache - nvmeTargets should be populated [NVMeTCP]", func(t *testing.T) { + setVariables() + nodeSvc.useNVME[firstGlobalID] = true + + clientMock.On("GetCluster", mock.Anything). + Return(gopowerstore.Cluster{Name: validClusterName}, nil) + clientMock.On("GetStorageNVMETCPTargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn"}, + }, + }, nil) + + nodeSvc.populateTargetsInCache(nodeSvc.Arrays()[firstValidIP]) + + if len(nodeSvc.nvmeTargets[firstGlobalID]) != 1 { + t.Errorf("Expected nvmeTargets to be populated") + } + }) + + t.Run("PopulateTargetsInCache - nvmeTargets should be populated in multiple networks [NVMeTCP]", func(t *testing.T) { + setVariables(withMockNumberOfNVMeTCPTargets(2)) + nodeSvc.useNVME[firstGlobalID] = true + + clientMock.On("GetCluster", mock.Anything). + Return(gopowerstore.Cluster{Name: validClusterName}, nil) + clientMock.On("GetStorageNVMETCPTargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "nqn"}, + NetworkID: "NW1", + }, + { + Address: "192.168.1.2", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "nqn"}, + NetworkID: "NW1", + }, + { + Address: "192.168.2.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "nqn"}, + NetworkID: "NW2", + }, + { + Address: "192.168.2.2", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "nqn"}, + NetworkID: "NW2", + }, + }, nil) + + nodeSvc.populateTargetsInCache(nodeSvc.Arrays()[firstValidIP]) + if len(nodeSvc.nvmeTargets[firstGlobalID]) != 4 { + t.Errorf("Expected nvmeTargets to be populated") + } + }) + + t.Run("PopulateTargetsInCache - iscsiTargets should be populated in multiple networks [iSCSI]", func(t *testing.T) { + setVariables(withMockNumberOfISCSITargets(4)) + nodeSvc.useNVME[firstGlobalID] = false + + clientMock.On("GetCluster", mock.Anything). + Return(gopowerstore.Cluster{Name: validClusterName}, nil) + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "nqn"}, + NetworkID: "NW1", + }, + { + Address: "192.168.1.2", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "nqn"}, + NetworkID: "NW1", + }, + { + Address: "192.168.2.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "nqn"}, + NetworkID: "NW2", + }, + { + Address: "192.168.2.2", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "nqn"}, + NetworkID: "NW2", + }, + }, nil) + + nodeSvc.populateTargetsInCache(nodeSvc.Arrays()[firstValidIP]) + if len(nodeSvc.iscsiTargets[firstGlobalID]) != 4 { + t.Errorf("Expected iscsiTargets to be populated") + } + }) + + t.Run("PopulateTargetsInCache - nvmeTargets should be populated [NVMeFC]", func(t *testing.T) { + setVariables() + nodeSvc.useNVME[firstGlobalID] = true + nodeSvc.useFC[firstGlobalID] = true + + clientMock.On("GetCluster", mock.Anything). + Return(gopowerstore.Cluster{Name: validClusterName}, nil) + clientMock.On("GetFCPorts", mock.Anything). + Return([]gopowerstore.FcPort{ + { + Wwn: "58:cc:f0:93:48:a0:03:a3", + IsLinkUp: true, + }, + }, nil) + + nodeSvc.populateTargetsInCache(nodeSvc.Arrays()[firstValidIP]) + + if len(nodeSvc.nvmeTargets[firstGlobalID]) != 1 { + t.Errorf("Expected nvmeTargets to be populated") + } + }) + + t.Run("PopulateTargetsInCache - iscsiTargets should not be populated [iSCSI]", func(t *testing.T) { + setVariables() + + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{}, errors.New("some error")) + + nodeSvc.populateTargetsInCache(nodeSvc.Arrays()[firstValidIP]) + + if len(nodeSvc.iscsiTargets[firstGlobalID]) != 0 { + t.Errorf("Expected iscsiTargets to be empty upon error") + } + }) + + t.Run("PopulateTargetsInCache - nvmeTargets should not be populated [NVMeTCP]", func(t *testing.T) { + setVariables() + nodeSvc.useNVME[firstGlobalID] = true + + clientMock.On("GetCluster", mock.Anything). + Return(gopowerstore.Cluster{Name: validClusterName}, nil) + clientMock.On("GetStorageNVMETCPTargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{}, errors.New("some error")) + + nodeSvc.populateTargetsInCache(nodeSvc.Arrays()[firstValidIP]) + + if len(nodeSvc.nvmeTargets[firstGlobalID]) != 0 { + t.Errorf("Expected nvmeTargets to be empty upon error") + } + }) + + t.Run("PopulateTargetsInCache - nvmeTargets should not be populated [NVMeFC]", func(t *testing.T) { + setVariables() + nodeSvc.useNVME[firstGlobalID] = true + nodeSvc.useFC[firstGlobalID] = true + + clientMock.On("GetCluster", mock.Anything). + Return(gopowerstore.Cluster{Name: validClusterName}, nil) + clientMock.On("GetFCPorts", mock.Anything). + Return([]gopowerstore.FcPort{}, errors.New("some error")) + + nodeSvc.populateTargetsInCache(nodeSvc.Arrays()[firstValidIP]) + + if len(nodeSvc.nvmeTargets[firstGlobalID]) != 0 { + t.Errorf("Expected nvmeTargets to be empty upon error") + } + }) +} diff --git a/pkg/node/node_test.go b/pkg/node/node_test.go index 3e1ca112..9f516ff8 100644 --- a/pkg/node/node_test.go +++ b/pkg/node/node_test.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2021-2023 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2021-2026 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,19 +21,21 @@ package node import ( "context" "errors" + "fmt" "net" "net/http" "os" "path" "path/filepath" + "strconv" "testing" - "github.com/container-storage-interface/spec/lib/go/csi" "github.com/dell/csi-powerstore/v2/mocks" "github.com/dell/csi-powerstore/v2/pkg/array" - "github.com/dell/csi-powerstore/v2/pkg/common" - "github.com/dell/csi-powerstore/v2/pkg/common/k8sutils" "github.com/dell/csi-powerstore/v2/pkg/controller" + "github.com/dell/csi-powerstore/v2/pkg/identifiers" + "github.com/dell/csi-powerstore/v2/pkg/identifiers/k8sutils" + "github.com/dell/csmlog" "github.com/dell/gobrick" csictx "github.com/dell/gocsi/context" "github.com/dell/gofsutil" @@ -42,54 +44,84 @@ import ( "github.com/dell/gopowerstore" "github.com/dell/gopowerstore/api" gopowerstoremock "github.com/dell/gopowerstore/mocks" - . "github.com/onsi/ginkgo" + "github.com/container-storage-interface/spec/lib/go/csi" + ginkgo "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" - . "github.com/onsi/gomega" + gomega "github.com/onsi/gomega" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + corev1 "k8s.io/api/core/v1" + k8score "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/rest" ) var ( - iscsiConnectorMock *mocks.ISCSIConnector - nvmeConnectorMock *mocks.NVMEConnector - fcConnectorMock *mocks.FcConnector - utilMock *mocks.UtilInterface - fsMock *mocks.FsInterface - nodeSvc *Service - clientMock *gopowerstoremock.Client - ctrlMock *mocks.ControllerInterface - iscsiLibMock *goiscsi.MockISCSI - nvmeLibMock *gonvme.MockNVMe - nodeLabelsRetrieverMock *mocks.NodeLabelsRetrieverInterface + iscsiConnectorMock *mocks.ISCSIConnector + nvmeConnectorMock *mocks.NVMEConnector + fcConnectorMock *mocks.FcConnector + utilMock *mocks.UtilInterface + fsMock *mocks.FsInterface + nodeSvc *Service + clientMock *gopowerstoremock.Client + ctrlMock *mocks.ControllerInterface + iscsiLibMock *goiscsi.MockISCSI + nvmeLibMock *gonvme.MockNVMe ) const ( - validBaseVolumeID = "39bb1b5f-5624-490d-9ece-18f7b28a904e" - validBlockVolumeID = "39bb1b5f-5624-490d-9ece-18f7b28a904e/gid1/scsi" - validClusterName = "localSystemName" - validNfsVolumeID = "39bb1b5f-5624-490d-9ece-18f7b28a904e/gid2/nfs" - validVolSize = 16 * 1024 * 1024 * 1024 - validLUNID = "3" - validLUNIDINT = 3 - nodeStagePrivateDir = "test/stage" - unimplementedErrMsg = "rpc error: code = Unimplemented desc = " - validNodeID = "csi-node-1a47a1b91c444a8a90193d8066669603-127.0.0.1" - validHostID = "e8f4c5f8-c2fc-4df4-bd99-c292c12b55be" - testErrMsg = "test err" - validDeviceWWN = "68ccf09800e23ab798312a05426acae0" - validDevPath = "/dev/sdag" - validDevName = "sdag" - validNfsExportPath = "/mnt/nfs" - validTargetPath = "/var/lib/kubelet/pods/dac33335-a31d-11e9-b46e-005056917428/" + + validBaseVolumeID = "39bb1b5f-5624-490d-9ece-18f7b28a904e" + validRemoteBaseVolumeID = "00000000-0000-0000-0000-000000000002" + validClusterName = "localSystemName" + validNfsVolumeID = "39bb1b5f-5624-490d-9ece-18f7b28a904e/gid2/nfs" + validRemoteVolID = "9f840c56-96e6-4de9-b5a3-27e7c20eaa77" + invalidBlockVolumeID = "39bb1b5f-5624-490d-9ece-18f7b28a904e/gid3/scsi" + validVolSize = 16 * 1024 * 1024 * 1024 + validLUNID = "3" + validLUNIDINT = 3 + nodeStagePrivateDir = "test/stage" + validNodeID = "csi-node-1a47a1b91c444a8a90193d8066669603-127.0.0.1" + validNodeID2 = "csi-node-90193d80666696031a47a1b91c444a8a-127.0.0.1" + validHostID = "e8f4c5f8-c2fc-4df4-bd99-c292c12b55be" + validHostName = "csi-node-1a47a1b91c444a8a90193d8066669603" + validDeviceWWN = "68ccf09800e23ab798312a05426acae0" + validDevName = "sdag" + validNfsExportPath = "/mnt/nfs" + validTargetPath = "/var/lib/kubelet/pods/dac33335-a31d-11e9-b46e-005056917428/" + "volumes/kubernetes.io~csi/csi-d91431aba3/mount" validStagingPath = "/var/lib/kubelet/plugins/kubernetes.io/csi/volumeDevices/" + "staging/csi-44b46e98ae/c875b4f0-172e-4238-aec7-95b379eb55db" - firstValidIP = "gid1" - secondValidIP = "gid2" - validNasName = "my-nas-name" - validNasID = "e8f4c5f8-c2fc-4df4-bd99-c292c12b55be" - validNfsServerID = "e8f4c5f8-c2fc-4dd2-bd99-c292c12b55be" - validEphemeralName = "ephemeral-39bb1b5f-5624-490d-9ece-18f7b28a904e/gid1/scsi" - ephemerallockfile = "/var/lib/kubelet/plugins/kubernetes.io/csi/pv/ephemeral/39bb1b5f-5624-490d-9ece-18f7b28a904e/gid1/scsi/id" + firstValidIP = "gid1" + secondValidIP = "gid2" + metroFirstValidIP = "gid3" + metroSecondValidIP = "gid4" + firstGlobalID = "unique1" + secondGlobalID = "unique2" + validNasName = "my-nas-name" + validNasID = "e8f4c5f8-c2fc-4df4-bd99-c292c12b55be" + validNfsServerID = "e8f4c5f8-c2fc-4dd2-bd99-c292c12b55be" + validEphemeralName = "ephemeral-39bb1b5f-5624-490d-9ece-18f7b28a904e/gid1/scsi" + ephemerallockfile = "/var/lib/kubelet/plugins/kubernetes.io/csi/pv/ephemeral/39bb1b5f-5624-490d-9ece-18f7b28a904e/gid1/scsi/id" + validMetroSessionID = "9abd0198-2733-4e46-b5fa-456e9c367184" + ProtoSCSI = "scsi" + + zoneLabelKey = "topology.kubernetes.io/zone" + zone1LabelValue = "zone1" + zone2LabelValue = "zone2" +) + +var ( + // format: // + validBlockVolumeHandle = filepath.Join(validBaseVolumeID, firstValidIP, ProtoSCSI) + + // format: //:/ + validMetroVolumeHandle = filepath.Join(validBlockVolumeHandle+":"+validRemoteBaseVolumeID, secondValidIP) + + zone1Label = map[string]string{zoneLabelKey: zone1LabelValue} + zone2Label = map[string]string{zoneLabelKey: zone2LabelValue} ) var ( @@ -97,59 +129,130 @@ var ( validFCTargetWWNNVMe = []string{"58ccf090496008aa", "58ccf090496008aa"} validFCTargetWWNNode = []string{"58ccf090c96008aa", "58ccf090c96008aa"} validFCTargetsWWPNPowerstore = []string{"58:cc:f0:93:48:a0:03:a3", "58:cc:f0:93:48:a0:02:a3"} - validFCTargetsInfo = []gobrick.FCTargetInfo{{WWPN: validFCTargetsWWPN[0]}, - {WWPN: validFCTargetsWWPN[1]}} + validFCTargetsInfo = []gobrick.FCTargetInfo{ + {WWPN: validFCTargetsWWPN[0]}, + {WWPN: validFCTargetsWWPN[1]}, + } validISCSIInitiators = []string{"iqn.1994-05.com.redhat:4db86abbe3c", "iqn.1994-05.com.redhat:2950c9ca441b"} validISCSIPortals = []string{"192.168.1.1:3260", "192.168.1.2:3260"} - validISCSITargets = []string{"iqn.2015-10.com.dell:dellemc-powerstore-fnm00180700173-a-39f17e0e", - "iqn.2015-10.com.dell:dellemc-powerstore-fnm00180700173-b-10de15a5"} - validNVMEInitiators = []string{"nqn.2014-08.org.nvmexpress:uuid:02a08600-57d6-4089-8736-bf1f7326990e", - "nqn.2014-08.org.nvmexpress:uuid:fa363a22-1c74-44f3-9932-1c35d5cf5c4d"} + validISCSITargets = []string{ + "iqn.2015-10.com.dell:dellemc-powerstore-fnm00180700173-a-39f17e0e", + "iqn.2015-10.com.dell:dellemc-powerstore-fnm00180700173-b-10de15a5", + } + validNVMEInitiators = []string{ + "nqn.2014-08.org.nvmexpress:uuid:02a08600-57d6-4089-8736-bf1f7326990e", + "nqn.2014-08.org.nvmexpress:uuid:fa363a22-1c74-44f3-9932-1c35d5cf5c4d", + } validNVMETCPPortals = []string{"192.168.1.1:4420", "192.168.1.2:4420"} - validNVMETCPTargets = []string{"nqn.1988-11.com.dell:powerstore:00:e6e2d5b871f1403E169D", - "nqn.1988-11.com.dell:powerstore:00:e6e2d5b871f1403E169D"} + validNVMETCPTargets = []string{ + "nqn.1988-11.com.dell:powerstore:00:e6e2d5b871f1403E169D", + "nqn.1988-11.com.dell:powerstore:00:e6e2d5b871f1403E169D", + } validNVMEFCPortals = []string{"nn-0x11ccf090c9200b1a:pn-0x11ccf09149280b1a", "nn-0x11ccf090c9200b1a:pn-0x11ccf09149280b1a"} - validNVMEFCTargets = []string{"nqn.1988-11.com.dell:powerstore:00:e6e2d5b871f1403E169D", - "nqn.1988-11.com.dell:powerstore:00:e6e2d5b871f1403E169D"} + validNVMEFCTargets = []string{ + "nqn.1988-11.com.dell:powerstore:00:e6e2d5b871f1403E169D", + "nqn.1988-11.com.dell:powerstore:00:e6e2d5b871f1403E169D", + } validISCSITargetInfo = []gobrick.ISCSITargetInfo{ {Portal: validISCSIPortals[0], Target: validISCSITargets[0]}, - {Portal: validISCSIPortals[1], Target: validISCSITargets[1]}} + {Portal: validISCSIPortals[1], Target: validISCSITargets[1]}, + } validGobrickISCSIVolumeINFO = gobrick.ISCSIVolumeInfo{ Targets: []gobrick.ISCSITargetInfo{ - {Portal: validISCSITargetInfo[0].Portal, - Target: validISCSITargetInfo[0].Target}, - {Portal: validISCSITargetInfo[1].Portal, Target: validISCSITargetInfo[1].Target}}, - Lun: validLUNIDINT} + { + Portal: validISCSITargetInfo[0].Portal, + Target: validISCSITargetInfo[0].Target, + }, + {Portal: validISCSITargetInfo[1].Portal, Target: validISCSITargetInfo[1].Target}, + }, + Lun: validLUNIDINT, + } validNVMETCPTargetInfo = []gobrick.NVMeTargetInfo{ {Portal: validNVMETCPPortals[0], Target: validNVMETCPTargets[0]}, - {Portal: validNVMETCPPortals[1], Target: validNVMETCPTargets[1]}} + {Portal: validNVMETCPPortals[1], Target: validNVMETCPTargets[1]}, + } validGobrickNVMETCPVolumeINFO = gobrick.NVMeVolumeInfo{ Targets: []gobrick.NVMeTargetInfo{ - {Portal: validNVMETCPTargetInfo[0].Portal, - Target: validNVMETCPTargetInfo[0].Target}, - {Portal: validNVMETCPTargetInfo[1].Portal, Target: validNVMETCPTargetInfo[1].Target}}, - WWN: validDeviceWWN} + { + Portal: validNVMETCPTargetInfo[0].Portal, + Target: validNVMETCPTargetInfo[0].Target, + }, + {Portal: validNVMETCPTargetInfo[1].Portal, Target: validNVMETCPTargetInfo[1].Target}, + }, + WWN: validDeviceWWN, + } validNVMEFCTargetInfo = []gobrick.NVMeTargetInfo{ {Portal: validNVMEFCPortals[0], Target: validNVMEFCTargets[0]}, - {Portal: validNVMEFCPortals[1], Target: validNVMEFCTargets[1]}} + {Portal: validNVMEFCPortals[1], Target: validNVMEFCTargets[1]}, + } validGobrickNVMEFCVolumeINFO = gobrick.NVMeVolumeInfo{ Targets: []gobrick.NVMeTargetInfo{ - {Portal: validNVMEFCTargetInfo[0].Portal, - Target: validNVMEFCTargetInfo[0].Target}, - {Portal: validNVMEFCTargetInfo[1].Portal, Target: validNVMEFCTargetInfo[1].Target}}, - WWN: validDeviceWWN} + { + Portal: validNVMEFCTargetInfo[0].Portal, + Target: validNVMEFCTargetInfo[0].Target, + }, + {Portal: validNVMEFCTargetInfo[1].Portal, Target: validNVMEFCTargetInfo[1].Target}, + }, + WWN: validDeviceWWN, + } validGobrickFCVolumeINFO = gobrick.FCVolumeInfo{ Targets: []gobrick.FCTargetInfo{ {WWPN: validFCTargetsWWPN[0]}, - {WWPN: validFCTargetsWWPN[1]}}, - Lun: validLUNIDINT} + {WWPN: validFCTargetsWWPN[1]}, + }, + Lun: validLUNIDINT, + } validGobrickDevice = gobrick.Device{Name: validDevName, WWN: validDeviceWWN, MultipathID: validDeviceWWN} + + validRemoteISCSIPortals = []string{"192.168.1.3:3260", "192.168.1.4:3260"} + validRemoteISCSITargets = []string{ + "iqn.2015-10.com.dell:dellemc-powerstore-fnm00180700174-a-39f17e0e", + "iqn.2015-10.com.dell:dellemc-powerstore-fnm00180700174-b-10de15a5", + } + validRemoteFCTargetsWWPN = []string{"58ccf09348a003a4", "58ccf09348a002a4"} + validRemoteISCSITargetInfo = []gobrick.ISCSITargetInfo{ + {Portal: validRemoteISCSIPortals[0], Target: validRemoteISCSITargets[0]}, + {Portal: validRemoteISCSIPortals[1], Target: validRemoteISCSITargets[1]}, + } ) +// default empty usage +var usage = []*csi.VolumeUsage{ + { + Available: 0, + Total: 0, + Used: 0, + Unit: csi.VolumeUsage_BYTES, + }, +} + +func setFSmocks() { + fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil) + fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return([]gofsutil.Info{}, nil) + fsMock.On("MkFileIdempotent", filepath.Join(nodeStagePrivateDir, validBaseVolumeID)).Return(true, nil) + fsMock.On("GetUtil").Return(utilMock) + utilMock.On("BindMount", mock.Anything, mock.Anything, mock.Anything).Return(nil) +} + func TestCSINodeService(t *testing.T) { - RegisterFailHandler(Fail) + defaultK8sConfigFunc := k8sutils.InClusterConfigFunc + defaultK8sClientsetFunc := k8sutils.NewForConfigFunc + + k8sutils.InClusterConfigFunc = func() (*rest.Config, error) { + return &rest.Config{}, nil + } + k8sutils.NewForConfigFunc = func(_ *rest.Config) (kubernetes.Interface, error) { + return fake.NewClientset(), nil + } + + defer func() { + k8sutils.InClusterConfigFunc = defaultK8sConfigFunc + k8sutils.NewForConfigFunc = defaultK8sClientsetFunc + }() + + gomega.RegisterFailHandler(ginkgo.Fail) junitReporter := reporters.NewJUnitReporter("node-svc.xml") - RunSpecsWithDefaultAndCustomReporters(t, "CSINodeService testing suite", []Reporter{junitReporter}) + ginkgo.RunSpecsWithDefaultAndCustomReporters(t, "CSINodeService testing suite", []ginkgo.Reporter{junitReporter}) } func getTestArrays() map[string]*array.PowerStoreArray { @@ -158,31 +261,160 @@ func getTestArrays() map[string]*array.PowerStoreArray { Endpoint: "https://192.168.0.1/api/rest", Username: "admin", Password: "pass", - BlockProtocol: common.ISCSITransport, + BlockProtocol: identifiers.ISCSITransport, Insecure: true, IsDefault: true, - GlobalID: "unique", + GlobalID: firstGlobalID, Client: clientMock, IP: firstValidIP, } second := &array.PowerStoreArray{ - Endpoint: "https://192.168.0.2/api/rest", + Endpoint: "https://192.168.0.2:9400/api/rest", Username: "admin", Password: "pass", NasName: validNasName, - BlockProtocol: common.NoneTransport, + BlockProtocol: identifiers.NoneTransport, Insecure: true, - GlobalID: "unique2", + GlobalID: secondGlobalID, Client: clientMock, IP: secondValidIP, } arrays[firstValidIP] = first arrays[secondValidIP] = second + + return arrays +} + +func getMetroTestArrays() map[string]*array.PowerStoreArray { + arrays := make(map[string]*array.PowerStoreArray) + first := &array.PowerStoreArray{ + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array3", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + HostConnectivity: &array.HostConnectivity{ + Metro: array.MetroConnectivityOptions{ + ColocatedLocal: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"zone1"}, + }, + }, + }, + }, + }, + ColocatedRemote: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"zone2"}, + }, + }, + }, + }, + }, + }, + }, + IP: metroFirstValidIP, + Client: clientMock, + } + second := &array.PowerStoreArray{ + Endpoint: "https://10.198.0.2/api/rest", + GlobalID: "Array4", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + HostConnectivity: &array.HostConnectivity{ + Metro: array.MetroConnectivityOptions{ + ColocatedRemote: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"zone2"}, + }, + }, + }, + }, + }, + }, + }, + IP: metroSecondValidIP, + Client: clientMock, + } + + arrays[metroFirstValidIP] = first + arrays[metroSecondValidIP] = second + return arrays } -func setVariables() { +func getBaseClient() *k8sutils.K8sClient { + return &k8sutils.K8sClient{ + Clientset: fake.NewClientset([]runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Labels: map[string]string{"topology.kubernetes.io/zone": "zone1"}, + }, + }, + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node2", + Labels: map[string]string{"topology.kubernetes.io/zone": "zone2"}, + }, + }, + }...), + } +} + +type variableOptions struct { + mockNumberOfNVMeTCPTargets int + mockNumberOfISCSITargets int +} + +type variableOption func(*variableOptions) + +func withMockNumberOfNVMeTCPTargets(count int) variableOption { + return func(vo *variableOptions) { + vo.mockNumberOfNVMeTCPTargets = count + } +} + +func withMockNumberOfISCSITargets(count int) variableOption { + return func(vo *variableOptions) { + vo.mockNumberOfISCSITargets = count + } +} + +func setVariables(options ...variableOption) { + option := &variableOptions{} + for _, vo := range options { + vo(option) + } + + mockNVMeOptions := make(map[string]string) + mockISCSIOptions := make(map[string]string) + if option.mockNumberOfNVMeTCPTargets != 0 { + mockNVMeOptions[gonvme.MockNumberOfTCPTargets] = strconv.Itoa(option.mockNumberOfNVMeTCPTargets) + } + if option.mockNumberOfISCSITargets != 0 { + mockISCSIOptions[goiscsi.MockNumberOfTargets] = strconv.Itoa(option.mockNumberOfISCSITargets) + } + iscsiConnectorMock = new(mocks.ISCSIConnector) nvmeConnectorMock = new(mocks.NVMEConnector) fcConnectorMock = new(mocks.FcConnector) @@ -190,11 +422,8 @@ func setVariables() { fsMock = new(mocks.FsInterface) ctrlMock = new(mocks.ControllerInterface) clientMock = new(gopowerstoremock.Client) - iscsiLibMock = goiscsi.NewMockISCSI(nil) - nvmeLibMock = gonvme.NewMockNVMe(nil) - nodeLabelsRetrieverMock = new(mocks.NodeLabelsRetrieverInterface) - k8sutils.NodeLabelsRetriever = nodeLabelsRetrieverMock - + iscsiLibMock = goiscsi.NewMockISCSI(mockISCSIOptions) + nvmeLibMock = gonvme.NewMockNVMe(mockNVMeOptions) arrays := getTestArrays() nodeSvc = &Service{ @@ -206,17 +435,23 @@ func setVariables() { iscsiLib: iscsiLibMock, nvmeLib: nvmeLibMock, nodeID: validNodeID, - useFC: false, - useNVME: false, initialized: true, isPodmonEnabled: false, + opts: Opts{ + KubeNodeName: "node1", + }, } + + k8sutils.Kubeclient = getBaseClient() + nodeSvc.iscsiTargets = make(map[string][]string) nodeSvc.nvmeTargets = make(map[string][]string) + nodeSvc.useFC = make(map[string]bool) + nodeSvc.useNVME = make(map[string]bool) old := ReachableEndPoint func() { ReachableEndPoint = old }() ReachableEndPoint = func(ip string) bool { - if ip == "192.168.1.1:3260" || ip == "192.168.1.2:3260" { + if ip == "192.168.1.1:3260" || ip == "192.168.1.2:3260" || ip == "192.168.1.3:3260" || ip == "192.168.1.4:3260" { return true } return false @@ -225,23 +460,32 @@ func setVariables() { nodeSvc.SetDefaultArray(arrays[firstValidIP]) } -func setDefaultNodeLabelsRetrieverMock() { - nodeLabelsRetrieverMock.On("BuildConfigFromFlags", mock.Anything, mock.Anything).Return(nil, nil) - nodeLabelsRetrieverMock.On("GetNodeLabels", mock.Anything, mock.Anything, mock.Anything).Return(nil, nil) - nodeLabelsRetrieverMock.On("InClusterConfig", mock.Anything).Return(nil, nil) - nodeLabelsRetrieverMock.On("NewForConfig", mock.Anything).Return(nil, nil) +func setDefaultNodeLabelsMock() { } -var _ = Describe("CSINodeService", func() { - BeforeEach(func() { - setVariables() +var options []variableOption + +var _ = ginkgo.Describe("CSINodeService", func() { + os.Setenv(identifiers.EnvKubeNodeName, "node1") + + ginkgo.BeforeEach(func() { + setVariables(options...) }) - Describe("calling Init()", func() { - When("there is no suitable host", func() { - It("should create this host", func() { + nasData := []gopowerstore.NAS{ + { + NfsServers: []gopowerstore.NFSServerInstance{ + { + IsNFSv4Enabled: true, + IsNFSv3Enabled: false, + }, + }, + }, + } + ginkgo.Describe("calling Init()", func() { + ginkgo.When("there is no suitable host", func() { + ginkgo.It("should create this host", func() { nodeSvc.nodeID = "" - fsMock.On("ReadFile", mock.Anything).Return([]byte("my-host-id"), nil) conn, _ := net.Dial("udp", "127.0.0.1:80") fsMock.On("NetDial", mock.Anything).Return( @@ -270,29 +514,35 @@ var _ = Describe("CSINodeService", func() { }}, Name: "host-name", }}, nil) - clientMock.On("GetCustomHTTPHeaders").Return(make(http.Header)) + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{}, nil) + clientMock.On("GetCustomHTTPHeaders").Return(api.NewSafeHeader().GetHeader()) clientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(3.0), nil) clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) clientMock.On("CreateHost", mock.Anything, mock.Anything). Return(gopowerstore.CreateResponse{ID: validHostID}, nil) + setDefaultNodeLabelsMock() nodeSvc.opts.NodeNamePrefix = "" err := nodeSvc.Init() - Expect(err).To(BeNil()) + gomega.Expect(err).To(gomega.BeNil()) }) }) - When("failed to read nodeID file", func() { - It("should fail", func() { + ginkgo.When("failed to read nodeID file", func() { + ginkgo.It("should fail", func() { nodeSvc.nodeID = "" + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{}, nil) fsMock.On("ReadFile", mock.Anything).Return([]byte("my-host-id"), errors.New("no such file")) + setDefaultNodeLabelsMock() err := nodeSvc.Init() - Expect(err.Error()).To(ContainSubstring("no such file")) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("no such file")) }) }) - When("failed to get outbound ip", func() { - It("should fail", func() { + ginkgo.When("failed to get outbound ip", func() { + ginkgo.It("should fail", func() { nodeSvc.nodeID = "" fsMock.On("ReadFile", mock.Anything).Return([]byte("my-host-id"), nil) @@ -323,15 +573,18 @@ var _ = Describe("CSINodeService", func() { }}, Name: "host-name", }}, nil) + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{}, nil) clientMock.On("CreateHost", mock.Anything, mock.Anything). Return(gopowerstore.CreateResponse{ID: validHostID}, nil) + setDefaultNodeLabelsMock() nodeSvc.opts.NodeNamePrefix = "" err := nodeSvc.Init() - Expect(err.Error()).To(ContainSubstring("Could not connect to PowerStore array")) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("Could not connect to PowerStore array")) }) }) - When("failed to get node id", func() { - It("should fail", func() { + ginkgo.When("failed to get node id", func() { + ginkgo.It("should fail", func() { nodeSvc.nodeID = "" fsMock.On("ReadFile", mock.Anything).Return([]byte("toooooooooooooooooooooo-looooooooooooooooooooooooooooooooooooooooong"), nil) @@ -362,69 +615,19 @@ var _ = Describe("CSINodeService", func() { }}, Name: "host-name", }}, nil) + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) clientMock.On("CreateHost", mock.Anything, mock.Anything). Return(gopowerstore.CreateResponse{ID: validHostID}, nil) + setDefaultNodeLabelsMock() nodeSvc.opts.NodeNamePrefix = "" err := nodeSvc.Init() - Expect(err.Error()).To(ContainSubstring("node name prefix is too long")) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("node name prefix is too long")) }) }) - When("there IS a suitable host", func() { - When("nodeID == hostName", func() { - It("should reuse host [no initiator updates]", func() { - iscsiConnectorMock.On("GetInitiatorName", mock.Anything). - Return(validISCSIInitiators, nil) - nvmeConnectorMock.On("GetInitiatorName", mock.Anything). - Return(validNVMEInitiators, nil) - fcConnectorMock.On("GetInitiatorPorts", mock.Anything). - Return(validFCTargetsWWPN, nil) - - clientMock.On("GetHostByName", mock.Anything, mock.AnythingOfType("string")). - Return(gopowerstore.Host{ - ID: "host-id", - Initiators: []gopowerstore.InitiatorInstance{{ - PortName: validISCSIInitiators[0], - PortType: gopowerstore.InitiatorProtocolTypeEnumISCSI, - }, - { - PortName: validISCSIInitiators[1], - PortType: gopowerstore.InitiatorProtocolTypeEnumISCSI, - }}, - Name: "host-name", - }, nil) - err := nodeSvc.Init() - Expect(err).To(BeNil()) - }) - - It("should modify host [update initiators]", func() { - iscsiConnectorMock.On("GetInitiatorName", mock.Anything). - Return(validISCSIInitiators, nil) - nvmeConnectorMock.On("GetInitiatorName", mock.Anything). - Return(validNVMEInitiators, nil) - fcConnectorMock.On("GetInitiatorPorts", mock.Anything). - Return(validFCTargetsWWPN, nil) - - clientMock.On("GetHostByName", mock.Anything, mock.AnythingOfType("string")). - Return(gopowerstore.Host{ - ID: "host-id", - Initiators: []gopowerstore.InitiatorInstance{{ - PortName: "not-matching-port-name", - PortType: gopowerstore.InitiatorProtocolTypeEnumISCSI, - }}, - Name: "host-name", - }, nil) - - clientMock.On("ModifyHost", mock.Anything, mock.Anything, "host-id"). - Return(gopowerstore.CreateResponse{}, nil) - - err := nodeSvc.Init() - Expect(err).To(BeNil()) - }) - }) - - When("nodeID != hostName", func() { - It("should reuse host", func() { + ginkgo.When("there IS a suitable host", func() { + ginkgo.When("nodeID != hostName", func() { + ginkgo.It("should reuse host", func() { nodeSvc.nodeID = "" fsMock.On("ReadFile", mock.Anything).Return([]byte("my-host-id"), nil) conn, _ := net.Dial("udp", "127.0.0.1:80") @@ -454,14 +657,18 @@ var _ = Describe("CSINodeService", func() { }}, Name: "host-name", }}, nil) + clientMock.On("ModifyHost", mock.Anything, mock.Anything, "host-id"). + Return(gopowerstore.CreateResponse{ID: "host-id"}, nil) + setDefaultNodeLabelsMock() + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) err := nodeSvc.Init() - Expect(err).To(BeNil()) + gomega.Expect(err).To(gomega.BeNil()) }) - It("should reuse host [CHAP]", func() { + ginkgo.It("should reuse host [CHAP]", func() { nodeSvc.nodeID = "" - _ = csictx.Setenv(context.Background(), common.EnvEnableCHAP, "true") + _ = csictx.Setenv(context.Background(), identifiers.EnvEnableCHAP, "true") conn, _ := net.Dial("udp", "127.0.0.1:80") fsMock.On("NetDial", mock.Anything).Return( conn, @@ -493,24 +700,26 @@ var _ = Describe("CSINodeService", func() { }}, nil) clientMock.On("ModifyHost", mock.Anything, mock.Anything, "host-id"). Return(gopowerstore.CreateResponse{ID: "host-id"}, nil) + setDefaultNodeLabelsMock() + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) err := nodeSvc.Init() - Expect(err).To(BeNil()) + gomega.Expect(err).To(gomega.BeNil()) }) }) }) - When("using FC", func() { - It("should create FC host", func() { - nodeSvc.Arrays()[firstValidIP].BlockProtocol = common.FcTransport + ginkgo.When("using FC", func() { + ginkgo.It("should create FC host", func() { + nodeSvc.Arrays()[firstValidIP].BlockProtocol = identifiers.FcTransport nodeSvc.nodeID = "" - nodeSvc.useFC = true + nodeSvc.useFC[firstGlobalID] = true conn, _ := net.Dial("udp", "127.0.0.1:80") fsMock.On("NetDial", mock.Anything).Return( conn, nil, ) - _ = csictx.Setenv(context.Background(), common.EnvFCPortsFilterFilePath, "filter-path") + _ = csictx.Setenv(context.Background(), identifiers.EnvFCPortsFilterFilePath, "filter-path") fsMock.On("ReadFile", mock.Anything).Return([]byte("my-host-id"), nil).Once() fsMock.On("ReadFile", "filter-path"). @@ -538,22 +747,24 @@ var _ = Describe("CSINodeService", func() { }}, Name: "host-name", }}, nil) - clientMock.On("GetCustomHTTPHeaders").Return(make(http.Header)) + clientMock.On("GetCustomHTTPHeaders").Return(api.NewSafeHeader().GetHeader()) clientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(3.0), nil) clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) clientMock.On("CreateHost", mock.Anything, mock.Anything). Return(gopowerstore.CreateResponse{ID: validHostID}, nil) + setDefaultNodeLabelsMock() + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) err := nodeSvc.Init() - Expect(err).To(BeNil()) + gomega.Expect(err).To(gomega.BeNil()) }) }) - When("using NVMe", func() { - It("should create NVMe host", func() { - nodeSvc.Arrays()[firstValidIP].BlockProtocol = common.NVMEFCTransport + ginkgo.When("using NVMe", func() { + ginkgo.It("should create NVMe host", func() { + nodeSvc.Arrays()[firstValidIP].BlockProtocol = identifiers.NVMEFCTransport nodeSvc.nodeID = "" - nodeSvc.useNVME = true + nodeSvc.useNVME[firstGlobalID] = true fsMock.On("ReadFile", mock.Anything).Return([]byte("my-host-id"), nil) conn, _ := net.Dial("udp", "127.0.0.1:80") fsMock.On("NetDial", mock.Anything).Return( @@ -583,99 +794,333 @@ var _ = Describe("CSINodeService", func() { Name: "host-name", }}, nil) - clientMock.On("GetCustomHTTPHeaders").Return(make(http.Header)) + clientMock.On("GetCustomHTTPHeaders").Return(api.NewSafeHeader().GetHeader()) clientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(3.0), nil) clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) clientMock.On("CreateHost", mock.Anything, mock.Anything). Return(gopowerstore.CreateResponse{ID: validHostID}, nil) + setDefaultNodeLabelsMock() + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) err := nodeSvc.Init() - Expect(err).To(BeNil()) + gomega.Expect(err).To(gomega.BeNil()) }) - }) - }) - Describe("calling nodeProbe", func() { + ginkgo.It("should create NVMe host and check for duplicate UUIDs", func() { + nodeSvc.Arrays()[firstValidIP].BlockProtocol = identifiers.NVMEFCTransport + nodeSvc.nodeID = "" + nodeSvc.useNVME[firstGlobalID] = true + fsMock.On("ReadFile", mock.Anything).Return([]byte("my-host-id"), nil) + conn, _ := net.Dial("udp", "127.0.0.1:80") + fsMock.On("NetDial", mock.Anything).Return( + conn, + nil, + ) + iscsiConnectorMock.On("GetInitiatorName", mock.Anything). + Return(validISCSIInitiators, nil) + nvmeConnectorMock.On("GetInitiatorName", mock.Anything). + Return(validNVMEInitiators, nil) + fcConnectorMock.On("GetInitiatorPorts", mock.Anything). + Return(validFCTargetsWWPN, nil) - When("failed to get host on array", func() { - It("should fail", func() { - nodeSvc.nodeID = "some-random-text" + nodeSvc.opts.KubeNodeName = identifiers.EnvKubeNodeName + nodeSvc.opts.KubeConfigPath = identifiers.EnvKubeConfigPath clientMock.On("GetHostByName", mock.Anything, mock.AnythingOfType("string")). Return(gopowerstore.Host{}, gopowerstore.APIError{ ErrorMsg: &api.ErrorMsg{ StatusCode: http.StatusNotFound, - Message: "not found", }, }) - arrays := getTestArrays() - err := nodeSvc.nodeProbe(context.Background(), arrays["gid1"]) - Expect(err.Error()).To(ContainSubstring("not found")) + clientMock.On("GetHosts", mock.Anything).Return( + []gopowerstore.Host{{ + ID: "host-id", + Initiators: []gopowerstore.InitiatorInstance{{ + PortName: "not-matching-port-name", + PortType: gopowerstore.InitiatorProtocolTypeEnumNVME, + }}, + Name: "host-name", + }}, nil) + + clientMock.On("GetCustomHTTPHeaders").Return(api.NewSafeHeader().GetHeader()) + clientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(3.0), nil) + clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) + clientMock.On("CreateHost", mock.Anything, mock.Anything). + Return(gopowerstore.CreateResponse{ID: validHostID}, nil) + setDefaultNodeLabelsMock() + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) + + k8sutils.Kubeclient.SetNodeLabel(context.Background(), "node1", "hostnqn-uuid", "duplicate-uuid") + k8sutils.Kubeclient.SetNodeLabel(context.Background(), "node2", "hostnqn-uuid", "duplicate-uuid") + + err := nodeSvc.Init() + gomega.Expect(err).To(gomega.BeNil()) }) }) - When("failed to get host on array but it's NFS only", func() { - It("should not fail", func() { - nodeSvc.nodeID = "some-random-text" + ginkgo.When("protocol flag initialization", func() { + ginkgo.It("should have correct entry for each array - NVMeTCP and iSCSI", func() { + nodeSvc.Arrays()[firstValidIP].BlockProtocol = identifiers.NVMETCPTransport + nodeSvc.Arrays()[secondValidIP].BlockProtocol = identifiers.ISCSITransport + nodeSvc.nodeID = "" + fsMock.On("ReadFile", mock.Anything).Return([]byte("my-host-id"), nil) + conn, _ := net.Dial("udp", "127.0.0.1:80") + fsMock.On("NetDial", mock.Anything).Return(conn, nil) + iscsiConnectorMock.On("GetInitiatorName", mock.Anything). + Return(validISCSIInitiators, nil) + nvmeConnectorMock.On("GetInitiatorName", mock.Anything). + Return(validNVMEInitiators, nil) + fcConnectorMock.On("GetInitiatorPorts", mock.Anything). + Return(validFCTargetsWWPN, nil) clientMock.On("GetHostByName", mock.Anything, mock.AnythingOfType("string")). Return(gopowerstore.Host{}, gopowerstore.APIError{ ErrorMsg: &api.ErrorMsg{ StatusCode: http.StatusNotFound, - Message: "not found", }, }) - nodeSvc.useNFS = true - arrays := getTestArrays() - err := nodeSvc.nodeProbe(context.Background(), arrays["gid1"]) - Expect(err).To(BeNil()) - nodeSvc.useNFS = false - }) - }) - - When("got host on array but iscsi initiators are not present", func() { - It("should fail", func() { - nodeSvc.nodeID = "some-random-text" - - clientMock.On("GetHostByName", mock.Anything, mock.AnythingOfType("string")).Return( - gopowerstore.Host{ - ID: "host-id", - Initiators: []gopowerstore.InitiatorInstance{}, - Name: "host-name", - }, nil) - - arrays := getTestArrays() + clientMock.On("GetHosts", mock.Anything).Return( + []gopowerstore.Host{{ + ID: "host-id", + Initiators: []gopowerstore.InitiatorInstance{{ + PortName: "not-matching-port-name", + PortType: gopowerstore.InitiatorProtocolTypeEnumNVME, + }}, + Name: "host-name", + }}, nil) + clientMock.On("GetCustomHTTPHeaders").Return(api.NewSafeHeader().GetHeader()) + clientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(3.0), nil) + clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) + clientMock.On("CreateHost", mock.Anything, mock.Anything). + Return(gopowerstore.CreateResponse{ID: validHostID}, nil) + setDefaultNodeLabelsMock() clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). Return([]gopowerstore.IPPoolAddress{}, nil) - err := nodeSvc.nodeProbe(context.Background(), arrays["gid1"]) - Expect(err.Error()).To(ContainSubstring("no active iscsi sessions")) + err := nodeSvc.Init() + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(nodeSvc.useNVME[firstGlobalID]).To(gomega.BeTrue()) + gomega.Expect(nodeSvc.useFC[firstGlobalID]).To(gomega.BeFalse()) + gomega.Expect(nodeSvc.useNVME[secondGlobalID]).To(gomega.BeFalse()) + gomega.Expect(nodeSvc.useFC[secondGlobalID]).To(gomega.BeFalse()) }) - }) - When("got host on array but nvme initiators are not present", func() { - It("should fail", func() { - nodeSvc.nodeID = "some-random-text" + ginkgo.It("should have correct entry for each array - iSCSI and NVMeFC", func() { + nodeSvc.Arrays()[firstValidIP].BlockProtocol = identifiers.ISCSITransport + nodeSvc.Arrays()[secondValidIP].BlockProtocol = identifiers.NVMEFCTransport + nodeSvc.nodeID = "" + fsMock.On("ReadFile", mock.Anything).Return([]byte("my-host-id"), nil) + conn, _ := net.Dial("udp", "127.0.0.1:80") + fsMock.On("NetDial", mock.Anything).Return(conn, nil) + iscsiConnectorMock.On("GetInitiatorName", mock.Anything). + Return(validISCSIInitiators, nil) + nvmeConnectorMock.On("GetInitiatorName", mock.Anything). + Return(validNVMEInitiators, nil) + fcConnectorMock.On("GetInitiatorPorts", mock.Anything). + Return(validFCTargetsWWPN, nil) - clientMock.On("GetHostByName", mock.Anything, mock.AnythingOfType("string")).Return( - gopowerstore.Host{ - ID: "host-id", - Initiators: []gopowerstore.InitiatorInstance{}, - Name: "host-name", - }, nil) + clientMock.On("GetHostByName", mock.Anything, mock.AnythingOfType("string")). + Return(gopowerstore.Host{}, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{ + StatusCode: http.StatusNotFound, + }, + }) + clientMock.On("GetHosts", mock.Anything).Return( + []gopowerstore.Host{{ + ID: "host-id", + Initiators: []gopowerstore.InitiatorInstance{{ + PortName: "not-matching-port-name", + PortType: gopowerstore.InitiatorProtocolTypeEnumNVME, + }}, + Name: "host-name", + }}, nil) + clientMock.On("GetCustomHTTPHeaders").Return(api.NewSafeHeader().GetHeader()) + clientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(3.0), nil) + clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) + clientMock.On("CreateHost", mock.Anything, mock.Anything). + Return(gopowerstore.CreateResponse{ID: validHostID}, nil) + setDefaultNodeLabelsMock() + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) + + err := nodeSvc.Init() + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(nodeSvc.useNVME[firstGlobalID]).To(gomega.BeFalse()) + gomega.Expect(nodeSvc.useFC[firstGlobalID]).To(gomega.BeFalse()) + gomega.Expect(nodeSvc.useNVME[secondGlobalID]).To(gomega.BeTrue()) + gomega.Expect(nodeSvc.useFC[secondGlobalID]).To(gomega.BeTrue()) + }) + + ginkgo.It("should set useNVME/useFC when transport is not set", func() { + nodeSvc.useNVME[firstGlobalID] = false + nodeSvc.useFC[firstGlobalID] = false + nodeSvc.nodeID = "" + fsMock.On("ReadFile", mock.Anything).Return([]byte("my-host-id"), nil) + conn, _ := net.Dial("udp", "127.0.0.1:80") + fsMock.On("NetDial", mock.Anything).Return(conn, nil) + iscsiConnectorMock.On("GetInitiatorName", mock.Anything). + Return(validISCSIInitiators, nil) + nvmeConnectorMock.On("GetInitiatorName", mock.Anything). + Return(validNVMEInitiators, nil) + fcConnectorMock.On("GetInitiatorPorts", mock.Anything). + Return(validFCTargetsWWPN, nil) + + clientMock.On("GetHostByName", mock.Anything, mock.AnythingOfType("string")). + Return(gopowerstore.Host{}, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{ + StatusCode: http.StatusNotFound, + }, + }) + + clientMock.On("GetHosts", mock.Anything).Return( + []gopowerstore.Host{{ + ID: "host-id", + Initiators: []gopowerstore.InitiatorInstance{{ + PortName: "not-matching-port-name", + PortType: gopowerstore.InitiatorProtocolTypeEnumNVME, + }}, + Name: "host-name", + }}, nil) + + clientMock.On("GetCustomHTTPHeaders").Return(api.NewSafeHeader().GetHeader()) + clientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(3.0), nil) + clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) + clientMock.On("CreateHost", mock.Anything, mock.Anything). + Return(gopowerstore.CreateResponse{ID: validHostID}, nil) + setDefaultNodeLabelsMock() + + nodeSvc.Arrays()[firstValidIP].BlockProtocol = "default_protocol" + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) + + err := nodeSvc.Init() + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(nodeSvc.useNVME[firstGlobalID]).To(gomega.BeTrue()) + gomega.Expect(nodeSvc.useFC[firstGlobalID]).To(gomega.BeTrue()) + }) + }) + + ginkgo.When("using NFS when length of all initiators is 0", func() { + ginkgo.It("should probe successfully", func() { + nodeSvc.nodeID = "" + // setup mocks + fsMock.On("ReadFile", mock.Anything).Return([]byte("my-host-id"), nil) + conn, _ := net.Dial("udp", "127.0.0.1:80") + fsMock.On("NetDial", mock.Anything).Return(conn, nil) + iscsiConnectorMock.On("GetInitiatorName", mock.Anything). + Return([]string{}, nil) + nvmeConnectorMock.On("GetInitiatorName", mock.Anything). + Return([]string{}, nil) + fcConnectorMock.On("GetInitiatorPorts", mock.Anything). + Return([]string{}, nil) + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) + + err := nodeSvc.Init() + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(nodeSvc.useNVME[firstGlobalID]).To(gomega.BeFalse()) + gomega.Expect(nodeSvc.useFC[firstGlobalID]).To(gomega.BeFalse()) + gomega.Expect(nodeSvc.useNVME[secondGlobalID]).To(gomega.BeFalse()) + gomega.Expect(nodeSvc.useFC[secondGlobalID]).To(gomega.BeFalse()) + gomega.Expect(nodeSvc.useNFS).To(gomega.BeTrue()) + }) + }) + }) + + ginkgo.Describe("calling nodeProbe", func() { + ginkgo.When("failed to get host on array", func() { + ginkgo.It("should fail", func() { + nodeSvc.nodeID = "some-random-text" + + clientMock.On("GetHostByName", mock.Anything, mock.AnythingOfType("string")). + Return(gopowerstore.Host{}, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{ + StatusCode: http.StatusNotFound, + Message: "not found", + }, + }) + arrays := getTestArrays() + err := nodeSvc.nodeProbe(context.Background(), arrays["gid1"]) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("not found")) + }) + }) + + ginkgo.When("failed to get host on array but it's NFS only", func() { + ginkgo.It("should not fail", func() { + nodeSvc.nodeID = "some-random-text" + + clientMock.On("GetHostByName", mock.Anything, mock.AnythingOfType("string")). + Return(gopowerstore.Host{}, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{ + StatusCode: http.StatusNotFound, + Message: "not found", + }, + }) + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) + nodeSvc.useNFS = true + arrays := getTestArrays() + err := nodeSvc.nodeProbe(context.Background(), arrays["gid1"]) + gomega.Expect(err).To(gomega.BeNil()) + nodeSvc.useNFS = false + }) + }) + + ginkgo.When("got host on array but iscsi initiators are not present", func() { + ginkgo.It("should fail", func() { + nodeSvc.nodeID = "some-random-text" + + clientMock.On("GetHostByName", mock.Anything, mock.AnythingOfType("string")).Return( + gopowerstore.Host{ + ID: "host-id", + Initiators: []gopowerstore.InitiatorInstance{}, + Name: "host-name", + }, nil) + + arrays := getTestArrays() + + clientMock.On("GetNASServers", mock.Anything). + Return(nasData, nil) + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{}, nil) + clientMock.On("GetStorageNVMETCPTargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{}, nil) + clientMock.On("GetCluster", mock.Anything). + Return(gopowerstore.Cluster{ + Name: validClusterName, + NVMeNQN: validNVMEInitiators[0], + }, nil) + err := nodeSvc.nodeProbe(context.Background(), arrays["gid1"]) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("no active iscsi sessions")) + }) + }) + + ginkgo.When("got host on array but nvme initiators are not present", func() { + ginkgo.It("should fail", func() { + nodeSvc.nodeID = "some-random-text" + + clientMock.On("GetHostByName", mock.Anything, mock.AnythingOfType("string")).Return( + gopowerstore.Host{ + ID: "host-id", + Initiators: []gopowerstore.InitiatorInstance{}, + Name: "host-name", + }, nil) arrays := getTestArrays() - nodeSvc.useNVME = true + nodeSvc.useNVME[firstGlobalID] = true clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). Return([]gopowerstore.IPPoolAddress{}, nil) + clientMock.On("GetStorageNVMETCPTargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{}, nil) + clientMock.On("GetCluster", mock.Anything). + Return(gopowerstore.Cluster{ + Name: validClusterName, + NVMeNQN: validNVMEInitiators[0], + }, nil) err := nodeSvc.nodeProbe(context.Background(), arrays["gid1"]) - nodeSvc.useNVME = false - Expect(err.Error()).To(ContainSubstring("no active nvme sessions")) + nodeSvc.useNVME[firstGlobalID] = false + gomega.Expect(err.Error()).To(gomega.ContainSubstring("no active nvme sessions")) }) }) - When("got host on array but iscsi initiators are not present and UseNFS is true at the beginning", func() { - It("should not fail", func() { + ginkgo.When("got host on array but iscsi initiators are not present and UseNFS is true at the beginning", func() { + ginkgo.It("should not fail", func() { nodeSvc.nodeID = "some-random-text" clientMock.On("GetHostByName", mock.Anything, mock.AnythingOfType("string")).Return( @@ -685,21 +1130,29 @@ var _ = Describe("CSINodeService", func() { { PortName: validISCSIPortals[0], PortType: gopowerstore.InitiatorProtocolTypeEnumISCSI, - }}, + }, + }, Name: "host-name", }, nil) clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). Return([]gopowerstore.IPPoolAddress{}, nil) + clientMock.On("GetStorageNVMETCPTargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{}, nil) + clientMock.On("GetCluster", mock.Anything). + Return(gopowerstore.Cluster{ + Name: validClusterName, + NVMeNQN: validNVMEInitiators[0], + }, nil) nodeSvc.useNFS = true arrays := getTestArrays() err := nodeSvc.nodeProbe(context.Background(), arrays["gid1"]) nodeSvc.useNFS = false - Expect(err).To(BeNil()) + gomega.Expect(err).To(gomega.BeNil()) }) }) - When("got host on array but nvme initiators are not present and UseNFS is true at the beginning", func() { - It("should not fail", func() { + ginkgo.When("got host on array but nvme initiators are not present and UseNFS is true at the beginning", func() { + ginkgo.It("should not fail", func() { nodeSvc.nodeID = "some-random-text" clientMock.On("GetHostByName", mock.Anything, mock.AnythingOfType("string")).Return( @@ -709,23 +1162,31 @@ var _ = Describe("CSINodeService", func() { { PortName: validISCSIPortals[0], PortType: gopowerstore.InitiatorProtocolTypeEnumISCSI, - }}, + }, + }, Name: "host-name", }, nil) clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). Return([]gopowerstore.IPPoolAddress{}, nil) + clientMock.On("GetStorageNVMETCPTargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{}, nil) + clientMock.On("GetCluster", mock.Anything). + Return(gopowerstore.Cluster{ + Name: validClusterName, + NVMeNQN: validNVMEInitiators[0], + }, nil) nodeSvc.useNFS = true - nodeSvc.useNVME = true + nodeSvc.useNVME[firstGlobalID] = true arrays := getTestArrays() err := nodeSvc.nodeProbe(context.Background(), arrays["gid1"]) nodeSvc.useNFS = false - nodeSvc.useNVME = false - Expect(err).To(BeNil()) + nodeSvc.useNVME[firstGlobalID] = false + gomega.Expect(err).To(gomega.BeNil()) }) }) - When("got host on array but it's NFS type at the beginning but later found iscsi active sessions", func() { - It("should not fail", func() { + ginkgo.When("got host on array but it's NFS type at the beginning but later found iscsi active sessions", func() { + ginkgo.It("should not fail", func() { nodeSvc.nodeID = "some-random-text" clientMock.On("GetHostByName", mock.Anything, mock.AnythingOfType("string")).Return( @@ -749,23 +1210,24 @@ var _ = Describe("CSINodeService", func() { }, PortName: validISCSIPortals[1], PortType: gopowerstore.InitiatorProtocolTypeEnumISCSI, - }}, + }, + }, Name: "host-name", }, nil) clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). Return([]gopowerstore.IPPoolAddress{}, nil) arrays := getTestArrays() nodeSvc.useNFS = true - nodeSvc.iscsiTargets["unique"] = []string{"iqn.2015-10.com.dell:dellemc-foobar-123-a-7ceb34a0"} + nodeSvc.iscsiTargets[firstGlobalID] = []string{"iqn.2015-10.com.dell:dellemc-foobar-123-a-7ceb34a0"} err := nodeSvc.nodeProbe(context.Background(), arrays["gid1"]) - Expect(err).To(BeNil()) - Expect(nodeSvc.useNFS).To(BeFalse()) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(nodeSvc.useNFS).To(gomega.BeFalse()) }) }) - When("got host on array but it's NFS type at the beginning but later found nvme active sessions", func() { - It("should not fail", func() { + ginkgo.When("got host on array but it's NFS type at the beginning but later found nvme active sessions", func() { + ginkgo.It("should not fail", func() { nodeSvc.nodeID = "some-random-text" clientMock.On("GetHostByName", mock.Anything, mock.AnythingOfType("string")).Return( @@ -789,76 +1251,84 @@ var _ = Describe("CSINodeService", func() { }, PortName: validNVMETCPPortals[1], PortType: gopowerstore.InitiatorProtocolTypeEnumNVME, - }}, + }, + }, Name: "host-name", }, nil) clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). Return([]gopowerstore.IPPoolAddress{}, nil) arrays := getTestArrays() nodeSvc.useNFS = true - nodeSvc.useNVME = true - nodeSvc.nvmeTargets["unique"] = []string{"nqn.1988-11.com.dell.mock:00:e6e2d5b871f1403E169D0"} + nodeSvc.useNVME[firstGlobalID] = true + nodeSvc.nvmeTargets[firstGlobalID] = []string{"nqn.1988-11.com.dell.mock:00:e6e2d5b871f1403E169D0"} err := nodeSvc.nodeProbe(context.Background(), arrays["gid1"]) - nodeSvc.useNVME = false - Expect(err).To(BeNil()) - Expect(nodeSvc.useNFS).To(BeFalse()) + nodeSvc.useNVME[firstGlobalID] = false + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(nodeSvc.useNFS).To(gomega.BeFalse()) }) }) - When("host as well as initiators are present but active sessions are not present on node", func() { - It("should fail", func() { + ginkgo.When("host as well as initiators are present but active sessions are not present on node", func() { + ginkgo.It("should fail", func() { nodeSvc.nodeID = "some-random-text" clientMock.On("GetHostByName", mock.Anything, mock.AnythingOfType("string")).Return( gopowerstore.Host{ ID: "host-id", - Initiators: []gopowerstore.InitiatorInstance{{ - PortName: validISCSIInitiators[0], - PortType: gopowerstore.InitiatorProtocolTypeEnumISCSI, - }, + Initiators: []gopowerstore.InitiatorInstance{ + { + PortName: validISCSIInitiators[0], + PortType: gopowerstore.InitiatorProtocolTypeEnumISCSI, + }, { PortName: validISCSIInitiators[1], PortType: gopowerstore.InitiatorProtocolTypeEnumISCSI, - }}, + }, + }, Name: "host-name", }, nil) clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). Return([]gopowerstore.IPPoolAddress{}, nil) arrays := getTestArrays() err := nodeSvc.nodeProbe(context.Background(), arrays["gid1"]) - Expect(err.Error()).To(ContainSubstring("no active iscsi sessions")) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("no active iscsi sessions")) }) }) - When("host as well as iscsi active sessions are present on array", func() { - It("should not fail", func() { + ginkgo.When("host as well as iscsi active sessions are present on array", func() { + ginkgo.It("should not fail", func() { nodeSvc.nodeID = "some-random-text" clientMock.On("GetHostByName", mock.Anything, mock.AnythingOfType("string")).Return( gopowerstore.Host{ ID: "host-id", - Initiators: []gopowerstore.InitiatorInstance{{ - PortName: validISCSIInitiators[0], - PortType: gopowerstore.InitiatorProtocolTypeEnumISCSI, - }, + Initiators: []gopowerstore.InitiatorInstance{ + { + PortName: validISCSIInitiators[0], + PortType: gopowerstore.InitiatorProtocolTypeEnumISCSI, + }, { PortName: validISCSIInitiators[1], PortType: gopowerstore.InitiatorProtocolTypeEnumISCSI, - }}, + }, + }, Name: "host-name", }, nil) - + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) arrays := getTestArrays() + err := nodeSvc.nodeProbe(context.Background(), arrays["gid1"]) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("no active iscsi sessions")) + + nodeSvc.iscsiTargets[firstGlobalID] = []string{"iqn.2015-10.com.dell:dellemc-foobar-123-a-7ceb34a0"} nodeSvc.startNodeToArrayConnectivityCheck(context.Background()) - nodeSvc.iscsiTargets["unique"] = []string{"iqn.2015-10.com.dell:dellemc-foobar-123-a-7ceb34a0"} - err := nodeSvc.nodeProbe(context.Background(), arrays["gid1"]) - Expect(err).To(BeNil()) + err = nodeSvc.nodeProbe(context.Background(), arrays["gid1"]) + gomega.Expect(err).To(gomega.BeNil()) }) }) - When("host as well as active sessions are present on array for FC protocol", func() { - It("should not fail", func() { + ginkgo.When("host as well as active sessions are present on array for FC protocol", func() { + ginkgo.It("should not fail", func() { nodeSvc.nodeID = "some-random-text" clientMock.On("GetHostByName", mock.Anything, mock.AnythingOfType("string")).Return( @@ -882,25 +1352,26 @@ var _ = Describe("CSINodeService", func() { }, PortName: validFCTargetsWWPN[1], PortType: gopowerstore.InitiatorProtocolTypeEnumFC, - }}, + }, + }, Name: "host-name", }, nil) - + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) arrays := getTestArrays() - if nodeSvc.useNVME { - nodeSvc.useNVME = false + if nodeSvc.useNVME[firstGlobalID] { + nodeSvc.useNVME[firstGlobalID] = false } - if !nodeSvc.useFC { - nodeSvc.useFC = true + if !nodeSvc.useFC[firstGlobalID] { + nodeSvc.useFC[firstGlobalID] = true } err := nodeSvc.nodeProbe(context.Background(), arrays["gid1"]) - Expect(err).To(BeNil()) + gomega.Expect(err).To(gomega.BeNil()) }) }) - When("host entry is found but no Active session on array for FC protocol", func() { - It("should not fail", func() { + ginkgo.When("host entry is found but no Active session on array for FC protocol", func() { + ginkgo.It("should not fail", func() { nodeSvc.nodeID = "some-random-text" clientMock.On("GetHostByName", mock.Anything, mock.AnythingOfType("string")).Return( @@ -909,90 +1380,82 @@ var _ = Describe("CSINodeService", func() { Initiators: []gopowerstore.InitiatorInstance{}, Name: "host-name", }, nil) - + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) arrays := getTestArrays() - if nodeSvc.useNVME { - nodeSvc.useNVME = false + if nodeSvc.useNVME[firstGlobalID] { + nodeSvc.useNVME[firstGlobalID] = false } - nodeSvc.useFC = true + nodeSvc.useFC[firstGlobalID] = true err := nodeSvc.nodeProbe(context.Background(), arrays["gid1"]) - Expect(err).ToNot(BeNil()) - if nodeSvc.useFC { - nodeSvc.useFC = false + gomega.Expect(err).ToNot(gomega.BeNil()) + if nodeSvc.useFC[firstGlobalID] { + nodeSvc.useFC[firstGlobalID] = false } }) }) }) - Describe("calling NodeStage()", func() { + ginkgo.Describe("calling NodeStage()", func() { stagingPath := filepath.Join(nodeStagePrivateDir, validBaseVolumeID) - - When("using iSCSI", func() { - It("should successfully stage iSCSI volume", func() { - iscsiConnectorMock.On("ConnectVolume", mock.Anything, gobrick.ISCSIVolumeInfo{ - Targets: validISCSITargetInfo, - Lun: validLUNIDINT, - }).Return(gobrick.Device{}, nil) - + ginkgo.When("using iSCSI", func() { + ginkgo.It("should successfully stage iSCSI volume", func() { + setDefaultClientMocks() + iscsiConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything).Return(gobrick.Device{}, nil) scsiStageVolumeOK(utilMock, fsMock) res, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: nodeStagePrivateDir, VolumeCapability: getCapabilityWithVoltypeAccessFstype( "mount", "single-writer", "ext4"), }) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.NodeStageVolumeResponse{})) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeStageVolumeResponse{})) }) }) - When("using NVMeFC", func() { - It("should successfully stage NVMeFC volume", func() { - nodeSvc.useNVME = true - nodeSvc.useFC = true - nvmeConnectorMock.On("ConnectVolume", mock.Anything, gobrick.NVMeVolumeInfo{ - Targets: validNVMEFCTargetInfo, - WWN: validDeviceWWN, - }, true).Return(gobrick.Device{}, nil) + ginkgo.When("using NVMeFC", func() { + ginkgo.It("should successfully stage NVMeFC volume", func() { + setDefaultClientMocks() + nodeSvc.useNVME[firstGlobalID] = true + nodeSvc.useFC[firstGlobalID] = true + nvmeConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything, true).Return(gobrick.Device{}, nil) scsiStageVolumeOK(utilMock, fsMock) res, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: nodeStagePrivateDir, VolumeCapability: getCapabilityWithVoltypeAccessFstype( "mount", "single-writer", "ext4"), }) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.NodeStageVolumeResponse{})) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeStageVolumeResponse{})) }) }) - When("using FC", func() { - It("should successfully stage FC volume", func() { - nodeSvc.useFC = true - fcConnectorMock.On("ConnectVolume", mock.Anything, gobrick.FCVolumeInfo{ - Targets: validFCTargetsInfo, - Lun: validLUNIDINT, - }).Return(gobrick.Device{}, nil) + ginkgo.When("using FC", func() { + ginkgo.It("should successfully stage FC volume", func() { + setDefaultClientMocks() + nodeSvc.useFC[firstGlobalID] = true + fcConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything).Return(gobrick.Device{}, nil) scsiStageVolumeOK(utilMock, fsMock) res, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: nodeStagePrivateDir, VolumeCapability: getCapabilityWithVoltypeAccessFstype( "mount", "single-writer", "ext4"), }) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.NodeStageVolumeResponse{})) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeStageVolumeResponse{})) }) }) - When("using NFS", func() { - It("should successfully stage NFS volume", func() { + ginkgo.When("using NFS", func() { + ginkgo.It("should successfully stage NFS volume", func() { stagingPath := filepath.Join(nodeStagePrivateDir, validBaseVolumeID) utilMock.On("Mount", mock.Anything, validNfsExportPath, stagingPath, "").Return(nil) fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(2) @@ -1002,7 +1465,7 @@ var _ = Describe("CSINodeService", func() { fsMock.On("Chmod", filepath.Join(stagingPath, commonNfsVolumeFolder), os.ModeSticky|os.ModePerm).Return(nil) fsMock.On("GetUtil").Return(utilMock) - publishContext := getValidPublishContext() + publishContext := make(map[string]string) publishContext["NfsExportPath"] = validNfsExportPath res, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ @@ -1013,13 +1476,13 @@ var _ = Describe("CSINodeService", func() { "mount", "multi-writer", "nfs"), }) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.NodeStageVolumeResponse{})) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeStageVolumeResponse{})) }) }) - When("using NFS with posix acls", func() { - It("should successfully stage NFS volume", func() { + ginkgo.When("using NFS with posix acls", func() { + ginkgo.It("should successfully stage NFS volume", func() { nfsv4ACLsMock := new(mocks.NFSv4ACLsInterface) stagingPath := filepath.Join(nodeStagePrivateDir, validBaseVolumeID) @@ -1031,20 +1494,21 @@ var _ = Describe("CSINodeService", func() { fsMock.On("Chmod", filepath.Join(stagingPath, commonNfsVolumeFolder), os.ModeSticky|os.ModePerm).Return(nil) fsMock.On("GetUtil").Return(utilMock) - publishContext := getValidPublishContext() + publishContext := make(map[string]string) publishContext["NfsExportPath"] = validNfsExportPath - publishContext[common.KeyNasName] = validNasName - publishContext[common.KeyNfsACL] = "0777" + publishContext[identifiers.KeyNasName] = validNasName + publishContext[identifiers.KeyNfsACL] = "0777" nfsServers := []gopowerstore.NFSServerInstance{ { - Id: validNfsServerID, + ID: validNfsServerID, IsNFSv4Enabled: true, }, } - clientMock.On("GetNfsServer", mock.Anything, validNasName).Return(gopowerstore.NFSServerInstance{Id: validNfsServerID, IsNFSv4Enabled: true}, nil) + clientMock.On("GetNfsServer", mock.Anything, validNasName).Return(gopowerstore.NFSServerInstance{ID: validNfsServerID, IsNFSv4Enabled: true}, nil) clientMock.On("GetNASByName", mock.Anything, validNasName).Return(gopowerstore.NAS{ID: validNasID, NfsServers: nfsServers}, nil) + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) nfsv4ACLsMock.On("SetNfsv4Acls", mock.Anything, mock.Anything).Return(nil) res, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ @@ -1055,13 +1519,13 @@ var _ = Describe("CSINodeService", func() { "mount", "multi-writer", "nfs"), }) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.NodeStageVolumeResponse{})) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeStageVolumeResponse{})) }) }) - When("using NFS with NFSv4 acls", func() { - It("should successfully stage NFS volume", func() { + ginkgo.When("using NFS with NFSv4 acls", func() { + ginkgo.It("should successfully stage NFS volume", func() { nfsv4ACLsMock := new(mocks.NFSv4ACLsInterface) stagingPath := filepath.Join(nodeStagePrivateDir, validBaseVolumeID) @@ -1073,21 +1537,21 @@ var _ = Describe("CSINodeService", func() { fsMock.On("Chmod", filepath.Join(stagingPath, commonNfsVolumeFolder), os.ModeSticky|os.ModePerm).Return(nil) fsMock.On("GetUtil").Return(utilMock) - publishContext := getValidPublishContext() + publishContext := make(map[string]string) publishContext["NfsExportPath"] = validNfsExportPath - publishContext[common.KeyNfsACL] = "A::OWNER@:RWX" + publishContext[identifiers.KeyNfsACL] = "A::OWNER@:RWX" nfsServers := []gopowerstore.NFSServerInstance{ { - Id: validNfsServerID, + ID: validNfsServerID, IsNFSv4Enabled: true, }, } nfsv4ACLsMock.On("SetNfsv4Acls", mock.Anything, mock.Anything).Return(nil) clientMock.On("GetNASByName", mock.Anything, "").Return(gopowerstore.NAS{ID: validNasID, NfsServers: nfsServers}, nil) - clientMock.On("GetNfsServer", mock.Anything, mock.Anything).Return(gopowerstore.NFSServerInstance{Id: validNfsServerID, IsNFSv4Enabled: true}, nil) - + clientMock.On("GetNfsServer", mock.Anything, mock.Anything).Return(gopowerstore.NFSServerInstance{ID: validNfsServerID, IsNFSv4Enabled: true}, nil) + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ VolumeId: validNfsVolumeID, PublishContext: publishContext, @@ -1098,13 +1562,145 @@ var _ = Describe("CSINodeService", func() { }) }) - When("volume is already staged", func() { - It("should return that stage is successful [SCSI]", func() { - iscsiConnectorMock.On("ConnectVolume", mock.Anything, gobrick.ISCSIVolumeInfo{ - Targets: validISCSITargetInfo, - Lun: validLUNIDINT, - }).Return(gobrick.Device{}, nil) + ginkgo.When("using iSCSI for Metro volume", func() { + ginkgo.When("hostConnectivity is not configured in the secret (backward compatibility)", func() { + ginkgo.It("should successfully stage Metro iSCSI volume", func() { + setDefaultClientMocks() + iscsiConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything).Return(gobrick.Device{}, nil).Times(4) + scsiStageVolumeOK(utilMock, fsMock) + scsiStageRemoteMetroVolumeOK(utilMock, fsMock) + metroVolumeID := fmt.Sprintf("%s:%s/%s", validBlockVolumeHandle, validRemoteVolID, secondValidIP) + res, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ + VolumeId: metroVolumeID, + PublishContext: getValidUniformMetroPublishContext(), + StagingTargetPath: nodeStagePrivateDir, + VolumeCapability: getCapabilityWithVoltypeAccessFstype( + "mount", "single-writer", "ext4"), + }) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeStageVolumeResponse{})) + }) + }) + + ginkgo.When("hostConnectivity is configured for non-uniform metro", func() { + defaultNodeID := nodeSvc.nodeID + ginkgo.BeforeEach(func() { + arrays := getTestArrays() + arrays[firstValidIP].HostConnectivity = &array.HostConnectivity{ + Local: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: zoneLabelKey, + Operator: "In", + Values: []string{zone1LabelValue}, + }, + }, + }, + }, + }, + } + arrays[secondValidIP].HostConnectivity = &array.HostConnectivity{ + Local: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: zoneLabelKey, + Operator: "In", + Values: []string{zone2LabelValue}, + }, + }, + }, + }, + }, + } + nodeSvc.SetArrays(arrays) + nodeSvc.SetDefaultArray(arrays[firstValidIP]) + k8sutils.Kubeclient = &k8sutils.K8sClient{ + Clientset: fake.NewClientset([]runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: validNodeID, + Labels: zone1Label, + Annotations: map[string]string{identifiers.KeyNodeID: "{" + strconv.Quote(identifiers.Name) + ":" + strconv.Quote(validNodeID) + "}"}, + }, + }, + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: validNodeID2, + Labels: zone2Label, + Annotations: map[string]string{identifiers.KeyNodeID: "{" + strconv.Quote(identifiers.Name) + ":" + strconv.Quote(validNodeID2) + "}"}, + }, + }, + }...), + } + }) + ginkgo.AfterEach(func() { + // resetting for other tests, since we're all using the same instance + arrays := getTestArrays() + nodeSvc.SetArrays(arrays) + nodeSvc.SetDefaultArray(arrays[firstValidIP]) + nodeSvc.nodeID = defaultNodeID + k8sutils.Kubeclient = getBaseClient() + }) + + ginkgo.It("should not stage local volume when local array no connectivity to node", func() { + // publish to the "remote" node + nodeSvc.nodeID = validNodeID2 + + clientMock.On("GetVolume", mock.Anything, validBaseVolumeID).Return(gopowerstore.Volume{ + ID: validBaseVolID, + MetroReplicationSessionID: validMetroSessionID, + }, nil) + clientMock.On("GetReplicationSessionByID", mock.Anything, validMetroSessionID).Return(gopowerstore.ReplicationSession{ + ID: validMetroSessionID, + State: gopowerstore.RsStateOk, + LocalResourceState: string(gopowerstore.ReplicationResourceStatePromoted), + }, nil) + + setDefaultClientMocks() + iscsiConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything).Return(gobrick.Device{}, nil).Times(2) + scsiStageRemoteMetroVolumeOK(utilMock, fsMock) + metroVolumeID := fmt.Sprintf("%s:%s/%s", validBlockVolumeHandle, validRemoteVolID, secondValidIP) + req := &csi.NodeStageVolumeRequest{ + VolumeId: metroVolumeID, + // do not include identifiers.TargetMapDeviceWWN in publish context + PublishContext: getValidRemoteMetroPublishContext(), + StagingTargetPath: nodeStagePrivateDir, + VolumeCapability: getCapabilityWithVoltypeAccessFstype( + "mount", "single-writer", "ext4"), + } + resp, err := nodeSvc.NodeStageVolume(context.Background(), req) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(resp).To(gomega.Equal(&csi.NodeStageVolumeResponse{})) + }) + + ginkgo.It("should not stage remote metro volume remote array has no connectivity to node", func() { + setDefaultClientMocks() + iscsiConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything).Return(gobrick.Device{}, nil).Times(2) + scsiStageVolumeOK(utilMock, fsMock) + metroVolumeID := fmt.Sprintf("%s:%s/%s", validBlockVolumeHandle, validRemoteVolID, secondValidIP) + req := &csi.NodeStageVolumeRequest{ + VolumeId: metroVolumeID, + // do not include identifiers.TargetMapRemoteDeviceWWN in publish context + PublishContext: getValidPublishContext(), + StagingTargetPath: nodeStagePrivateDir, + VolumeCapability: getCapabilityWithVoltypeAccessFstype( + "mount", "single-writer", "ext4"), + } + resp, err := nodeSvc.NodeStageVolume(context.Background(), req) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(resp).To(gomega.Equal(&csi.NodeStageVolumeResponse{})) + }) + }) + }) + ginkgo.When("volume is already staged", func() { + ginkgo.It("should return that stage is successful [SCSI]", func() { + setDefaultClientMocks() + iscsiConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything).Return(gobrick.Device{}, nil) mountInfo := []gofsutil.Info{ { Device: validDevName, @@ -1117,17 +1713,17 @@ var _ = Describe("CSINodeService", func() { utilMock.On("GetDiskFormat", mock.Anything, stagingPath).Return("", nil) res, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: nodeStagePrivateDir, VolumeCapability: getCapabilityWithVoltypeAccessFstype( "mount", "single-writer", "ext4"), }) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.NodeStageVolumeResponse{})) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeStageVolumeResponse{})) }) - It("should return that stage is successful [NFS]", func() { + ginkgo.It("should return that stage is successful [NFS]", func() { publishContext := getValidPublishContext() publishContext["NfsExportPath"] = validNfsExportPath @@ -1141,7 +1737,7 @@ var _ = Describe("CSINodeService", func() { fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return(mountInfo, nil) fsMock.On("GetUtil").Return(utilMock) utilMock.On("GetDiskFormat", mock.Anything, stagingPath).Return("", nil) - + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) res, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ VolumeId: validNfsVolumeID, PublishContext: publishContext, @@ -1149,52 +1745,67 @@ var _ = Describe("CSINodeService", func() { VolumeCapability: getCapabilityWithVoltypeAccessFstype( "mount", "multi-writer", "nfs"), }) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.NodeStageVolumeResponse{})) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeStageVolumeResponse{})) }) }) - When("missing volume capabilities", func() { - It("should fail", func() { + ginkgo.When("missing volume capabilities", func() { + ginkgo.It("should fail", func() { req := &csi.NodeStageVolumeRequest{} - + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) res, err := nodeSvc.NodeStageVolume(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).NotTo(BeNil()) - Expect(err.Error()).To(ContainSubstring("volume capability is required")) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("volume capability is required")) }) }) - When("missing volume VolumeId", func() { - It("should fail", func() { + ginkgo.When("missing volume VolumeID", func() { + ginkgo.It("should fail", func() { req := &csi.NodeStageVolumeRequest{ VolumeCapability: getCapabilityWithVoltypeAccessFstype("mount", "single-writer", "ext4"), StagingTargetPath: nodeStagePrivateDir, } + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) + res, err := nodeSvc.NodeStageVolume(context.Background(), req) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("volume ID is required")) + }) + }) + ginkgo.When("invalid array ID", func() { + ginkgo.It("should fail", func() { + req := &csi.NodeStageVolumeRequest{ + VolumeCapability: getCapabilityWithVoltypeAccessFstype("mount", "single-writer", "ext4"), + StagingTargetPath: nodeStagePrivateDir, + VolumeId: invalidBlockVolumeID, + } + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) res, err := nodeSvc.NodeStageVolume(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).NotTo(BeNil()) - Expect(err.Error()).To(ContainSubstring("volume ID is required")) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("can't find array with ID")) }) }) - When("missing volume stage path", func() { - It("should fail", func() { + ginkgo.When("missing volume stage path", func() { + ginkgo.It("should fail", func() { req := &csi.NodeStageVolumeRequest{ VolumeCapability: getCapabilityWithVoltypeAccessFstype("mount", "single-writer", "ext4"), - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, } - + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) res, err := nodeSvc.NodeStageVolume(context.Background(), req) - Expect(res).To(BeNil()) - Expect(err).NotTo(BeNil()) - Expect(err.Error()).To(ContainSubstring("staging target path is required")) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("staging target path is required")) }) }) - When("device is found but not ready", func() { - BeforeEach(func() { + ginkgo.When("device is found but not ready", func() { + ginkgo.BeforeEach(func() { mountInfo := []gofsutil.Info{ { Device: validDevName, @@ -1203,10 +1814,7 @@ var _ = Describe("CSINodeService", func() { }, } - iscsiConnectorMock.On("ConnectVolume", mock.Anything, gobrick.ISCSIVolumeInfo{ - Targets: validISCSITargetInfo, - Lun: validLUNIDINT, - }).Return(gobrick.Device{}, nil) + iscsiConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything).Return(gobrick.Device{}, nil) // Stage utilMock.On("BindMount", mock.Anything, "/dev", @@ -1223,151 +1831,168 @@ var _ = Describe("CSINodeService", func() { utilMock.On("Unmount", mock.Anything, stagingPath).Return(nil).Once() }) - It("should unstage and stage again", func() { + ginkgo.It("should unstage and stage again", func() { + setDefaultClientMocks() fsMock.On("Remove", stagingPath).Return(nil).Once() res, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: nodeStagePrivateDir, VolumeCapability: getCapabilityWithVoltypeAccessFstype( "mount", "single-writer", "ext4"), }) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.NodeStageVolumeResponse{})) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeStageVolumeResponse{})) }) - When("unstaging fails", func() { - It("should fail", func() { + ginkgo.When("unstaging fails", func() { + setVariables() + setDefaultClientMocks() + ginkgo.It("should fail", func() { + setDefaultClientMocks() e := errors.New("os-error") fsMock.On("Remove", stagingPath).Return(e).Once() fsMock.On("IsNotExist", e).Return(false) fsMock.On("IsDeviceOrResourceBusy", e).Return(false) res, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: nodeStagePrivateDir, VolumeCapability: getCapabilityWithVoltypeAccessFstype( "mount", "single-writer", "ext4"), }) - Expect(err).ToNot(BeNil()) - Expect(res).To(BeNil()) - Expect(err.Error()).To(ContainSubstring("failed to unmount volume")) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("failed to unmount volume")) }) }) }) - When("publish context is incorrect", func() { - It("should fail [deviceWWN]", func() { + ginkgo.When("publish context is incorrect", func() { + ginkgo.It("should fail [deviceWWN]", func() { + setDefaultClientMocks() + setFSmocks() + originalIsNodeConnectedToArrayFunc := isNodeConnectedToArrayFunc + isNodeConnectedToArrayFunc = func(_ context.Context, _ string, _ *array.PowerStoreArray) bool { + return true + } + defer func() { isNodeConnectedToArrayFunc = originalIsNodeConnectedToArrayFunc }() + iscsiConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything).Return(gobrick.Device{}, nil) res, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ - VolumeId: validBlockVolumeID, - PublishContext: map[string]string{}, + VolumeId: validBlockVolumeHandle, + PublishContext: map[string]string{ + identifiers.TargetMapLUNAddress: validLUNID, + }, StagingTargetPath: nodeStagePrivateDir, VolumeCapability: getCapabilityWithVoltypeAccessFstype( "mount", "single-writer", "ext4"), }) - Expect(err).ToNot(BeNil()) - Expect(res).To(BeNil()) - Expect(err.Error()).To(ContainSubstring("deviceWWN must be in publish context")) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("deviceWWN must be in publish context")) }) - It("should fail [volumeLUNAddress]", func() { - res, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ - VolumeId: validBlockVolumeID, + ginkgo.It("should fail [iscsiTargets]", func() { + setDefaultClientMocks() + setFSmocks() + iscsiConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything).Return(gobrick.Device{}, nil) + _, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ + VolumeId: validBlockVolumeHandle, PublishContext: map[string]string{ - common.PublishContextDeviceWWN: validDeviceWWN, + identifiers.TargetMapDeviceWWN: validDeviceWWN, + identifiers.TargetMapLUNAddress: validLUNID, }, StagingTargetPath: nodeStagePrivateDir, VolumeCapability: getCapabilityWithVoltypeAccessFstype( "mount", "single-writer", "ext4"), }) - Expect(err).ToNot(BeNil()) - Expect(res).To(BeNil()) - Expect(err.Error()).To(ContainSubstring("volumeLUNAddress must be in publish context")) + gomega.Expect(err).To(gomega.BeNil()) }) - It("should fail [iscsiTargets]", func() { - res, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ - VolumeId: validBlockVolumeID, + ginkgo.It("should fail [nvmefcTargets]", func() { + setDefaultClientMocks() + setFSmocks() + nvmeConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything, true).Return(gobrick.Device{}, nil) + nodeSvc.useNVME[firstGlobalID] = true + nodeSvc.useFC[firstGlobalID] = true + _, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ + VolumeId: validBlockVolumeHandle, PublishContext: map[string]string{ - common.PublishContextDeviceWWN: validDeviceWWN, - common.PublishContextLUNAddress: validLUNID, + identifiers.TargetMapDeviceWWN: validDeviceWWN, + identifiers.TargetMapLUNAddress: validLUNID, }, StagingTargetPath: nodeStagePrivateDir, VolumeCapability: getCapabilityWithVoltypeAccessFstype( "mount", "single-writer", "ext4"), }) - Expect(err).ToNot(BeNil()) - Expect(res).To(BeNil()) - Expect(err.Error()).To(ContainSubstring("iscsiTargets data must be in publish context")) + gomega.Expect(err).To(gomega.BeNil()) }) - It("should fail [nvmefcTargets]", func() { - nodeSvc.useNVME = true - nodeSvc.useFC = true - res, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ - VolumeId: validBlockVolumeID, + ginkgo.It("should fail [nvmetcpTargets]", func() { + setDefaultClientMocks() + setFSmocks() + nvmeConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything, false).Return(gobrick.Device{}, nil) + nodeSvc.useNVME[firstGlobalID] = true + nodeSvc.useFC[firstGlobalID] = false + _, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ + VolumeId: validBlockVolumeHandle, PublishContext: map[string]string{ - common.PublishContextDeviceWWN: validDeviceWWN, - common.PublishContextLUNAddress: validLUNID, + identifiers.TargetMapDeviceWWN: validDeviceWWN, + identifiers.TargetMapLUNAddress: validLUNID, }, StagingTargetPath: nodeStagePrivateDir, VolumeCapability: getCapabilityWithVoltypeAccessFstype( "mount", "single-writer", "ext4"), }) - Expect(err).ToNot(BeNil()) - Expect(res).To(BeNil()) - Expect(err.Error()).To(ContainSubstring("NVMeFC Targets data must be in publish context")) + gomega.Expect(err).To(gomega.BeNil()) }) - It("should fail [fcTargets]", func() { - nodeSvc.useFC = true - res, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ - VolumeId: validBlockVolumeID, + ginkgo.It("should fail [fcTargets]", func() { + setDefaultClientMocks() + setFSmocks() + fcConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything).Return(gobrick.Device{}, nil) + nodeSvc.useFC[firstGlobalID] = true + _, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ + VolumeId: validBlockVolumeHandle, PublishContext: map[string]string{ - common.PublishContextDeviceWWN: validDeviceWWN, - common.PublishContextLUNAddress: validLUNID, + identifiers.TargetMapDeviceWWN: validDeviceWWN, + identifiers.TargetMapLUNAddress: validLUNID, }, StagingTargetPath: nodeStagePrivateDir, VolumeCapability: getCapabilityWithVoltypeAccessFstype( "mount", "single-writer", "ext4"), }) - Expect(err).ToNot(BeNil()) - Expect(res).To(BeNil()) - Expect(err.Error()).To(ContainSubstring("fcTargets data must be in publish context")) + gomega.Expect(err).To(gomega.BeNil()) }) }) - When("can not connect device", func() { - It("should fail", func() { + ginkgo.When("can not connect device", func() { + ginkgo.It("should fail", func() { + setDefaultClientMocks() e := errors.New("connection-error") - iscsiConnectorMock.On("ConnectVolume", mock.Anything, gobrick.ISCSIVolumeInfo{ - Targets: validISCSITargetInfo, - Lun: validLUNIDINT, - }).Return(gobrick.Device{}, e) + iscsiConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything).Return(gobrick.Device{}, e) scsiStageVolumeOK(utilMock, fsMock) res, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: nodeStagePrivateDir, VolumeCapability: getCapabilityWithVoltypeAccessFstype( "mount", "single-writer", "ext4"), }) - Expect(err).ToNot(BeNil()) - Expect(res).To(BeNil()) - Expect(err.Error()).To(ContainSubstring("unable to find device after multiple discovery attempts")) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("unable to find device after multiple discovery attempts")) }) }) - When("mount fails", func() { - It("should fail", func() { + ginkgo.When("mount fails", func() { + ginkgo.It("should fail", func() { + setDefaultClientMocks() e := errors.New("mount-error") - iscsiConnectorMock.On("ConnectVolume", mock.Anything, gobrick.ISCSIVolumeInfo{ - Targets: validISCSITargetInfo, - Lun: validLUNIDINT, - }).Return(gobrick.Device{}, nil) + iscsiConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything).Return(gobrick.Device{}, nil) fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(2) fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return([]gofsutil.Info{}, nil) fsMock.On("MkFileIdempotent", filepath.Join(nodeStagePrivateDir, validBaseVolumeID)).Return(true, nil) @@ -1376,39 +2001,367 @@ var _ = Describe("CSINodeService", func() { utilMock.On("BindMount", mock.Anything, "/dev", filepath.Join(nodeStagePrivateDir, validBaseVolumeID)).Return(e) res, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, + PublishContext: getValidPublishContext(), + StagingTargetPath: nodeStagePrivateDir, + VolumeCapability: getCapabilityWithVoltypeAccessFstype( + "mount", "single-writer", "ext4"), + }) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("error bind disk")) + }) + }) + + ginkgo.When("when mount call fails [NFS]", func() { + publishContext := getValidPublishContext() + publishContext["NfsExportPath"] = validNfsExportPath + var req *csi.NodeStageVolumeRequest + ginkgo.BeforeEach(func() { + req = &csi.NodeStageVolumeRequest{ + VolumeId: validNfsVolumeID, + PublishContext: publishContext, + StagingTargetPath: nodeStagePrivateDir, + VolumeCapability: getCapabilityWithVoltypeAccessFstype( + "mount", "multi-writer", "nfs"), + } + }) + + ginkgo.It("should fail [MkdirAll target folder]", func() { + stagingPath := filepath.Join(nodeStagePrivateDir, validBaseVolumeID) + fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(2) + fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return([]gofsutil.Info{}, nil) + fsMock.On("MkdirAll", stagingPath, mock.Anything).Return(errors.New("some-error")) + + res, err := nodeSvc.NodeStageVolume(context.Background(), req) + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("can't create target folder")) + }) + + ginkgo.It("should fail [Mount]", func() { + stagingPath := filepath.Join(nodeStagePrivateDir, validBaseVolumeID) + fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(2) + fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return([]gofsutil.Info{}, nil) + fsMock.On("MkdirAll", stagingPath, mock.Anything).Return(nil).Once() + fsMock.On("GetUtil").Return(utilMock) + utilMock.On("Mount", mock.Anything, validNfsExportPath, stagingPath, "").Return(errors.New("some-error")) + + res, err := nodeSvc.NodeStageVolume(context.Background(), req) + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("error mount nfs share")) + }) + + ginkgo.It("should fail [MkdirAll common folder]", func() { + stagingPath := filepath.Join(nodeStagePrivateDir, validBaseVolumeID) + fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(2) + fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return([]gofsutil.Info{}, nil) + fsMock.On("MkdirAll", stagingPath, mock.Anything).Return(nil).Once() + fsMock.On("GetUtil").Return(utilMock) + utilMock.On("Mount", mock.Anything, validNfsExportPath, stagingPath, "").Return(nil) + fsMock.On("MkdirAll", filepath.Join(stagingPath, commonNfsVolumeFolder), mock.Anything).Return(errors.New("some-error")) + + res, err := nodeSvc.NodeStageVolume(context.Background(), req) + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("can't create common folder")) + }) + + ginkgo.It("should fail [Chmod]", func() { + stagingPath := filepath.Join(nodeStagePrivateDir, validBaseVolumeID) + fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(2) + fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return([]gofsutil.Info{}, nil) + fsMock.On("MkdirAll", stagingPath, mock.Anything).Return(nil).Once() + fsMock.On("GetUtil").Return(utilMock) + utilMock.On("Mount", mock.Anything, validNfsExportPath, stagingPath, "").Return(nil) + fsMock.On("MkdirAll", filepath.Join(stagingPath, commonNfsVolumeFolder), mock.Anything).Return(nil) + fsMock.On("Chmod", filepath.Join(stagingPath, commonNfsVolumeFolder), os.ModeSticky|os.ModePerm).Return(errors.New("some-error")) + + res, err := nodeSvc.NodeStageVolume(context.Background(), req) + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("can't change permissions of folder")) + }) + }) + + ginkgo.When("when ModifyNFSExport call fails [NFS]", func() { + ginkgo.It("should fail", func() { + stagingPath := filepath.Join(nodeStagePrivateDir, validBaseVolumeID) + fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(2) + fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return([]gofsutil.Info{}, nil) + fsMock.On("MkdirAll", stagingPath, mock.Anything).Return(nil).Once() + fsMock.On("GetUtil").Return(utilMock) + utilMock.On("Mount", mock.Anything, validNfsExportPath, stagingPath, "").Return(nil) + fsMock.On("MkdirAll", filepath.Join(stagingPath, commonNfsVolumeFolder), mock.Anything).Return(nil) + fsMock.On("Chmod", filepath.Join(stagingPath, commonNfsVolumeFolder), os.ModeSticky|os.ModePerm).Return(nil) + clientMock.On("ModifyNFSExport", mock.Anything, mock.Anything, mock.Anything).Return(gopowerstore.CreateResponse{}, errors.New("some-error")) + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) + originalIsNodeConnectedToArrayFunc := isNodeConnectedToArrayFunc + isNodeConnectedToArrayFunc = func(_ context.Context, _ string, _ *array.PowerStoreArray) bool { + return true + } + defer func() { isNodeConnectedToArrayFunc = originalIsNodeConnectedToArrayFunc }() + publishContext := getValidPublishContext() + publishContext["NfsExportPath"] = validNfsExportPath + publishContext["allowRoot"] = "false" + publishContext["NatIP"] = "192.168.1.1" + req := &csi.NodeStageVolumeRequest{ + VolumeId: validNfsVolumeID, + PublishContext: publishContext, + StagingTargetPath: nodeStagePrivateDir, + VolumeCapability: getCapabilityWithVoltypeAccessFstype( + "mount", "multi-writer", "nfs"), + } + res, err := nodeSvc.NodeStageVolume(context.Background(), req) + + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("failure when modifying nfs export")) + }) + }) + + ginkgo.When("when a uniform metro session is fractured", func() { + var defaultCreateOrUpdateJournalEntryFunc func(ctx context.Context, name string, volumeHandle array.VolumeHandle, deferredArrayID string, nodeName string, operation string, request []byte) error + var defaultCheckMetroStateFunc func(ctx context.Context, volumeHandle array.VolumeHandle, client gopowerstore.Client, client2 gopowerstore.Client) (*array.MetroFracturedResponse, bool, error) + + ginkgo.BeforeEach(func() { + defaultCreateOrUpdateJournalEntryFunc = createOrUpdateJournalEntryFunc + defaultCheckMetroStateFunc = checkMetroStateFunc + }) + ginkgo.AfterEach(func() { + // resetting for other tests, since we're all using the same instance + createOrUpdateJournalEntryFunc = defaultCreateOrUpdateJournalEntryFunc + checkMetroStateFunc = defaultCheckMetroStateFunc + }) + + ginkgo.It("should create a journal entry when remote is not staged", func() { + setDefaultClientMocks() + clientMock.On("GetHostByName", mock.Anything, validNodeID).Return(gopowerstore.Host{ID: validHostID}, nil) + originalIsNodeConnectedToArrayFunc := isNodeConnectedToArrayFunc + isNodeConnectedToArrayFunc = func(_ context.Context, _ string, _ *array.PowerStoreArray) bool { + return true + } + defer func() { isNodeConnectedToArrayFunc = originalIsNodeConnectedToArrayFunc }() + iscsiConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything).Return(gobrick.Device{}, nil) + scsiStageVolumeOK(utilMock, fsMock) + + // RemoteVolume get fails. So remoteVolume stage fails. + clientMock.On("GetVolume", mock.Anything, "00000000-0000-0000-0000-000000000002").Return(gopowerstore.Volume{}, errors.New("some-error")) + checkMetroStateFunc = func(_ context.Context, _ array.VolumeHandle, _ gopowerstore.Client, _ gopowerstore.Client) (*array.MetroFracturedResponse, bool, error) { + return &array.MetroFracturedResponse{IsFractured: true, State: "Promoted"}, false, nil + } + createOrUpdateJournalEntryFunc = func(_ context.Context, _ string, _ array.VolumeHandle, _ string, _ string, _ string, _ []byte) error { + return nil + } + + res, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ + VolumeId: validMetroVolumeHandle, + PublishContext: getValidPublishContext(), + StagingTargetPath: nodeStagePrivateDir, + VolumeCapability: getCapabilityWithVoltypeAccessFstype( + "mount", "single-writer", "ext4"), + }) + + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeStageVolumeResponse{})) + }) + + ginkgo.It("fails when both arrays are unreachable", func() { + // ensure checking metro state will fail + clientMock.On("GetVolume", mock.Anything, validBaseVolumeID).Return(gopowerstore.Volume{}, errors.New("source array offline")) + clientMock.On("GetVolume", mock.Anything, validRemoteBaseVolumeID).Return(gopowerstore.Volume{}, errors.New("target array offline")) + + originalIsNodeConnectedToArrayFunc := isNodeConnectedToArrayFunc + isNodeConnectedToArrayFunc = func(_ context.Context, _ string, _ *array.PowerStoreArray) bool { + return true + } + defer func() { isNodeConnectedToArrayFunc = originalIsNodeConnectedToArrayFunc }() + + setDefaultClientMocks() + clientMock.On("GetHostByName", mock.Anything, validNodeID).Return(gopowerstore.Host{ID: validHostID}, nil) + iscsiConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything).Return(gobrick.Device{}, nil) + scsiStageVolumeOK(utilMock, fsMock) + // RemoteVolume get fails. So remoteVolume stage fails. + clientMock.On("GetVolume", mock.Anything, "00000000-0000-0000-0000-000000000002").Return(gopowerstore.Volume{}, errors.New("some-error")) + checkMetroStateFunc = func(_ context.Context, _ array.VolumeHandle, _ gopowerstore.Client, _ gopowerstore.Client) (*array.MetroFracturedResponse, bool, error) { + return nil, false, fmt.Errorf("failed to get metro session info") + } + + res, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ + VolumeId: validMetroVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: nodeStagePrivateDir, VolumeCapability: getCapabilityWithVoltypeAccessFstype( "mount", "single-writer", "ext4"), }) - Expect(err).ToNot(BeNil()) - Expect(res).To(BeNil()) - Expect(err.Error()).To(ContainSubstring("error bind disk")) + + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(res).To(gomega.BeNil()) + }) + + ginkgo.It("creates a journal entry when staging local fails", func() { + // when checking metro state, indicate it is fractured and local is demoted + clientMock.On("GetVolume", mock.Anything, validBaseVolumeID).Return(gopowerstore.Volume{ + ID: validBaseVolumeID, + MetroReplicationSessionID: validMetroSessionID, + }, nil) + clientMock.On("GetReplicationSessionByID", mock.Anything, validMetroSessionID).Return(gopowerstore.ReplicationSession{ + State: "Fractured", + LocalResourceState: "Demoted", + }, nil) + + clientMock.On("GetVolume", mock.Anything, validRemoteBaseVolumeID).Return(gopowerstore.Volume{ + ID: validRemoteBaseVolumeID, + MetroReplicationSessionID: validMetroSessionID, + }, nil) + + originalIsNodeConnectedToArrayFunc := isNodeConnectedToArrayFunc + isNodeConnectedToArrayFunc = func(_ context.Context, _ string, _ *array.PowerStoreArray) bool { + return true + } + defer func() { isNodeConnectedToArrayFunc = originalIsNodeConnectedToArrayFunc }() + + createOrUpdateJournalEntryFunc = func(_ context.Context, _ string, _ array.VolumeHandle, _ string, _ string, _ string, _ []byte) error { + return nil + } + + setDefaultClientMocks() + iscsiConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything).Return(gobrick.Device{}, nil) + scsiStageVolumeFail(utilMock, fsMock) + scsiStageRemoteMetroVolumeOK(utilMock, fsMock) + + res, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ + VolumeId: validMetroVolumeHandle, + PublishContext: getValidUniformMetroPublishContext(), + StagingTargetPath: nodeStagePrivateDir, + VolumeCapability: getCapabilityWithVoltypeAccessFstype( + "mount", "single-writer", "ext4"), + }) + + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeStageVolumeResponse{})) + }) + + ginkgo.It("creates a journal entry when staging remote fails", func() { + // when checking metro state, indicate it is fractured and local is promoted + clientMock.On("GetVolume", mock.Anything, validBaseVolumeID).Return(gopowerstore.Volume{ + ID: validBaseVolumeID, + MetroReplicationSessionID: validMetroSessionID, + }, nil) + clientMock.On("GetReplicationSessionByID", mock.Anything, validMetroSessionID).Return(gopowerstore.ReplicationSession{ + State: "Fractured", + LocalResourceState: "Promoted", + }, nil) + + clientMock.On("GetVolume", mock.Anything, validRemoteBaseVolumeID).Return(gopowerstore.Volume{ + ID: validRemoteBaseVolumeID, + MetroReplicationSessionID: validMetroSessionID, + }, nil) + + createOrUpdateJournalEntryFunc = func(_ context.Context, _ string, _ array.VolumeHandle, _ string, _ string, _ string, _ []byte) error { + return nil + } + originalIsNodeConnectedToArrayFunc := isNodeConnectedToArrayFunc + isNodeConnectedToArrayFunc = func(_ context.Context, _ string, _ *array.PowerStoreArray) bool { + return true + } + defer func() { isNodeConnectedToArrayFunc = originalIsNodeConnectedToArrayFunc }() + + setDefaultClientMocks() + iscsiConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything).Return(gobrick.Device{}, nil) + // successfully stage the local + scsiStageVolumeOK(utilMock, fsMock) + // fail staging the remote + scsiStageVolumeFail(utilMock, fsMock) + + res, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ + VolumeId: validMetroVolumeHandle, + PublishContext: getValidUniformMetroPublishContext(), + StagingTargetPath: nodeStagePrivateDir, + VolumeCapability: getCapabilityWithVoltypeAccessFstype( + "mount", "single-writer", "ext4"), + }) + + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeStageVolumeResponse{})) + }) + + ginkgo.It("fails to create journal entry", func() { + // stage the local side, should fail on the remote, triggering the creation of the journal entry + clientMock.On("GetVolume", mock.Anything, validBaseVolumeID).Return(gopowerstore.Volume{ + ID: validBaseVolumeID, + MetroReplicationSessionID: validMetroSessionID, + }, nil) + clientMock.On("GetReplicationSessionByID", mock.Anything, validMetroSessionID).Return(gopowerstore.ReplicationSession{ + State: "Fractured", + LocalResourceState: "Promoted", + }, nil) + + clientMock.On("GetVolume", mock.Anything, validRemoteBaseVolumeID).Return(gopowerstore.Volume{ + ID: validRemoteBaseVolumeID, + MetroReplicationSessionID: validMetroSessionID, + }, nil) + + originalIsNodeConnectedToArrayFunc := isNodeConnectedToArrayFunc + isNodeConnectedToArrayFunc = func(_ context.Context, _ string, _ *array.PowerStoreArray) bool { + return true + } + defer func() { isNodeConnectedToArrayFunc = originalIsNodeConnectedToArrayFunc }() + + setDefaultClientMocks() + clientMock.On("GetHostByName", mock.Anything, validNodeID).Return(gopowerstore.Host{ID: validHostID}, nil) + iscsiConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything).Return(gobrick.Device{}, nil) + scsiStageVolumeOK(utilMock, fsMock) + scsiStageVolumeFail(utilMock, fsMock) + + createOrUpdateJournalEntryFunc = func(_ context.Context, _ string, _ array.VolumeHandle, _ string, _ string, _ string, _ []byte) error { + return fmt.Errorf("unable to create journal entry") + } + + res, err := nodeSvc.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ + VolumeId: validMetroVolumeHandle, + PublishContext: getValidUniformMetroPublishContext(), + StagingTargetPath: nodeStagePrivateDir, + VolumeCapability: getCapabilityWithVoltypeAccessFstype( + "mount", "single-writer", "ext4"), + }) + + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(res).To(gomega.BeNil()) }) }) }) - Describe("calling NodeUnstage()", func() { + ginkgo.Describe("calling NodeUnstage()", func() { stagingPath := filepath.Join(nodeStagePrivateDir, validBaseVolumeID) - When("unstaging block volume", func() { - It("should succeed [iSCSI]", func() { + ginkgo.When("unstaging block volume", func() { + ginkgo.It("should succeed [iSCSI]", func() { mountInfo := []gofsutil.Info{ { Device: validDevName, Path: stagingPath, }, } - + clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ + Description: "", + ID: validBlockVolumeHandle, + Name: "name", + Size: controller.MaxVolumeSizeBytes / 200, + }, nil) fsMock.On("GetUtil").Return(utilMock) fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(2) fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return(mountInfo, nil) - + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) utilMock.On("Unmount", mock.Anything, stagingPath).Return(nil) fsMock.On("Remove", stagingPath).Return(nil) - fsMock.On("WriteFile", path.Join(nodeSvc.opts.TmpDir, validBaseVolumeID), []byte(validDevName), os.FileMode(0640)).Return(nil) + fsMock.On("WriteFile", path.Join(nodeSvc.opts.TmpDir, validBaseVolumeID), []byte(validDevName), os.FileMode(0o640)).Return(nil) iscsiConnectorMock.On("DisconnectVolumeByDeviceName", mock.Anything, validDevName).Return(nil) @@ -1416,13 +2369,13 @@ var _ = Describe("CSINodeService", func() { fsMock.On("IsNotExist", mock.Anything).Return(false) res, err := nodeSvc.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, StagingTargetPath: nodeStagePrivateDir, }) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.NodeUnstageVolumeResponse{})) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeUnstageVolumeResponse{})) }) - It("should fail, no targetPath [iSCSI]", func() { + ginkgo.It("should fail, no targetPath [iSCSI]", func() { mountInfo := []gofsutil.Info{ { Device: validDevName, @@ -1433,11 +2386,11 @@ var _ = Describe("CSINodeService", func() { fsMock.On("GetUtil").Return(utilMock) fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(2) fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return(mountInfo, nil) - + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) utilMock.On("Unmount", mock.Anything, stagingPath).Return(nil) fsMock.On("Remove", stagingPath).Return(nil) - fsMock.On("WriteFile", path.Join(nodeSvc.opts.TmpDir, validBaseVolumeID), []byte(validDevName), os.FileMode(0640)).Return(nil) + fsMock.On("WriteFile", path.Join(nodeSvc.opts.TmpDir, validBaseVolumeID), []byte(validDevName), os.FileMode(0o640)).Return(nil) iscsiConnectorMock.On("DisconnectVolumeByDeviceName", mock.Anything, validDevName).Return(nil) @@ -1445,19 +2398,33 @@ var _ = Describe("CSINodeService", func() { fsMock.On("IsNotExist", mock.Anything).Return(false) _, err := nodeSvc.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, StagingTargetPath: "", }) - Expect(err.Error()).To(ContainSubstring("staging target path is required")) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("staging target path is required")) + }) + ginkgo.It("should fail, invalid array ID", func() { + _, err := nodeSvc.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ + VolumeId: invalidBlockVolumeID, + StagingTargetPath: nodeStagePrivateDir, + }) + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("can't find array with ID")) }) - It("should fail, because no mounts [iSCSI]", func() { + ginkgo.It("should fail, because no mounts [iSCSI]", func() { mountInfo := []gofsutil.Info{ { Device: validDevName, Path: stagingPath, }, } - + clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ + Description: "", + ID: validBlockVolumeHandle, + Name: "name", + Size: controller.MaxVolumeSizeBytes / 200, + }, nil) + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) fsMock.On("GetUtil").Return(utilMock) fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, errors.New("fail")) fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return(mountInfo, nil) @@ -1465,7 +2432,7 @@ var _ = Describe("CSINodeService", func() { utilMock.On("Unmount", mock.Anything, stagingPath).Return(nil) fsMock.On("Remove", stagingPath).Return(nil) - fsMock.On("WriteFile", path.Join(nodeSvc.opts.TmpDir, validBaseVolumeID), []byte(validDevName), os.FileMode(0640)).Return(nil) + fsMock.On("WriteFile", path.Join(nodeSvc.opts.TmpDir, validBaseVolumeID), []byte(validDevName), os.FileMode(0o640)).Return(nil) iscsiConnectorMock.On("DisconnectVolumeByDeviceName", mock.Anything, validDevName).Return(nil) @@ -1473,19 +2440,25 @@ var _ = Describe("CSINodeService", func() { fsMock.On("IsNotExist", mock.Anything).Return(false) _, err := nodeSvc.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, StagingTargetPath: nodeStagePrivateDir, }) - Expect(err.Error()).To(ContainSubstring("could not reliably determine existing mount for path")) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("could not reliably determine existing mount for path")) }) - It("should fail, failed to unmount [iSCSI]", func() { + ginkgo.It("should fail, failed to unmount [iSCSI]", func() { mountInfo := []gofsutil.Info{ { Device: validDevName, Path: stagingPath, }, } - + clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ + Description: "", + ID: validBlockVolumeHandle, + Name: "name", + Size: controller.MaxVolumeSizeBytes / 200, + }, nil) + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) fsMock.On("GetUtil").Return(utilMock) fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(2) fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return(mountInfo, nil) @@ -1493,7 +2466,7 @@ var _ = Describe("CSINodeService", func() { utilMock.On("Unmount", mock.Anything, stagingPath).Return(errors.New("failed unmount")) fsMock.On("Remove", stagingPath).Return(nil) - fsMock.On("WriteFile", path.Join(nodeSvc.opts.TmpDir, validBaseVolumeID), []byte(validDevName), os.FileMode(0640)).Return(nil) + fsMock.On("WriteFile", path.Join(nodeSvc.opts.TmpDir, validBaseVolumeID), []byte(validDevName), os.FileMode(0o640)).Return(nil) iscsiConnectorMock.On("DisconnectVolumeByDeviceName", mock.Anything, validDevName).Return(nil) @@ -1501,19 +2474,24 @@ var _ = Describe("CSINodeService", func() { fsMock.On("IsNotExist", mock.Anything).Return(false) _, err := nodeSvc.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, StagingTargetPath: nodeStagePrivateDir, }) - Expect(err.Error()).To(ContainSubstring("could not unmount de")) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("could not unmount de")) }) - It("should succeed, without path in mouninfo [iSCSI]", func() { + ginkgo.It("should succeed, without path in mouninfo [iSCSI]", func() { mountInfo := []gofsutil.Info{ { Device: validDevName, Path: "invalid", }, } - + clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ + Description: "", + ID: validBlockVolumeHandle, + Name: "name", + Size: controller.MaxVolumeSizeBytes / 200, + }, nil) fsMock.On("GetUtil").Return(utilMock) fsMock.On("ReadFile", mock.Anything).Return([]byte{}, nil) fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return(mountInfo, nil) @@ -1521,7 +2499,7 @@ var _ = Describe("CSINodeService", func() { utilMock.On("Unmount", mock.Anything, stagingPath).Return(nil) fsMock.On("Remove", stagingPath).Return(nil) - fsMock.On("WriteFile", path.Join(nodeSvc.opts.TmpDir, validBaseVolumeID), []byte(validDevName), os.FileMode(0640)).Return(nil) + fsMock.On("WriteFile", path.Join(nodeSvc.opts.TmpDir, validBaseVolumeID), []byte(validDevName), os.FileMode(0o640)).Return(nil) iscsiConnectorMock.On("DisconnectVolumeByDeviceName", mock.Anything, validDevName).Return(nil) @@ -1529,14 +2507,52 @@ var _ = Describe("CSINodeService", func() { fsMock.On("IsNotExist", mock.Anything).Return(false) res, err := nodeSvc.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, + StagingTargetPath: nodeStagePrivateDir, + }) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeUnstageVolumeResponse{})) + }) + + ginkgo.It("should succeed for Metro volume [iSCSI]", func() { + mountInfo := []gofsutil.Info{{Device: validDevName, Path: stagingPath}} + remoteStagingPath := filepath.Join(nodeStagePrivateDir, validRemoteVolID) + remoteMountInfo := []gofsutil.Info{{Device: validDevName, Path: remoteStagingPath}} + clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ + Description: "", + ID: validBlockVolumeHandle, + Name: "name", + Size: controller.MaxVolumeSizeBytes / 200, + }, nil) + fsMock.On("GetUtil").Return(utilMock) + fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(4) + fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return(mountInfo, nil).Once() + fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return(remoteMountInfo, nil).Once() + + utilMock.On("Unmount", mock.Anything, stagingPath).Return(nil).Once() + utilMock.On("Unmount", mock.Anything, remoteStagingPath).Return(nil).Once() + fsMock.On("Remove", stagingPath).Return(nil) + fsMock.On("Remove", remoteStagingPath).Return(nil) + fsMock.On("WriteFile", path.Join(nodeSvc.opts.TmpDir, validBaseVolumeID), []byte(validDevName), os.FileMode(0o640)).Return(nil) + fsMock.On("WriteFile", path.Join(nodeSvc.opts.TmpDir, validRemoteVolID), []byte(validDevName), os.FileMode(0o640)).Return(nil) + + iscsiConnectorMock.On("DisconnectVolumeByDeviceName", mock.Anything, validDevName).Return(nil) + + fsMock.On("Remove", path.Join(nodeSvc.opts.TmpDir, validBaseVolumeID)).Return(nil).Once() + fsMock.On("Remove", path.Join(nodeSvc.opts.TmpDir, validRemoteVolID)).Return(nil).Once() + fsMock.On("IsNotExist", mock.Anything).Return(false) + + metroVolumeID := fmt.Sprintf("%s:%s/%s", validBlockVolumeHandle, validRemoteVolID, secondValidIP) + res, err := nodeSvc.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ + VolumeId: metroVolumeID, StagingTargetPath: nodeStagePrivateDir, }) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.NodeUnstageVolumeResponse{})) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeUnstageVolumeResponse{})) }) - It("should succeed [FC]", func() { - nodeSvc.useFC = true + + ginkgo.It("should succeed [FC]", func() { + nodeSvc.useFC[firstGlobalID] = true mountInfo := []gofsutil.Info{ { Device: validDevName, @@ -1544,6 +2560,14 @@ var _ = Describe("CSINodeService", func() { }, } + clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ + Description: "", + ID: validBlockVolumeHandle, + Name: "name", + Size: controller.MaxVolumeSizeBytes / 200, + Wwn: "naa.68ccf09800e23ab798312a05426acae0", + }, nil) + fsMock.On("GetUtil").Return(utilMock) fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(2) fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return(mountInfo, nil) @@ -1551,30 +2575,36 @@ var _ = Describe("CSINodeService", func() { utilMock.On("Unmount", mock.Anything, stagingPath).Return(nil) fsMock.On("Remove", stagingPath).Return(nil) - fsMock.On("WriteFile", path.Join(nodeSvc.opts.TmpDir, validBaseVolumeID), []byte(validDevName), os.FileMode(0640)).Return(nil) - + fsMock.On("WriteFile", path.Join(nodeSvc.opts.TmpDir, validBaseVolumeID), []byte(validDevName), os.FileMode(0o640)).Return(nil) + fcConnectorMock.On("DisconnectVolumeByWWN", mock.Anything, validDeviceWWN).Return(errors.New("mock disconnect failure")).Once() + fcConnectorMock.On("DisconnectVolumeByWWN", mock.Anything, validDeviceWWN).Return(nil) fcConnectorMock.On("DisconnectVolumeByDeviceName", mock.Anything, validDevName).Return(nil) fsMock.On("Remove", path.Join(nodeSvc.opts.TmpDir, validBaseVolumeID)).Return(nil) fsMock.On("IsNotExist", mock.Anything).Return(false) res, err := nodeSvc.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, StagingTargetPath: nodeStagePrivateDir, }) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.NodeUnstageVolumeResponse{})) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeUnstageVolumeResponse{})) }) - It("should succeed [NVMe]", func() { - nodeSvc.useNVME = true + ginkgo.It("should succeed [NVMe]", func() { + nodeSvc.useNVME[firstGlobalID] = true mountInfo := []gofsutil.Info{ { Device: validDevName, Path: stagingPath, }, } - + clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ + Description: "", + ID: validBlockVolumeHandle, + Name: "name", + Size: controller.MaxVolumeSizeBytes / 200, + }, nil) fsMock.On("GetUtil").Return(utilMock) fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(2) fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return(mountInfo, nil) @@ -1582,7 +2612,7 @@ var _ = Describe("CSINodeService", func() { utilMock.On("Unmount", mock.Anything, stagingPath).Return(nil) fsMock.On("Remove", stagingPath).Return(nil) - fsMock.On("WriteFile", path.Join(nodeSvc.opts.TmpDir, validBaseVolumeID), []byte(validDevName), os.FileMode(0640)).Return(nil) + fsMock.On("WriteFile", path.Join(nodeSvc.opts.TmpDir, validBaseVolumeID), []byte(validDevName), os.FileMode(0o640)).Return(nil) nvmeConnectorMock.On("DisconnectVolumeByDeviceName", mock.Anything, validDevName).Return(nil) @@ -1590,13 +2620,13 @@ var _ = Describe("CSINodeService", func() { fsMock.On("IsNotExist", mock.Anything).Return(false) res, err := nodeSvc.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, StagingTargetPath: nodeStagePrivateDir, }) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.NodeUnstageVolumeResponse{})) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeUnstageVolumeResponse{})) }) - It("should succeed, on device or resource busy error", func() { + ginkgo.It("should succeed, on device or resource busy error", func() { remnantStagingPath := "/noderoot/" + stagingPath mountInfo := []gofsutil.Info{ { @@ -1607,6 +2637,12 @@ var _ = Describe("CSINodeService", func() { Path: remnantStagingPath, }, } + clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ + Description: "", + ID: validBlockVolumeHandle, + Name: "name", + Size: controller.MaxVolumeSizeBytes / 200, + }, nil) fsMock.On("GetUtil").Return(utilMock) fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(4) @@ -1620,55 +2656,106 @@ var _ = Describe("CSINodeService", func() { fsMock.On("Remove", stagingPath).Return(nil).Once() fsMock.On("IsNotExist", mock.Anything).Return(false) - fsMock.On("WriteFile", path.Join(nodeSvc.opts.TmpDir, validBaseVolumeID), []byte(validDevName), os.FileMode(0640)).Return(nil) + fsMock.On("WriteFile", path.Join(nodeSvc.opts.TmpDir, validBaseVolumeID), []byte(validDevName), os.FileMode(0o640)).Return(nil) iscsiConnectorMock.On("DisconnectVolumeByDeviceName", mock.Anything, validDevName).Return(nil) fsMock.On("Remove", path.Join(nodeSvc.opts.TmpDir, validBaseVolumeID)).Return(nil) res, err := nodeSvc.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, StagingTargetPath: nodeStagePrivateDir, }) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.NodeUnstageVolumeResponse{})) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeUnstageVolumeResponse{})) }) - }) - When("unstaging nfs volume", func() { - It("should succeed", func() { + ginkgo.It("should succeed when volume has already been deleted on array [iSCSI]", func() { mountInfo := []gofsutil.Info{ { Device: validDevName, Path: stagingPath, }, } - + clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{}, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{ + StatusCode: http.StatusNotFound, + }, + }) fsMock.On("GetUtil").Return(utilMock) fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(2) fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return(mountInfo, nil) - + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) utilMock.On("Unmount", mock.Anything, stagingPath).Return(nil) fsMock.On("Remove", stagingPath).Return(nil) + fsMock.On("WriteFile", path.Join(nodeSvc.opts.TmpDir, validBaseVolumeID), []byte(validDevName), os.FileMode(0o640)).Return(nil) + + iscsiConnectorMock.On("DisconnectVolumeByDeviceName", mock.Anything, validDevName).Return(nil) + + fsMock.On("Remove", path.Join(nodeSvc.opts.TmpDir, validBaseVolumeID)).Return(nil) + fsMock.On("IsNotExist", mock.Anything).Return(false) res, err := nodeSvc.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ - VolumeId: validNfsVolumeID, + VolumeId: validBlockVolumeHandle, StagingTargetPath: nodeStagePrivateDir, }) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.NodeUnstageVolumeResponse{})) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeUnstageVolumeResponse{})) }) - }) - }) - - Describe("calling NodePublish()", func() { - stagingPath := filepath.Join(validStagingPath, validBaseVolumeID) - - When("publishing block volume as mount", func() { - It("should succeed", func() { - fsMock.On("GetUtil").Return(utilMock) - fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(2) - fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return([]gofsutil.Info{}, nil) + ginkgo.It("should failed due to inability to get volume [iSCSI]", func() { + clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{}, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{ + StatusCode: http.StatusBadGateway, + }, + }) + + _, err := nodeSvc.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ + VolumeId: validBlockVolumeHandle, + StagingTargetPath: nodeStagePrivateDir, + }) + gomega.Expect(err).ToNot(gomega.BeNil()) + }) + }) + ginkgo.When("unstaging nfs volume", func() { + ginkgo.It("should succeed", func() { + mountInfo := []gofsutil.Info{ + { + Device: validDevName, + Path: stagingPath, + }, + } + clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ + Description: "", + ID: validBlockVolumeHandle, + Name: "name", + Size: controller.MaxVolumeSizeBytes / 200, + }, nil) + fsMock.On("GetUtil").Return(utilMock) + fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(2) + fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return(mountInfo, nil) + + utilMock.On("Unmount", mock.Anything, stagingPath).Return(nil) + + fsMock.On("Remove", stagingPath).Return(nil) + + res, err := nodeSvc.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ + VolumeId: validNfsVolumeID, + StagingTargetPath: nodeStagePrivateDir, + }) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeUnstageVolumeResponse{})) + }) + }) + }) + + ginkgo.Describe("calling NodePublish()", func() { + stagingPath := filepath.Join(validStagingPath, validBaseVolumeID) + + ginkgo.When("publishing block volume as mount", func() { + ginkgo.It("should succeed", func() { + fsMock.On("GetUtil").Return(utilMock) + fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(2) + fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return([]gofsutil.Info{}, nil) fsMock.On("MkdirAll", validTargetPath, mock.Anything).Return(nil) utilMock.On("GetDiskFormat", mock.Anything, stagingPath).Return("", nil) @@ -1676,19 +2763,19 @@ var _ = Describe("CSINodeService", func() { utilMock.On("Mount", mock.Anything, stagingPath, validTargetPath, "").Return(nil) res, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, VolumeCapability: getCapabilityWithVoltypeAccessFstype("mount", "single-writer", ""), Readonly: false, }) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.NodePublishVolumeResponse{})) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodePublishVolumeResponse{})) }) }) - When("publishing block volume as mount with RO", func() { - It("should fail", func() { + ginkgo.When("publishing block volume as mount with RO", func() { + ginkgo.It("should fail", func() { fsMock.On("GetUtil").Return(utilMock) fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(2) fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return([]gofsutil.Info{}, nil) @@ -1699,18 +2786,18 @@ var _ = Describe("CSINodeService", func() { utilMock.On("Mount", mock.Anything, stagingPath, validTargetPath, "").Return(nil) _, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, VolumeCapability: getCapabilityWithVoltypeAccessFstype("mount", "single-writer", ""), Readonly: true, }) - Expect(err.Error()).To(ContainSubstring("RO mount required but no fs detected on staged volume")) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("RO mount required but no fs detected on staged volume")) }) }) - When("publishing block volume as mount with RO, fs exists", func() { - It("should succeed", func() { + ginkgo.When("publishing block volume as mount with RO, fs exists", func() { + ginkgo.It("should succeed", func() { fsMock.On("GetUtil").Return(utilMock) fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(2) fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return([]gofsutil.Info{}, nil) @@ -1721,18 +2808,18 @@ var _ = Describe("CSINodeService", func() { utilMock.On("Mount", mock.Anything, stagingPath, validTargetPath, "ext4", "ro").Return(nil) _, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, VolumeCapability: getCapabilityWithVoltypeAccessFstype("mount", "single-writer", "ext4"), Readonly: true, }) - Expect(err).To(BeNil()) + gomega.Expect(err).To(gomega.BeNil()) }) }) - When("publishing block volume as mount and unable to create dirs", func() { - It("should fail", func() { + ginkgo.When("publishing block volume as mount and unable to create dirs", func() { + ginkgo.It("should fail", func() { fsMock.On("GetUtil").Return(utilMock) fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(2) fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return([]gofsutil.Info{}, nil) @@ -1743,18 +2830,18 @@ var _ = Describe("CSINodeService", func() { utilMock.On("Mount", mock.Anything, stagingPath, validTargetPath, "").Return(nil) _, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, - VolumeCapability: getCapabilityWithVoltypeAccessFstype("mount", "single-writer", ""), + VolumeCapability: getCapabilityWithVoltypeAccessFstype("mount", "multiple-reader", ""), Readonly: false, }) - Expect(err.Error()).To(ContainSubstring("can't create target dir")) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("can't create target dir")) }) }) - When("publishing block volume as mount and getformat fails", func() { - It("should fail", func() { + ginkgo.When("publishing block volume as mount and getformat fails", func() { + ginkgo.It("should fail", func() { fsMock.On("GetUtil").Return(utilMock) fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(2) fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return([]gofsutil.Info{}, nil) @@ -1765,18 +2852,18 @@ var _ = Describe("CSINodeService", func() { utilMock.On("Mount", mock.Anything, stagingPath, validTargetPath, "").Return(nil) _, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, VolumeCapability: getCapabilityWithVoltypeAccessFstype("mount", "single-writer", ""), Readonly: false, }) - Expect(err.Error()).To(ContainSubstring("error while trying to detect fs")) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("error while trying to detect fs")) }) }) - When("publishing block volume as mount and disk preformatted", func() { - It("should fail", func() { + ginkgo.When("publishing block volume as mount and disk preformatted", func() { + ginkgo.It("should fail", func() { fsMock.On("GetUtil").Return(utilMock) fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(2) fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return([]gofsutil.Info{}, nil) @@ -1787,18 +2874,18 @@ var _ = Describe("CSINodeService", func() { utilMock.On("Mount", mock.Anything, stagingPath, validTargetPath, "ext4").Return(nil) _, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, VolumeCapability: getCapabilityWithVoltypeAccessFstype("mount", "single-writer", "xfs"), Readonly: false, }) - Expect(err.Error()).To(ContainSubstring("Target device already formatted")) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("Target device already formatted")) }) }) - When("publishing formatting failed", func() { - It("should succeed", func() { + ginkgo.When("publishing formatting failed", func() { + ginkgo.It("should succeed", func() { fsMock.On("GetUtil").Return(utilMock) fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(2) fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return([]gofsutil.Info{}, nil) @@ -1809,18 +2896,43 @@ var _ = Describe("CSINodeService", func() { utilMock.On("Mount", mock.Anything, stagingPath, validTargetPath, "").Return(nil) _, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, VolumeCapability: getCapabilityWithVoltypeAccessFstype("mount", "single-writer", ""), Readonly: false, }) - Expect(err.Error()).To(ContainSubstring("can't format staged device")) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("can't format staged device")) + }) + }) + ginkgo.When("publishing block volume as mount with multi-writer", func() { + ginkgo.It("should succeed", func() { + fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil) + fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return([]gofsutil.Info{}, nil) + fsMock.On("GetUtil").Return(utilMock) + fsMock.On("MkFileIdempotent", validTargetPath).Return(true, nil) + utilMock.On("BindMount", mock.Anything, stagingPath, validTargetPath).Return(nil) + fsMock.On("MkdirAll", validTargetPath, mock.Anything).Return(nil) + utilMock.On("GetDiskFormat", mock.Anything, stagingPath).Return("", nil) + fsMock.On("ExecCommand", "mkfs.ext4", "-E", "nodiscard", "-F", stagingPath).Return([]byte{}, nil) + utilMock.On("Mount", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) + utilMock.On("Unmount", mock.Anything, mock.Anything).Return(nil) + fsMock.On("RemoveAll", mock.Anything).Return(nil) + + _, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ + VolumeId: validBlockVolumeHandle, + PublishContext: getValidPublishContext(), + StagingTargetPath: validStagingPath, + TargetPath: validTargetPath, + VolumeCapability: getCapabilityWithVoltypeAccessFstype("mount", "multiple-writer", ""), + Readonly: false, + }) + gomega.Expect(err).To(gomega.BeNil()) }) }) - When("publishing block volume as raw block", func() { - It("should succeed", func() { + ginkgo.When("publishing block volume as raw block", func() { + ginkgo.It("should succeed", func() { fsMock.On("GetUtil").Return(utilMock) fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(2) fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return([]gofsutil.Info{}, nil) @@ -1829,19 +2941,19 @@ var _ = Describe("CSINodeService", func() { utilMock.On("BindMount", mock.Anything, stagingPath, validTargetPath).Return(nil) res, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, VolumeCapability: getCapabilityWithVoltypeAccessFstype("block", "single-writer", "ext4"), Readonly: false, }) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.NodePublishVolumeResponse{})) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodePublishVolumeResponse{})) }) }) - When("publishing block volume as raw block with RO", func() { - It("should fail", func() { + ginkgo.When("publishing block volume as raw block with RO", func() { + ginkgo.It("should fail", func() { fsMock.On("GetUtil").Return(utilMock) fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(2) fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return([]gofsutil.Info{}, nil) @@ -1850,18 +2962,18 @@ var _ = Describe("CSINodeService", func() { utilMock.On("BindMount", mock.Anything, stagingPath, validTargetPath, "ro").Return(nil) _, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, VolumeCapability: getCapabilityWithVoltypeAccessFstype("block", "single-writer", "ext4"), Readonly: true, }) - Expect(err.Error()).To(ContainSubstring("read only not supported for Block Volume")) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("read only not supported for Block Volume")) }) }) - When("publishing block and unable to create target", func() { - It("should fail", func() { + ginkgo.When("publishing block and unable to create target", func() { + ginkgo.It("should fail", func() { fsMock.On("GetUtil").Return(utilMock) fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(2) fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return([]gofsutil.Info{}, nil) @@ -1870,18 +2982,18 @@ var _ = Describe("CSINodeService", func() { utilMock.On("BindMount", mock.Anything, stagingPath, validTargetPath).Return(nil) _, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, VolumeCapability: getCapabilityWithVoltypeAccessFstype("block", "single-writer", "ext4"), Readonly: false, }) - Expect(err.Error()).To(ContainSubstring("can't create target file")) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("can't create target file")) }) }) - When("publishing block and unable to bind disk", func() { - It("should fail", func() { + ginkgo.When("publishing block and unable to bind disk", func() { + ginkgo.It("should fail", func() { fsMock.On("GetUtil").Return(utilMock) fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(2) fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return([]gofsutil.Info{}, nil) @@ -1890,18 +3002,56 @@ var _ = Describe("CSINodeService", func() { utilMock.On("BindMount", mock.Anything, stagingPath, validTargetPath).Return(errors.New("failed to bind")) _, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, + PublishContext: getValidPublishContext(), + StagingTargetPath: validStagingPath, + TargetPath: validTargetPath, + VolumeCapability: getCapabilityWithVoltypeAccessFstype("block", "single-writer", "ext4"), + Readonly: false, + }) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("error bind disk")) + }) + }) + ginkgo.When("publishing block and unable to get target mounts", func() { + ginkgo.It("should fail", func() { + fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, errors.New("fail")) + + _, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ + VolumeId: validBlockVolumeHandle, + PublishContext: getValidPublishContext(), + StagingTargetPath: validStagingPath, + TargetPath: validTargetPath, + VolumeCapability: getCapabilityWithVoltypeAccessFstype("block", "single-writer", "ext4"), + Readonly: false, + }) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("can't check mounts for path")) + }) + }) + ginkgo.When("publishing block but volume already mounted with different capabilities", func() { + ginkgo.It("should fail", func() { + mountInfo := []gofsutil.Info{ + { + Device: validDevName, + Path: validTargetPath, + Opts: []string{"ro"}, + }, + } + fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil) + fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return(mountInfo, nil) + + _, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, VolumeCapability: getCapabilityWithVoltypeAccessFstype("block", "single-writer", "ext4"), Readonly: false, }) - Expect(err.Error()).To(ContainSubstring("error bind disk")) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("volume already mounted but with different capabilities")) }) }) - When("publishing nfs volume", func() { - It("should succeed", func() { + ginkgo.When("publishing nfs volume", func() { + ginkgo.It("should succeed", func() { fsMock.On("GetUtil").Return(utilMock) fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(2) fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return([]gofsutil.Info{}, nil) @@ -1919,12 +3069,12 @@ var _ = Describe("CSINodeService", func() { VolumeCapability: getCapabilityWithVoltypeAccessFstype("mount", "multi-writer", "nfs"), Readonly: false, }) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.NodePublishVolumeResponse{})) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodePublishVolumeResponse{})) }) }) - When("No volume ID specified", func() { - It("should fail", func() { + ginkgo.When("No volume ID specified", func() { + ginkgo.It("should fail", func() { res, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ VolumeId: "", PublishContext: getValidPublishContext(), @@ -1933,54 +3083,54 @@ var _ = Describe("CSINodeService", func() { VolumeCapability: getCapabilityWithVoltypeAccessFstype("block", "single-writer", "ext4"), Readonly: false, }) - Expect(err.Error()).To(ContainSubstring("volume ID is required")) - Expect(res).To(BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("volume ID is required")) + gomega.Expect(res).To(gomega.BeNil()) }) }) - When("No target path specified", func() { - It("should fail", func() { + ginkgo.When("No target path specified", func() { + ginkgo.It("should fail", func() { res, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: "", VolumeCapability: getCapabilityWithVoltypeAccessFstype("block", "single-writer", "ext4"), Readonly: false, }) - Expect(err.Error()).To(ContainSubstring("targetPath is required")) - Expect(res).To(BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("targetPath is required")) + gomega.Expect(res).To(gomega.BeNil()) }) }) - When("Invalid volume capabilities specified", func() { - It("should fail", func() { + ginkgo.When("Invalid volume capabilities specified", func() { + ginkgo.It("should fail", func() { res, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, VolumeCapability: nil, Readonly: false, }) - Expect(err.Error()).To(ContainSubstring("VolumeCapability is required")) - Expect(res).To(BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("VolumeCapability is required")) + gomega.Expect(res).To(gomega.BeNil()) }) }) - When("No staging target path specified", func() { - It("should fail", func() { + ginkgo.When("No staging target path specified", func() { + ginkgo.It("should fail", func() { res, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: "", TargetPath: validTargetPath, VolumeCapability: getCapabilityWithVoltypeAccessFstype("block", "single-writer", "ext4"), Readonly: false, }) - Expect(err.Error()).To(ContainSubstring("stagingPath is required")) - Expect(res).To(BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("stagingPath is required")) + gomega.Expect(res).To(gomega.BeNil()) }) }) - When("unable to create target dir", func() { - It("should fail", func() { + ginkgo.When("unable to create target dir", func() { + ginkgo.It("should fail", func() { fsMock.On("GetUtil").Return(utilMock) fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(2) fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return([]gofsutil.Info{}, nil) @@ -1998,11 +3148,11 @@ var _ = Describe("CSINodeService", func() { VolumeCapability: getCapabilityWithVoltypeAccessFstype("mount", "multi-writer", "nfs"), Readonly: true, }) - Expect(err.Error()).To(ContainSubstring("can't create target folder")) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("can't create target folder")) }) }) - When("publishing nfs with ro", func() { - It("should succeed", func() { + ginkgo.When("publishing nfs with ro", func() { + ginkgo.It("should succeed", func() { fsMock.On("GetUtil").Return(utilMock) fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(2) fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return([]gofsutil.Info{}, nil) @@ -2020,12 +3170,12 @@ var _ = Describe("CSINodeService", func() { VolumeCapability: getCapabilityWithVoltypeAccessFstype("mount", "multi-writer", "nfs"), Readonly: true, }) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.NodePublishVolumeResponse{})) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodePublishVolumeResponse{})) }) }) - When("unable to bind disk", func() { - It("should fail", func() { + ginkgo.When("unable to bind disk", func() { + ginkgo.It("should fail", func() { fsMock.On("GetUtil").Return(utilMock) fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(2) fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return([]gofsutil.Info{}, nil) @@ -2043,14 +3193,14 @@ var _ = Describe("CSINodeService", func() { VolumeCapability: getCapabilityWithVoltypeAccessFstype("mount", "multi-writer", "nfs"), Readonly: true, }) - Expect(err.Error()).To(ContainSubstring("error bind disk")) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("error bind disk")) }) }) }) - Describe("calling NodeUnpublish()", func() { - When("unpublishing block volume", func() { - It("should succeed", func() { + ginkgo.Describe("calling NodeUnpublish()", func() { + ginkgo.When("unpublishing block volume", func() { + ginkgo.It("should succeed", func() { mountInfo := []gofsutil.Info{ { Device: validDevName, @@ -2064,17 +3214,18 @@ var _ = Describe("CSINodeService", func() { fsMock.On("Stat", mock.Anything).Return(&mocks.FileInfo{}, os.ErrNotExist) utilMock.On("Unmount", mock.Anything, validTargetPath).Return(nil) + fsMock.On("Remove", mock.Anything).Return(nil) res, err := nodeSvc.NodeUnpublishVolume(context.Background(), &csi.NodeUnpublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, TargetPath: validTargetPath, }) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.NodeUnpublishVolumeResponse{})) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeUnpublishVolumeResponse{})) }) }) - When("unpublishing nfs volume", func() { - It("should succeed", func() { + ginkgo.When("unpublishing nfs volume", func() { + ginkgo.It("should succeed", func() { mountInfo := []gofsutil.Info{ { Device: validDevName, @@ -2088,53 +3239,52 @@ var _ = Describe("CSINodeService", func() { fsMock.On("Stat", mock.Anything).Return(&mocks.FileInfo{}, os.ErrNotExist) utilMock.On("Unmount", mock.Anything, validTargetPath).Return(nil) + fsMock.On("Remove", mock.Anything).Return(nil) res, err := nodeSvc.NodeUnpublishVolume(context.Background(), &csi.NodeUnpublishVolumeRequest{ VolumeId: validNfsVolumeID, TargetPath: validTargetPath, }) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.NodeUnpublishVolumeResponse{})) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeUnpublishVolumeResponse{})) }) }) - When("No target path specified", func() { - It("should fail", func() { - + ginkgo.When("No target path specified", func() { + ginkgo.It("should fail", func() { res, err := nodeSvc.NodeUnpublishVolume(context.Background(), &csi.NodeUnpublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, TargetPath: "", }) - Expect(err.Error()).To(Equal("rpc error: code = InvalidArgument desc = target path required")) - Expect(res).To(BeNil()) + gomega.Expect(err.Error()).To(gomega.Equal("rpc error: code = InvalidArgument desc = target path required")) + gomega.Expect(res).To(gomega.BeNil()) }) }) - When("Unable to get volID", func() { - It("should fail", func() { - + ginkgo.When("Unable to get volID", func() { + ginkgo.It("should fail", func() { res, err := nodeSvc.NodeUnpublishVolume(context.Background(), &csi.NodeUnpublishVolumeRequest{ VolumeId: "", TargetPath: validTargetPath, }) - Expect(err.Error()).To(Equal("rpc error: code = InvalidArgument desc = volume ID is required")) - Expect(res).To(BeNil()) + gomega.Expect(err.Error()).To(gomega.Equal("rpc error: code = InvalidArgument desc = volume ID is required")) + gomega.Expect(res).To(gomega.BeNil()) }) }) - When("Unable to get TargetMounts", func() { - It("should fail", func() { + ginkgo.When("Unable to get TargetMounts", func() { + ginkgo.It("should fail", func() { fsMock.On("GetUtil").Return(utilMock) fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(2) fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return(nil, errors.New("error")) fsMock.On("Stat", mock.Anything).Return(&mocks.FileInfo{}, os.ErrNotExist) res, err := nodeSvc.NodeUnpublishVolume(context.Background(), &csi.NodeUnpublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, TargetPath: validTargetPath, }) - Expect(err.Error()).To(ContainSubstring("could not reliably determine existing mount status")) - Expect(res).To(BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("could not reliably determine existing mount status")) + gomega.Expect(res).To(gomega.BeNil()) }) }) - When("Unable to perform unmount", func() { - It("should fail", func() { + ginkgo.When("Unable to perform unmount", func() { + ginkgo.It("should fail", func() { mountInfo := []gofsutil.Info{ { Device: validDevName, @@ -2149,24 +3299,23 @@ var _ = Describe("CSINodeService", func() { fsMock.On("Stat", mock.Anything).Return(&mocks.FileInfo{}, os.ErrNotExist) utilMock.On("Unmount", mock.Anything, validTargetPath).Return(errors.New("Unmount failed")) res, err := nodeSvc.NodeUnpublishVolume(context.Background(), &csi.NodeUnpublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, TargetPath: validTargetPath, }) - Expect(err.Error()).To(ContainSubstring("could not unmount dev")) - Expect(res).To(BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("could not unmount dev")) + gomega.Expect(res).To(gomega.BeNil()) }) }) }) - Describe("calling NodeExpandVolume() online", func() { + ginkgo.Describe("calling NodeExpandVolume() online", func() { stagingPath := filepath.Join(validStagingPath, validBaseVolumeID) - When("everything is correct", func() { - It("should succeed [ext4]", func() { - + ginkgo.When("everything is correct", func() { + ginkgo.It("should succeed [ext4]", func() { fsMock.On("GetUtil").Return(utilMock) clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ Description: "", - ID: validBlockVolumeID, + ID: validBlockVolumeHandle, Name: "name", Size: controller.MaxVolumeSizeBytes / 200, }, nil) @@ -2179,16 +3328,38 @@ var _ = Describe("CSINodeService", func() { utilMock.On("FindFSType", mock.Anything, mock.Anything).Return("ext4", nil) fsMock.On("ExecCommandOutput", mock.Anything, mock.Anything, mock.Anything).Return([]byte("version 5.0.0"), nil) utilMock.On("ResizeFS", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) - res, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeID, false)) - Ω(err).To(BeNil()) - Ω(res).To(Equal(&csi.NodeExpandVolumeResponse{})) + res, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeHandle, false)) + gomega.Ω(err).To(gomega.BeNil()) + gomega.Ω(res).To(gomega.Equal(&csi.NodeExpandVolumeResponse{})) + }) + ginkgo.It("should succeed when Auth is enabled and volume has tenant prefix in it[ext4]", func() { + fsMock.On("GetUtil").Return(utilMock) + clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ + Description: "", + ID: validBlockVolumeHandle, + Name: "tn1-csivol-123456", + Size: controller.MaxVolumeSizeBytes / 200, + }, nil) + utilMock.On("GetMountInfoFromDevice", mock.Anything, mock.Anything).Return(&gofsutil.DeviceMountInfo{ + DeviceNames: []string{validDevName}, + MPathName: "", + MountPoint: stagingPath, + }, nil) + utilMock.On("DeviceRescan", mock.Anything, mock.Anything).Return(nil) + utilMock.On("FindFSType", mock.Anything, mock.Anything).Return("ext4", nil) + fsMock.On("ExecCommandOutput", mock.Anything, mock.Anything, mock.Anything).Return([]byte("version 5.0.0"), nil) + utilMock.On("ResizeFS", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) + os.Setenv("X_CSM_AUTH_ENABLED", "true") + res, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeHandle, false)) + gomega.Ω(err).To(gomega.BeNil()) + gomega.Ω(res).To(gomega.Equal(&csi.NodeExpandVolumeResponse{})) }) - It("should succeed [xfs]", func() { + ginkgo.It("should succeed [xfs]", func() { fsMock.On("GetUtil").Return(utilMock) clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ Description: "", - ID: validBlockVolumeID, + ID: validBlockVolumeHandle, Name: "name", Size: controller.MaxVolumeSizeBytes / 200, }, nil) @@ -2201,17 +3372,44 @@ var _ = Describe("CSINodeService", func() { utilMock.On("FindFSType", mock.Anything, mock.Anything).Return("xfs", nil) fsMock.On("ExecCommandOutput", mock.Anything, mock.Anything, mock.Anything).Return([]byte("version 5.0.0"), nil) utilMock.On("ResizeFS", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) - res, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeID, false)) - Ω(err).To(BeNil()) - Ω(res).To(Equal(&csi.NodeExpandVolumeResponse{})) + res, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeHandle, false)) + gomega.Ω(err).To(gomega.BeNil()) + gomega.Ω(res).To(gomega.Equal(&csi.NodeExpandVolumeResponse{})) + }) + ginkgo.It("should succeed [metro-volumes]", func() { + fsMock.On("GetUtil").Return(utilMock) + metroVolumeID := fmt.Sprintf("%s:%s/%s", validBlockVolumeHandle, validRemoteVolID, secondValidIP) + clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ + Description: "", + ID: metroVolumeID, + Name: "name", + Size: controller.MaxVolumeSizeBytes / 200, + MetroReplicationSessionID: validMetroSessionID, + }, nil) + clientMock.On("GetReplicationSessionByID", mock.Anything, validMetroSessionID).Return(gopowerstore.ReplicationSession{ + ID: validMetroSessionID, + State: gopowerstore.RsStateOk, + }, nil).Times(1) + utilMock.On("GetMountInfoFromDevice", mock.Anything, mock.Anything).Return(&gofsutil.DeviceMountInfo{ + DeviceNames: []string{validDevName}, + MPathName: "", + MountPoint: stagingPath, + }, nil) + utilMock.On("DeviceRescan", mock.Anything, mock.Anything).Return(nil) + utilMock.On("FindFSType", mock.Anything, mock.Anything).Return("ext4", nil) + fsMock.On("ExecCommandOutput", mock.Anything, mock.Anything, mock.Anything).Return([]byte("version 5.0.0"), nil) + utilMock.On("ResizeFS", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) + res, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(metroVolumeID, false)) + gomega.Ω(err).To(gomega.BeNil()) + gomega.Ω(res).To(gomega.Equal(&csi.NodeExpandVolumeResponse{})) }) }) - When("it failed to find mount info", func() { - It("should fail ResizeFS() [xfs]", func() { + ginkgo.When("it failed to find mount info", func() { + ginkgo.It("should fail ResizeFS() [xfs]", func() { fsMock.On("GetUtil").Return(utilMock) clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ Description: "", - ID: validBlockVolumeID, + ID: validBlockVolumeHandle, Name: "name", Size: controller.MaxVolumeSizeBytes / 200, }, nil) @@ -2224,15 +3422,15 @@ var _ = Describe("CSINodeService", func() { utilMock.On("FindFSType", mock.Anything, mock.Anything).Return("xfs", nil) fsMock.On("ExecCommandOutput", mock.Anything, mock.Anything, mock.Anything).Return([]byte("version 5.0.0"), nil) utilMock.On("ResizeFS", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(errors.New("resize Failed ext4")) - res, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeID, false)) - Ω(err.Error()).To(ContainSubstring("resize Failed ext4")) - Ω(res).To(BeNil()) + res, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeHandle, false)) + gomega.Ω(err.Error()).To(gomega.ContainSubstring("resize Failed ext4")) + gomega.Ω(res).To(gomega.BeNil()) }) - It("should fail ResizeFS() [ext4]", func() { + ginkgo.It("should fail ResizeFS() [ext4]", func() { fsMock.On("GetUtil").Return(utilMock) clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ Description: "", - ID: validBlockVolumeID, + ID: validBlockVolumeHandle, Name: "name", Size: controller.MaxVolumeSizeBytes / 200, }, nil) @@ -2246,23 +3444,21 @@ var _ = Describe("CSINodeService", func() { utilMock.On("FindFSType", mock.Anything, mock.Anything).Return("ext4", nil) fsMock.On("ExecCommandOutput", mock.Anything, mock.Anything, mock.Anything).Return([]byte("version 5.0.0"), nil) utilMock.On("ResizeFS", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(errors.New("resize Failed xfs")) - res, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeID, false)) - Ω(err.Error()).To(ContainSubstring("resize Failed xfs")) - Ω(res).To(BeNil()) + res, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeHandle, false)) + gomega.Ω(err.Error()).To(gomega.ContainSubstring("resize Failed xfs")) + gomega.Ω(res).To(gomega.BeNil()) }) }) - }) - Describe("calling NodeExpandVolume() offline", func() { + ginkgo.Describe("calling NodeExpandVolume() offline", func() { stagingPath := filepath.Join(validStagingPath, validBaseVolumeID) - When("everything is correct", func() { - It("should succeed [ext4]", func() { - + ginkgo.When("everything is correct", func() { + ginkgo.It("should succeed [ext4]", func() { fsMock.On("GetUtil").Return(utilMock) clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ Description: "", - ID: validBlockVolumeID, + ID: validBlockVolumeHandle, Name: "name", Size: controller.MaxVolumeSizeBytes / 200, }, nil) @@ -2286,18 +3482,17 @@ var _ = Describe("CSINodeService", func() { utilMock.On("FindFSType", mock.Anything, mock.Anything).Return("ext4", nil) fsMock.On("ExecCommandOutput", mock.Anything, mock.Anything, mock.Anything).Return([]byte("version 5.0.0"), nil) utilMock.On("ResizeFS", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) - res, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeID, false)) - Ω(err).To(BeNil()) - Ω(res).To(Equal(&csi.NodeExpandVolumeResponse{})) + res, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeHandle, false)) + gomega.Ω(err).To(gomega.BeNil()) + gomega.Ω(res).To(gomega.Equal(&csi.NodeExpandVolumeResponse{})) }) }) - When("using multipath", func() { - It("should succeed [ext4]", func() { - + ginkgo.When("using multipath", func() { + ginkgo.It("should succeed [ext4]", func() { fsMock.On("GetUtil").Return(utilMock) clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ Description: "", - ID: validBlockVolumeID, + ID: validBlockVolumeHandle, Name: "name", Size: controller.MaxVolumeSizeBytes / 200, }, nil) @@ -2322,19 +3517,18 @@ var _ = Describe("CSINodeService", func() { utilMock.On("FindFSType", mock.Anything, mock.Anything).Return("xfs", nil) fsMock.On("ExecCommandOutput", mock.Anything, mock.Anything, mock.Anything).Return([]byte("version 5.0.0"), nil) utilMock.On("ResizeFS", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) - res, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeID, false)) - Ω(err).To(BeNil()) - Ω(res).To(Equal(&csi.NodeExpandVolumeResponse{})) + res, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeHandle, false)) + gomega.Ω(err).To(gomega.BeNil()) + gomega.Ω(res).To(gomega.Equal(&csi.NodeExpandVolumeResponse{})) }) }) - When("using block mode", func() { - It("should succeed [ext4]", func() { - + ginkgo.When("using block mode", func() { + ginkgo.It("should succeed [ext4]", func() { fsMock.On("GetUtil").Return(utilMock) clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ Description: "", - ID: validBlockVolumeID, + ID: validBlockVolumeHandle, Name: "name", Size: controller.MaxVolumeSizeBytes / 200, Wwn: "naa.6090a038f0cd4e5bdaa8248e6856d4fe:3", @@ -2348,82 +3542,84 @@ var _ = Describe("CSINodeService", func() { utilMock.On("DeviceRescan", mock.Anything, mock.Anything).Return(nil) utilMock.On("GetMpathNameFromDevice", mock.Anything, mock.Anything).Return("mpatha", nil) utilMock.On("ResizeMultipath", mock.Anything, mock.Anything).Return(nil) - res, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeID, true)) - Ω(err).To(BeNil()) - Ω(res).To(Equal(&csi.NodeExpandVolumeResponse{})) + res, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeHandle, true)) + gomega.Ω(err).To(gomega.BeNil()) + gomega.Ω(res).To(gomega.Equal(&csi.NodeExpandVolumeResponse{})) }) }) - When("Unable to parse volid", func() { - It("should fail", func() { - clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ - Description: "", - ID: validBlockVolumeID, - Name: "name", - Size: controller.MaxVolumeSizeBytes / 200, - Wwn: "naa.6090a038f0cd4e5bdaa8248e6856d4fe:3", - }, nil) + ginkgo.When("using NFS mode", func() { + // workaround for https://github.com/kubernetes/kubernetes/issues/131419 + ginkgo.It("should succeed [nfs]", func() { + // nothing to mock as it is a NO-OP + res, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validNfsVolumeID, false)) + gomega.Ω(err).To(gomega.BeNil()) + gomega.Ω(res).To(gomega.Equal(&csi.NodeExpandVolumeResponse{})) + }) + }) + ginkgo.When("the request is missing the Volume ID", func() { + ginkgo.It("should fail", func() { _, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest("", true)) - Ω(err.Error()).To(ContainSubstring("incorrect volume id")) - + gomega.Ω(err.Error()).To(gomega.ContainSubstring("unable to parse volume handle. volumeHandle is empty")) }) }) - When("no target path", func() { - It("should fail", func() { - + ginkgo.When("the array ID is not valid", func() { + ginkgo.It("should fail", func() { + _, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(invalidBlockVolumeID, true)) + gomega.Ω(err.Error()).To(gomega.ContainSubstring("failed to find array with given ID")) + }) + }) + ginkgo.When("no target path", func() { + ginkgo.It("should fail", func() { fsMock.On("GetUtil").Return(utilMock) clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ Description: "", - ID: validBlockVolumeID, + ID: validBlockVolumeHandle, Name: "name", Size: controller.MaxVolumeSizeBytes / 200, Wwn: "naa.6090a038f0cd4e5bdaa8248e6856d4fe:3", }, nil) _, err := nodeSvc.NodeExpandVolume(context.Background(), &csi.NodeExpandVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, VolumePath: "", CapacityRange: &csi.CapacityRange{ RequiredBytes: 2234234, LimitBytes: controller.MaxVolumeSizeBytes, }, }) - Ω(err.Error()).To(ContainSubstring("targetPath is required")) - + gomega.Ω(err.Error()).To(gomega.ContainSubstring("targetPath is required")) }) }) - When("volume is not found", func() { - It("should fail", func() { - + ginkgo.When("volume is not found", func() { + ginkgo.It("should fail", func() { fsMock.On("GetUtil").Return(utilMock) clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ Description: "", - ID: validBlockVolumeID, + ID: validBlockVolumeHandle, Name: "name", Size: controller.MaxVolumeSizeBytes / 200, Wwn: "naa.6090a038f0cd4e5bdaa8248e6856d4fe:3", }, errors.New("err")).Times(1) _, err := nodeSvc.NodeExpandVolume(context.Background(), &csi.NodeExpandVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, VolumePath: validTargetPath, CapacityRange: &csi.CapacityRange{ RequiredBytes: 2234234, LimitBytes: controller.MaxVolumeSizeBytes, }, }) - Ω(err.Error()).To(ContainSubstring("Volume not found")) - + gomega.Ω(err.Error()).To(gomega.ContainSubstring("Volume not found")) }) }) - When("Unable to create mount target", func() { - It("should fail", func() { - + ginkgo.When("Unable to create mount target", func() { + ginkgo.It("should fail", func() { fsMock.On("GetUtil").Return(utilMock) clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ Description: "", - ID: validBlockVolumeID, + ID: validBlockVolumeHandle, Name: "name", Size: controller.MaxVolumeSizeBytes / 200, }, nil) @@ -2433,18 +3629,16 @@ var _ = Describe("CSINodeService", func() { MountPoint: stagingPath, }, errors.New("offline")).Times(1) fsMock.On("MkdirAll", mock.Anything, mock.Anything).Return(errors.New("Unable to create dirs")) - _, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeID, false)) - Ω(err.Error()).To(ContainSubstring("Failed to find mount info for")) - + _, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeHandle, false)) + gomega.Ω(err.Error()).To(gomega.ContainSubstring("Failed to find mount info for")) }) }) - When("Unable to perform mount", func() { - It("should fail", func() { - + ginkgo.When("Unable to perform mount", func() { + ginkgo.It("should fail", func() { fsMock.On("GetUtil").Return(utilMock) clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ Description: "", - ID: validBlockVolumeID, + ID: validBlockVolumeHandle, Name: "name", Size: controller.MaxVolumeSizeBytes / 200, }, nil) @@ -2455,18 +3649,16 @@ var _ = Describe("CSINodeService", func() { }, errors.New("offline")).Times(1) fsMock.On("MkdirAll", mock.Anything, mock.Anything).Return(nil) utilMock.On("Mount", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(errors.New("bad mount")) - _, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeID, false)) - Ω(err.Error()).To(ContainSubstring("Failed to find mount info for")) - + _, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeHandle, false)) + gomega.Ω(err.Error()).To(gomega.ContainSubstring("Failed to find mount info for")) }) }) - When("Unable to perform unmount", func() { - It("should succeed", func() { - + ginkgo.When("Unable to perform unmount", func() { + ginkgo.It("should succeed", func() { fsMock.On("GetUtil").Return(utilMock) clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ Description: "", - ID: validBlockVolumeID, + ID: validBlockVolumeHandle, Name: "name", Size: controller.MaxVolumeSizeBytes / 200, }, nil) @@ -2491,19 +3683,17 @@ var _ = Describe("CSINodeService", func() { utilMock.On("FindFSType", mock.Anything, mock.Anything).Return("xfs", nil) fsMock.On("ExecCommandOutput", mock.Anything, mock.Anything, mock.Anything).Return([]byte("version 5.0.0"), nil) utilMock.On("ResizeFS", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) - res, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeID, false)) - Ω(err).To(BeNil()) - Ω(res).To(Equal(&csi.NodeExpandVolumeResponse{})) - + res, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeHandle, false)) + gomega.Ω(err).To(gomega.BeNil()) + gomega.Ω(res).To(gomega.Equal(&csi.NodeExpandVolumeResponse{})) }) }) - When("Unable to find mount info", func() { - It("should fail", func() { - + ginkgo.When("Unable to find mount info", func() { + ginkgo.It("should fail", func() { fsMock.On("GetUtil").Return(utilMock) clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ Description: "", - ID: validBlockVolumeID, + ID: validBlockVolumeHandle, Name: "name", Size: controller.MaxVolumeSizeBytes / 200, }, nil) @@ -2523,18 +3713,16 @@ var _ = Describe("CSINodeService", func() { MountPoint: stagingPath, }, errors.New("again")).Times(1) - _, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeID, false)) - Ω(err.Error()).To(ContainSubstring("Failed to find mount info for")) - + _, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeHandle, false)) + gomega.Ω(err.Error()).To(gomega.ContainSubstring("Failed to find mount info for")) }) }) - When("Unable to rescan the device", func() { - It("should fail", func() { - + ginkgo.When("Unable to rescan the device", func() { + ginkgo.It("should fail", func() { fsMock.On("GetUtil").Return(utilMock) clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ Description: "", - ID: validBlockVolumeID, + ID: validBlockVolumeHandle, Name: "name", Size: controller.MaxVolumeSizeBytes / 200, }, nil) @@ -2554,18 +3742,16 @@ var _ = Describe("CSINodeService", func() { MountPoint: stagingPath, }, nil).Times(1) utilMock.On("DeviceRescan", mock.Anything, mock.Anything).Return(errors.New("Failed to rescan device")) - _, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeID, false)) - Ω(err.Error()).To(ContainSubstring("Failed to rescan device")) - + _, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeHandle, false)) + gomega.Ω(err.Error()).To(gomega.ContainSubstring("Failed to rescan device")) }) }) - When("Unable to resize mpath", func() { - It("should fail", func() { - + ginkgo.When("Unable to resize mpath", func() { + ginkgo.It("should fail", func() { fsMock.On("GetUtil").Return(utilMock) clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ Description: "", - ID: validBlockVolumeID, + ID: validBlockVolumeHandle, Name: "name", Size: controller.MaxVolumeSizeBytes / 200, }, nil) @@ -2586,16 +3772,16 @@ var _ = Describe("CSINodeService", func() { }, nil).Times(1) utilMock.On("DeviceRescan", mock.Anything, mock.Anything).Return(nil) utilMock.On("ResizeMultipath", mock.Anything, mock.Anything).Return(errors.New("mpath resize error")) - _, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeID, false)) - Ω(err.Error()).To(ContainSubstring("mpath resize error")) - + _, err := nodeSvc.NodeExpandVolume(context.Background(), getNodeVolumeExpandValidRequest(validBlockVolumeHandle, false)) + gomega.Ω(err.Error()).To(gomega.ContainSubstring("mpath resize error")) }) }) }) - Describe("Calling EphemeralNodePublish()", func() { - When("everything's correct", func() { - It("should succeed", func() { + ginkgo.Describe("Calling EphemeralNodePublish()", func() { + ginkgo.When("everything's correct", func() { + ginkgo.It("should succeed", func() { + setDefaultClientMocks() fsMock.On("Stat", mock.Anything).Return(&mocks.FileInfo{}, nil) fsMock.On("MkdirAll", mock.Anything, mock.Anything).Return(nil).Times(2) fsMock.On("Create", mock.Anything).Return(&os.File{}, nil) @@ -2605,7 +3791,7 @@ var _ = Describe("CSINodeService", func() { CapacityBytes: validVolSize, VolumeId: filepath.Join(validBaseVolumeID, firstValidIP, "scsi"), VolumeContext: map[string]string{ - common.KeyArrayID: firstValidIP, + identifiers.KeyArrayID: firstValidIP, }, }, }, nil) @@ -2621,10 +3807,7 @@ var _ = Describe("CSINodeService", func() { "FCWWPN1": "58ccf09348a002a3", }, }, nil) - iscsiConnectorMock.On("ConnectVolume", mock.Anything, gobrick.ISCSIVolumeInfo{ - Targets: validISCSITargetInfo, - Lun: validLUNIDINT, - }).Return(gobrick.Device{}, nil) + iscsiConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything).Return(gobrick.Device{}, nil) fsMock.On("GetUtil").Return(utilMock) utilMock.On("BindMount", mock.Anything, "/dev", mock.Anything).Return(nil) fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil) @@ -2636,7 +3819,7 @@ var _ = Describe("CSINodeService", func() { fsMock.On("ExecCommand", "mkfs.ext4", "-E", "nodiscard", "-F", mock.Anything).Return([]byte{}, nil) utilMock.On("Mount", mock.Anything, mock.Anything, mock.Anything, "ext4").Return(nil) res, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, @@ -2647,13 +3830,13 @@ var _ = Describe("CSINodeService", func() { "size": "2Gi", }, }) - Ω(err).To(BeNil()) - Ω(res).To(Equal(&csi.NodePublishVolumeResponse{})) + gomega.Ω(err).To(gomega.BeNil()) + gomega.Ω(res).To(gomega.Equal(&csi.NodePublishVolumeResponse{})) }) }) - When("Child ControllerPublish() is failing", func() { + ginkgo.When("Child ControllerPublish() is failing", func() { capabilities := getCapabilityWithVoltypeAccessFstype("mount", "single-writer", "ext4") - It("should cleanup and call unpublish", func() { + ginkgo.It("should cleanup and call unpublish", func() { fsMock.On("Stat", mock.Anything).Return(&mocks.FileInfo{}, nil) fsMock.On("MkdirAll", mock.Anything, mock.Anything).Return(nil).Times(2) fsMock.On("Create", mock.Anything).Return(&os.File{}, nil) @@ -2664,21 +3847,22 @@ var _ = Describe("CSINodeService", func() { VolumeCapabilities: []*csi.VolumeCapability{capabilities}, Parameters: map[string]string{ "csi.storage.k8s.io/ephemeral": "true", - "size": "2Gi"}, + "size": "2Gi", + }, }).Return(&csi.CreateVolumeResponse{ Volume: &csi.Volume{ CapacityBytes: validVolSize, VolumeId: filepath.Join(validBaseVolumeID, firstValidIP, "scsi"), VolumeContext: map[string]string{ - common.KeyArrayID: firstValidIP, + identifiers.KeyArrayID: firstValidIP, }, }, }, nil) ctrlMock.On("ControllerPublishVolume", mock.Anything, &csi.ControllerPublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, NodeId: validNodeID, VolumeContext: map[string]string{ - common.KeyArrayID: firstValidIP, + identifiers.KeyArrayID: firstValidIP, }, VolumeCapability: getCapabilityWithVoltypeAccessFstype("mount", "single-writer", "ext4"), }).Return(&csi.ControllerPublishVolumeResponse{ @@ -2706,9 +3890,10 @@ var _ = Describe("CSINodeService", func() { fsMock.On("Stat", mock.Anything).Return(&mocks.FileInfo{}, os.ErrNotExist) utilMock.On("Unmount", mock.Anything, mock.Anything).Return(nil) + fsMock.On("Remove", mock.Anything).Return(nil) _, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, @@ -2719,11 +3904,12 @@ var _ = Describe("CSINodeService", func() { "size": "2Gi", }, }) - Ω(err.Error()).To(ContainSubstring("inline ephemeral controller publish failed")) + gomega.Ω(err.Error()).To(gomega.ContainSubstring("inline ephemeral controller publish failed")) }) }) - When("Child NodeStage() is failing", func() { - It("should cleanup and call unpublish", func() { + ginkgo.When("Child NodeStage() is failing", func() { + ginkgo.It("should cleanup and call unpublish", func() { + setDefaultClientMocks() fsMock.On("Stat", mock.Anything).Return(&mocks.FileInfo{}, nil) fsMock.On("MkdirAll", mock.Anything, mock.Anything).Return(nil).Times(2) fsMock.On("Create", mock.Anything).Return(&os.File{}, nil) @@ -2733,7 +3919,7 @@ var _ = Describe("CSINodeService", func() { CapacityBytes: validVolSize, VolumeId: filepath.Join(validBaseVolumeID, firstValidIP, "scsi"), VolumeContext: map[string]string{ - common.KeyArrayID: firstValidIP, + identifiers.KeyArrayID: firstValidIP, }, }, }, nil) @@ -2750,10 +3936,7 @@ var _ = Describe("CSINodeService", func() { }, }, nil) - iscsiConnectorMock.On("ConnectVolume", mock.Anything, gobrick.ISCSIVolumeInfo{ - Targets: validISCSITargetInfo, - Lun: validLUNIDINT, - }).Return(gobrick.Device{}, nil) + iscsiConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything).Return(gobrick.Device{}, nil) utilMock.On("BindMount", mock.Anything, "/dev", mock.Anything).Return(nil) fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(2) fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return([]gofsutil.Info{}, nil) @@ -2775,7 +3958,7 @@ var _ = Describe("CSINodeService", func() { utilMock.On("Unmount", mock.Anything, mock.Anything).Return(nil) _, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, @@ -2786,15 +3969,15 @@ var _ = Describe("CSINodeService", func() { "size": "2Gi", }, }) - Ω(err.Error()).To(ContainSubstring("inline ephemeral node stage failed")) + gomega.Ω(err.Error()).To(gomega.ContainSubstring("inline ephemeral node stage failed")) }) }) - When("Failed to parse size. Bad string", func() { - It("should fail", func() { + ginkgo.When("Failed to parse size. Bad string", func() { + ginkgo.It("should fail", func() { fsMock.On("Stat", mock.Anything).Return(&mocks.FileInfo{}, nil) fsMock.On("MkdirAll", mock.Anything, mock.Anything).Return(nil).Times(2) _, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, @@ -2805,15 +3988,15 @@ var _ = Describe("CSINodeService", func() { "size": "Dear SP please give me enough capacity", }, }) - Ω(err.Error()).To(ContainSubstring("inline ephemeral parse size failed")) + gomega.Ω(err.Error()).To(gomega.ContainSubstring("inline ephemeral parse size failed")) }) }) - When("Failed to create mount paths", func() { - It("should fail", func() { + ginkgo.When("Failed to create mount paths", func() { + ginkgo.It("should fail", func() { fsMock.On("Stat", mock.Anything).Return(&mocks.FileInfo{}, os.ErrNotExist) fsMock.On("MkdirAll", mock.Anything, mock.Anything).Return(errors.New("err")).Times(2) _, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, @@ -2824,11 +4007,11 @@ var _ = Describe("CSINodeService", func() { "size": "2 Gi", }, }) - Ω(err.Error()).To(ContainSubstring("Unable to create directory for mounting ephemeral volumes")) + gomega.Ω(err.Error()).To(gomega.ContainSubstring("Unable to create directory for mounting ephemeral volumes")) }) }) - When("Inline ephemeral create volume fails", func() { - It("should fail", func() { + ginkgo.When("Inline ephemeral create volume fails", func() { + ginkgo.It("should fail", func() { fsMock.On("Stat", mock.Anything).Return(&mocks.FileInfo{}, nil) fsMock.On("MkdirAll", mock.Anything, mock.Anything).Return(nil).Times(2) fsMock.On("Create", mock.Anything).Return(&os.File{}, nil) @@ -2838,12 +4021,12 @@ var _ = Describe("CSINodeService", func() { CapacityBytes: validVolSize, VolumeId: filepath.Join(validBaseVolumeID, firstValidIP, "scsi"), VolumeContext: map[string]string{ - common.KeyArrayID: firstValidIP, + identifiers.KeyArrayID: firstValidIP, }, }, }, errors.New("Failed")) _, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, @@ -2854,11 +4037,11 @@ var _ = Describe("CSINodeService", func() { "size": "2 Gi", }, }) - Ω(err.Error()).To(ContainSubstring("inline ephemeral create volume failed")) + gomega.Ω(err.Error()).To(gomega.ContainSubstring("inline ephemeral create volume failed")) }) }) - When("fs.Create after createVolume fails", func() { - It("should fail", func() { + ginkgo.When("fs.Create after createVolume fails", func() { + ginkgo.It("should fail", func() { fsMock.On("Stat", mock.Anything).Return(&mocks.FileInfo{}, nil) fsMock.On("MkdirAll", mock.Anything, mock.Anything).Return(nil).Times(2) fsMock.On("Create", mock.Anything).Return(&os.File{}, errors.New("Failed to create")) @@ -2868,12 +4051,12 @@ var _ = Describe("CSINodeService", func() { CapacityBytes: validVolSize, VolumeId: filepath.Join(validBaseVolumeID, firstValidIP, "scsi"), VolumeContext: map[string]string{ - common.KeyArrayID: firstValidIP, + identifiers.KeyArrayID: firstValidIP, }, }, }, nil) _, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, @@ -2884,11 +4067,11 @@ var _ = Describe("CSINodeService", func() { "size": "2 Gi", }, }) - Ω(err.Error()).To(ContainSubstring("Failed to create")) + gomega.Ω(err.Error()).To(gomega.ContainSubstring("Failed to create")) }) }) - When("fs.Writestring fails", func() { - It("should fail", func() { + ginkgo.When("fs.Writestring fails", func() { + ginkgo.It("should fail", func() { fsMock.On("Stat", mock.Anything).Return(&mocks.FileInfo{}, nil) fsMock.On("MkdirAll", mock.Anything, mock.Anything).Return(nil).Times(2) fsMock.On("Create", mock.Anything).Return(&os.File{}, nil) @@ -2898,12 +4081,12 @@ var _ = Describe("CSINodeService", func() { CapacityBytes: validVolSize, VolumeId: filepath.Join(validBaseVolumeID, firstValidIP, "scsi"), VolumeContext: map[string]string{ - common.KeyArrayID: firstValidIP, + identifiers.KeyArrayID: firstValidIP, }, }, }, nil) _, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, @@ -2914,12 +4097,13 @@ var _ = Describe("CSINodeService", func() { "size": "2 Gi", }, }) - Ω(err.Error()).To(ContainSubstring("Failed to write string")) + gomega.Ω(err.Error()).To(gomega.ContainSubstring("Failed to write string")) }) }) }) - When("everything's correct", func() { - It("should succeed", func() { + ginkgo.When("everything's correct", func() { + ginkgo.It("should succeed", func() { + setDefaultClientMocks() fsMock.On("Stat", mock.Anything).Return(&mocks.FileInfo{}, nil) fsMock.On("MkdirAll", mock.Anything, mock.Anything).Return(nil).Times(2) fsMock.On("Create", mock.Anything).Return(&os.File{}, nil) @@ -2929,7 +4113,7 @@ var _ = Describe("CSINodeService", func() { CapacityBytes: validVolSize, VolumeId: filepath.Join(validBaseVolumeID, firstValidIP, "scsi"), VolumeContext: map[string]string{ - common.KeyArrayID: firstValidIP, + identifiers.KeyArrayID: firstValidIP, }, }, }, nil) @@ -2945,10 +4129,7 @@ var _ = Describe("CSINodeService", func() { "FCWWPN1": "58ccf09348a002a3", }, }, nil) - iscsiConnectorMock.On("ConnectVolume", mock.Anything, gobrick.ISCSIVolumeInfo{ - Targets: validISCSITargetInfo, - Lun: validLUNIDINT, - }).Return(gobrick.Device{}, nil) + iscsiConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything).Return(gobrick.Device{}, nil) fsMock.On("GetUtil").Return(utilMock) utilMock.On("BindMount", mock.Anything, "/dev", mock.Anything).Return(nil) fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil) @@ -2960,7 +4141,7 @@ var _ = Describe("CSINodeService", func() { fsMock.On("ExecCommand", "mkfs.xfs", "-K", mock.Anything, "-m", mock.Anything).Return([]byte{}, nil) utilMock.On("Mount", mock.Anything, mock.Anything, mock.Anything, "xfs", mock.Anything).Return(errors.New("err")) _, err := nodeSvc.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: validStagingPath, TargetPath: validTargetPath, @@ -2971,20 +4152,26 @@ var _ = Describe("CSINodeService", func() { "size": "2Gi", }, }) - Ω(err.Error()).To(ContainSubstring("inline ephemeral node publish failed")) + gomega.Ω(err.Error()).To(gomega.ContainSubstring("inline ephemeral node publish failed")) }) }) - Describe("Calling EphemeralNodeUnPublish()", func() { - When("everything is correct", func() { - It("should succeed", func() { + ginkgo.Describe("Calling EphemeralNodeUnPublish()", func() { + ginkgo.When("everything is correct", func() { + ginkgo.It("should succeed", func() { mountInfo := []gofsutil.Info{ { Device: validDevName, Path: validTargetPath, }, } - fsMock.On("ReadFile", ephemerallockfile).Return([]byte(validBlockVolumeID), nil) + clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ + Description: "", + ID: validBlockVolumeHandle, + Name: "name", + Size: controller.MaxVolumeSizeBytes / 200, + }, nil) + fsMock.On("ReadFile", ephemerallockfile).Return([]byte(validBlockVolumeHandle), nil) fsMock.On("Stat", mock.Anything).Return(&mocks.FileInfo{}, nil) fsMock.On("GetUtil").Return(utilMock) @@ -2994,7 +4181,7 @@ var _ = Describe("CSINodeService", func() { utilMock.On("Unmount", mock.Anything, mock.Anything).Return(nil) fsMock.On("Remove", mock.Anything).Return(nil) - fsMock.On("WriteFile", mock.Anything, mock.Anything, os.FileMode(0640)).Return(nil) + fsMock.On("WriteFile", mock.Anything, mock.Anything, os.FileMode(0o640)).Return(nil) iscsiConnectorMock.On("DisconnectVolumeByDeviceName", mock.Anything, mock.Anything).Return(nil) @@ -3002,22 +4189,22 @@ var _ = Describe("CSINodeService", func() { fsMock.On("IsNotExist", mock.Anything).Return(false) fsMock.On("ReadFile", mock.Anything).Return([]byte("Some data"), nil) ctrlMock.On("ControllerUnpublishVolume", mock.Anything, &csi.ControllerUnpublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, NodeId: validNodeID, }).Return(&csi.ControllerUnpublishVolumeResponse{}, nil) ctrlMock.On("DeleteVolume", mock.Anything, &csi.DeleteVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, }).Return(&csi.DeleteVolumeResponse{}, nil) res, err := nodeSvc.NodeUnpublishVolume(context.Background(), &csi.NodeUnpublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, TargetPath: validTargetPath, }) - Ω(err).To(BeNil()) - Ω(res).To(Equal(&csi.NodeUnpublishVolumeResponse{})) + gomega.Ω(err).To(gomega.BeNil()) + gomega.Ω(res).To(gomega.Equal(&csi.NodeUnpublishVolumeResponse{})) }) }) - When("no vlocak file", func() { - It("should fail", func() { + ginkgo.When("no vlocak file", func() { + ginkgo.It("should fail", func() { mountInfo := []gofsutil.Info{ { Device: validDevName, @@ -3031,23 +4218,29 @@ var _ = Describe("CSINodeService", func() { fsMock.On("Stat", mock.Anything).Return(&mocks.FileInfo{}, nil) fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil) fsMock.On("GetUtil").Return(utilMock) - fsMock.On("ReadFile", ephemerallockfile).Return([]byte(validBlockVolumeID), os.ErrNotExist) + fsMock.On("ReadFile", ephemerallockfile).Return([]byte(validBlockVolumeHandle), os.ErrNotExist) _, err := nodeSvc.NodeUnpublishVolume(context.Background(), &csi.NodeUnpublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, TargetPath: validTargetPath, }) - Ω(err.Error()).To(ContainSubstring("Was unable to read lockfile")) + gomega.Ω(err.Error()).To(gomega.ContainSubstring("Was unable to read lockfile")) }) }) - When("controller unpublish fails", func() { - It("should fail", func() { + ginkgo.When("controller unpublish fails", func() { + ginkgo.It("should fail", func() { mountInfo := []gofsutil.Info{ { Device: validDevName, Path: validTargetPath, }, } - fsMock.On("ReadFile", ephemerallockfile).Return([]byte(validBlockVolumeID), nil) + clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ + Description: "", + ID: validBlockVolumeHandle, + Name: "name", + Size: controller.MaxVolumeSizeBytes / 200, + }, nil) + fsMock.On("ReadFile", ephemerallockfile).Return([]byte(validBlockVolumeHandle), nil) fsMock.On("Stat", mock.Anything).Return(&mocks.FileInfo{}, nil) fsMock.On("GetUtil").Return(utilMock) @@ -3057,7 +4250,7 @@ var _ = Describe("CSINodeService", func() { utilMock.On("Unmount", mock.Anything, mock.Anything).Return(nil) fsMock.On("Remove", mock.Anything).Return(nil) - fsMock.On("WriteFile", mock.Anything, mock.Anything, os.FileMode(0640)).Return(nil) + fsMock.On("WriteFile", mock.Anything, mock.Anything, os.FileMode(0o640)).Return(nil) iscsiConnectorMock.On("DisconnectVolumeByDeviceName", mock.Anything, mock.Anything).Return(nil) @@ -3065,28 +4258,34 @@ var _ = Describe("CSINodeService", func() { fsMock.On("IsNotExist", mock.Anything).Return(false) fsMock.On("ReadFile", mock.Anything).Return([]byte("Some data"), nil) ctrlMock.On("ControllerUnpublishVolume", mock.Anything, &csi.ControllerUnpublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, NodeId: validNodeID, }).Return(&csi.ControllerUnpublishVolumeResponse{}, errors.New("failed")) ctrlMock.On("DeleteVolume", mock.Anything, &csi.DeleteVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, }).Return(&csi.DeleteVolumeResponse{}, nil) _, err := nodeSvc.NodeUnpublishVolume(context.Background(), &csi.NodeUnpublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, TargetPath: validTargetPath, }) - Ω(err.Error()).To(ContainSubstring("Inline ephemeral controller unpublish")) + gomega.Ω(err.Error()).To(gomega.ContainSubstring("Inline ephemeral controller unpublish")) }) }) - When("controller delete volume fails", func() { - It("should fail", func() { + ginkgo.When("controller delete volume fails", func() { + ginkgo.It("should fail", func() { mountInfo := []gofsutil.Info{ { Device: validDevName, Path: validTargetPath, }, } - fsMock.On("ReadFile", ephemerallockfile).Return([]byte(validBlockVolumeID), nil) + clientMock.On("GetVolume", mock.Anything, mock.Anything).Return(gopowerstore.Volume{ + Description: "", + ID: validBlockVolumeHandle, + Name: "name", + Size: controller.MaxVolumeSizeBytes / 200, + }, nil) + fsMock.On("ReadFile", ephemerallockfile).Return([]byte(validBlockVolumeHandle), nil) fsMock.On("Stat", mock.Anything).Return(&mocks.FileInfo{}, nil) fsMock.On("GetUtil").Return(utilMock) @@ -3096,7 +4295,7 @@ var _ = Describe("CSINodeService", func() { utilMock.On("Unmount", mock.Anything, mock.Anything).Return(nil) fsMock.On("Remove", mock.Anything).Return(nil) - fsMock.On("WriteFile", mock.Anything, mock.Anything, os.FileMode(0640)).Return(nil) + fsMock.On("WriteFile", mock.Anything, mock.Anything, os.FileMode(0o640)).Return(nil) iscsiConnectorMock.On("DisconnectVolumeByDeviceName", mock.Anything, mock.Anything).Return(nil) @@ -3104,24 +4303,26 @@ var _ = Describe("CSINodeService", func() { fsMock.On("IsNotExist", mock.Anything).Return(false) fsMock.On("ReadFile", mock.Anything).Return([]byte("Some data"), nil) ctrlMock.On("ControllerUnpublishVolume", mock.Anything, &csi.ControllerUnpublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, NodeId: validNodeID, }).Return(&csi.ControllerUnpublishVolumeResponse{}, nil) ctrlMock.On("DeleteVolume", mock.Anything, &csi.DeleteVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, }).Return(&csi.DeleteVolumeResponse{}, errors.New("failed")) _, err := nodeSvc.NodeUnpublishVolume(context.Background(), &csi.NodeUnpublishVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, TargetPath: validTargetPath, }) - Ω(err.Error()).To(ContainSubstring("failed")) + gomega.Ω(err.Error()).To(gomega.ContainSubstring("failed")) }) }) }) - Describe("calling NodeGetInfo()", func() { - When("managing multiple arrays", func() { - It("should return correct topology segments", func() { + ginkgo.Describe("calling NodeGetInfo()", func() { + ginkgo.When("managing multiple arrays", func() { + ginkgo.It("should return correct topology segments when nfs is enabled", func() { + clientMock.On("GetNASServers", mock.Anything). + Return(nasData, nil) clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). Return([]gopowerstore.IPPoolAddress{ { @@ -3138,26 +4339,25 @@ var _ = Describe("CSINodeService", func() { conn, nil, ) - setDefaultNodeLabelsRetrieverMock() + setDefaultNodeLabelsMock() res, err := nodeSvc.NodeGetInfo(context.Background(), &csi.NodeGetInfoRequest{}) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.NodeGetInfoResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeGetInfoResponse{ NodeId: nodeSvc.nodeID, AccessibleTopology: &csi.Topology{ Segments: map[string]string{ - common.Name + "/" + firstValidIP + "-nfs": "true", - common.Name + "/" + firstValidIP + "-iscsi": "true", - common.Name + "/" + secondValidIP + "-nfs": "true", + identifiers.Name + "/" + firstValidIP + "-nfs": "true", + identifiers.Name + "/" + firstValidIP + "-iscsi": "true", + identifiers.Name + "/" + secondValidIP + "-nfs": "true", }, }, MaxVolumesPerNode: 0, })) }) - }) - - When("node label max-powerstore-volumes-per-node is set and retrieved successfully", func() { - It("should return correct MaxVolumesPerNode in response", func() { + ginkgo.It("should return correct topology segments when Auth V2 is enabled", func() { + clientMock.On("GetNASServers", mock.Anything). + Return(nasData, nil) clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). Return([]gopowerstore.IPPoolAddress{ { @@ -3169,34 +4369,35 @@ var _ = Describe("CSINodeService", func() { IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn2"}, }, }, nil) - conn, _ := net.Dial("udp", "127.0.0.1:80") + conn, _ := net.Dial("udp", "127.0.0.1:9400") fsMock.On("NetDial", mock.Anything).Return( conn, nil, ) - nodeLabelsRetrieverMock.On("BuildConfigFromFlags", mock.Anything, mock.Anything).Return(nil, nil) - nodeLabelsRetrieverMock.On("GetNodeLabels", mock.Anything, mock.Anything, mock.Anything).Return(map[string]string{"max-powerstore-volumes-per-node": "2"}, nil) - nodeLabelsRetrieverMock.On("InClusterConfig", mock.Anything).Return(nil, nil) - nodeLabelsRetrieverMock.On("NewForConfig", mock.Anything).Return(nil, nil) + setDefaultNodeLabelsMock() res, err := nodeSvc.NodeGetInfo(context.Background(), &csi.NodeGetInfoRequest{}) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.NodeGetInfoResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeGetInfoResponse{ NodeId: nodeSvc.nodeID, AccessibleTopology: &csi.Topology{ Segments: map[string]string{ - common.Name + "/" + firstValidIP + "-nfs": "true", - common.Name + "/" + firstValidIP + "-iscsi": "true", - common.Name + "/" + secondValidIP + "-nfs": "true", + identifiers.Name + "/" + firstValidIP + "-nfs": "true", + identifiers.Name + "/" + firstValidIP + "-iscsi": "true", + identifiers.Name + "/" + secondValidIP + "-nfs": "true", }, }, - MaxVolumesPerNode: 2, + MaxVolumesPerNode: 0, })) }) }) - - When("there is some issue while retrieving node labels", func() { - It("should return proper error", func() { + ginkgo.When("managing multiple arrays", func() { + ginkgo.It("should return correct topology segments when nfs is disabled", func() { + // disable nfs server to to check negetive behaviour + nasData[0].NfsServers[0].IsNFSv4Enabled = false + nasData[0].NfsServers[0].IsNFSv3Enabled = false + clientMock.On("GetNASServers", mock.Anything). + Return(nasData, nil) clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). Return([]gopowerstore.IPPoolAddress{ { @@ -3213,29 +4414,28 @@ var _ = Describe("CSINodeService", func() { conn, nil, ) - nodeLabelsRetrieverMock.On("BuildConfigFromFlags", mock.Anything, mock.Anything).Return(nil, nil) - nodeLabelsRetrieverMock.On("GetNodeLabels", mock.Anything, mock.Anything, mock.Anything).Return(nil, nil) - nodeLabelsRetrieverMock.On("InClusterConfig", mock.Anything).Return(nil, errors.New("Unable to create kubeclientset")) - nodeLabelsRetrieverMock.On("NewForConfig", mock.Anything).Return(nil, nil) + setDefaultNodeLabelsMock() res, err := nodeSvc.NodeGetInfo(context.Background(), &csi.NodeGetInfoRequest{}) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.NodeGetInfoResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeGetInfoResponse{ NodeId: nodeSvc.nodeID, AccessibleTopology: &csi.Topology{ Segments: map[string]string{ - common.Name + "/" + firstValidIP + "-nfs": "true", - common.Name + "/" + firstValidIP + "-iscsi": "true", - common.Name + "/" + secondValidIP + "-nfs": "true", + identifiers.Name + "/" + firstValidIP + "-iscsi": "true", }, }, MaxVolumesPerNode: 0, })) }) }) - - When("MaxVolumesPerNode is set via environment variable at the time of installation", func() { - It("should return correct MaxVolumesPerNode in response", func() { + ginkgo.When("node label max-powerstore-volumes-per-node is set and retrieved successfully", func() { + ginkgo.It("should return correct MaxVolumesPerNode in response", func() { + // enabling back nfs servers + nasData[0].NfsServers[0].IsNFSv4Enabled = true + nasData[0].NfsServers[0].IsNFSv3Enabled = false + clientMock.On("GetNASServers", mock.Anything). + Return(nasData, nil) clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). Return([]gopowerstore.IPPoolAddress{ { @@ -3247,23 +4447,24 @@ var _ = Describe("CSINodeService", func() { IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn2"}, }, }, nil) + conn, _ := net.Dial("udp", "127.0.0.1:80") fsMock.On("NetDial", mock.Anything).Return( conn, nil, ) - setDefaultNodeLabelsRetrieverMock() - nodeSvc.opts.MaxVolumesPerNode = 2 + + k8sutils.Kubeclient.SetNodeLabel(context.Background(), nodeSvc.opts.KubeNodeName, "max-powerstore-volumes-per-node", "2") res, err := nodeSvc.NodeGetInfo(context.Background(), &csi.NodeGetInfoRequest{}) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.NodeGetInfoResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeGetInfoResponse{ NodeId: nodeSvc.nodeID, AccessibleTopology: &csi.Topology{ Segments: map[string]string{ - common.Name + "/" + firstValidIP + "-nfs": "true", - common.Name + "/" + firstValidIP + "-iscsi": "true", - common.Name + "/" + secondValidIP + "-nfs": "true", + identifiers.Name + "/" + firstValidIP + "-nfs": "true", + identifiers.Name + "/" + firstValidIP + "-iscsi": "true", + identifiers.Name + "/" + secondValidIP + "-nfs": "true", }, }, MaxVolumesPerNode: 2, @@ -3271,16 +4472,18 @@ var _ = Describe("CSINodeService", func() { }) }) - When("Portals are not discoverable", func() { - It("should return correct topology segments", func() { + ginkgo.When("there is some issue while retrieving node labels", func() { + ginkgo.It("should return proper error", func() { + clientMock.On("GetNASServers", mock.Anything). + Return(nasData, nil) clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). Return([]gopowerstore.IPPoolAddress{ { - Address: "192.168.1.3", + Address: "192.168.1.1", IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn"}, }, { - Address: "192.168.1.4", + Address: "192.168.1.2", IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn2"}, }, }, nil) @@ -3289,16 +4492,16 @@ var _ = Describe("CSINodeService", func() { conn, nil, ) - setDefaultNodeLabelsRetrieverMock() res, err := nodeSvc.NodeGetInfo(context.Background(), &csi.NodeGetInfoRequest{}) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.NodeGetInfoResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeGetInfoResponse{ NodeId: nodeSvc.nodeID, AccessibleTopology: &csi.Topology{ Segments: map[string]string{ - common.Name + "/" + firstValidIP + "-nfs": "true", - common.Name + "/" + secondValidIP + "-nfs": "true", + identifiers.Name + "/" + firstValidIP + "-nfs": "true", + identifiers.Name + "/" + firstValidIP + "-iscsi": "true", + identifiers.Name + "/" + secondValidIP + "-nfs": "true", }, }, MaxVolumesPerNode: 0, @@ -3306,83 +4509,221 @@ var _ = Describe("CSINodeService", func() { }) }) - When("we can not get targets from array", func() { - It("should not return iscsi topology key", func() { - e := "internal error" + ginkgo.When("calling NodeGetInfo with metro and match labels for zone1", func() { + ginkgo.It("should return correct NodeGetInfoResponse", func() { + clientMock.On("GetNASServers", mock.Anything). + Return(nasData, nil) clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). - Return([]gopowerstore.IPPoolAddress{}, errors.New(e)) + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn"}, + }, + { + Address: "192.168.1.2", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn2"}, + }, + }, nil) + conn, _ := net.Dial("udp", "127.0.0.1:80") + fsMock.On("NetDial", mock.Anything).Return( + conn, + nil, + ) + + nodeSvc.SetArrays(getMetroTestArrays()) + res, err := nodeSvc.NodeGetInfo(context.Background(), &csi.NodeGetInfoRequest{}) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res.AccessibleTopology.Segments).To(gomega.HaveKeyWithValue("topology.kubernetes.io/zone", "zone1")) + }) + }) + ginkgo.When("calling NodeGetInfo with metro and match labels for zone2", func() { + ginkgo.It("should return correct NodeGetInfo response", func() { + clientMock.On("GetNASServers", mock.Anything). + Return(nasData, nil) + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn"}, + }, + { + Address: "192.168.1.2", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn2"}, + }, + }, nil) conn, _ := net.Dial("udp", "127.0.0.1:80") fsMock.On("NetDial", mock.Anything).Return( conn, nil, ) - setDefaultNodeLabelsRetrieverMock() + nodeSvc.SetArrays(getMetroTestArrays()) + + k8sutils.Kubeclient.SetNodeLabel(context.Background(), nodeSvc.opts.KubeNodeName, "topology.kubernetes.io/zone", "zone2") res, err := nodeSvc.NodeGetInfo(context.Background(), &csi.NodeGetInfoRequest{}) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res.AccessibleTopology.Segments).To(gomega.HaveKeyWithValue("topology.kubernetes.io/zone", "zone2")) + }) + }) + + ginkgo.When("MaxVolumesPerNode is set via environment variable at the time of installation", func() { + ginkgo.It("should return correct MaxVolumesPerNode in response", func() { + clientMock.On("GetNASServers", mock.Anything). + Return(nasData, nil) + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn"}, + }, + { + Address: "192.168.1.2", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn2"}, + }, + }, nil) + conn, _ := net.Dial("udp", "127.0.0.1:80") + fsMock.On("NetDial", mock.Anything).Return( + conn, + nil, + ) + setDefaultNodeLabelsMock() + nodeSvc.opts.MaxVolumesPerNode = 2 - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.NodeGetInfoResponse{ + res, err := nodeSvc.NodeGetInfo(context.Background(), &csi.NodeGetInfoRequest{}) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeGetInfoResponse{ NodeId: nodeSvc.nodeID, AccessibleTopology: &csi.Topology{ Segments: map[string]string{ - common.Name + "/" + firstValidIP + "-nfs": "true", - common.Name + "/" + secondValidIP + "-nfs": "true", + identifiers.Name + "/" + firstValidIP + "-nfs": "true", + identifiers.Name + "/" + firstValidIP + "-iscsi": "true", + identifiers.Name + "/" + secondValidIP + "-nfs": "true", }, }, - MaxVolumesPerNode: 0, + MaxVolumesPerNode: 2, })) }) }) - When("target can not be discovered", func() { - It("should not return iscsi topology key", func() { - goiscsi.GOISCSIMock.InduceDiscoveryError = true - gonvme.GONVMEMock.InduceDiscoveryError = true - + ginkgo.When("Portals are not discoverable", func() { + ginkgo.It("should return correct topology segments", func() { + clientMock.On("GetNASServers", mock.Anything). + Return(nasData, nil) clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). Return([]gopowerstore.IPPoolAddress{ { - Address: "192.168.1.1", + Address: "192.168.1.5", IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn"}, }, + { + Address: "192.168.1.6", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn2"}, + }, }, nil) conn, _ := net.Dial("udp", "127.0.0.1:80") fsMock.On("NetDial", mock.Anything).Return( conn, nil, ) - setDefaultNodeLabelsRetrieverMock() + setDefaultNodeLabelsMock() res, err := nodeSvc.NodeGetInfo(context.Background(), &csi.NodeGetInfoRequest{}) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.NodeGetInfoResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeGetInfoResponse{ NodeId: nodeSvc.nodeID, AccessibleTopology: &csi.Topology{ Segments: map[string]string{ - common.Name + "/" + firstValidIP + "-nfs": "true", - common.Name + "/" + secondValidIP + "-nfs": "true", + identifiers.Name + "/" + firstValidIP + "-nfs": "true", + identifiers.Name + "/" + secondValidIP + "-nfs": "true", }, }, MaxVolumesPerNode: 0, })) - gonvme.GONVMEMock.InduceDiscoveryError = false }) }) - When("using FC", func() { - It("should return FC topology segments", func() { - nodeSvc.useFC = true + ginkgo.When("we can not get targets from array", func() { + ginkgo.It("should not return iscsi topology key", func() { + e := "internal error" + clientMock.On("GetNASServers", mock.Anything). + Return(nasData, nil) + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{}, errors.New(e)) conn, _ := net.Dial("udp", "127.0.0.1:80") fsMock.On("NetDial", mock.Anything).Return( conn, nil, ) - clientMock.On("GetHostByName", mock.Anything, nodeSvc.nodeID). - Return(gopowerstore.Host{ - ID: "host-id", - Initiators: []gopowerstore.InitiatorInstance{ - { - ActiveSessions: []gopowerstore.ActiveSessionInstance{ + setDefaultNodeLabelsMock() + + res, err := nodeSvc.NodeGetInfo(context.Background(), &csi.NodeGetInfoRequest{}) + + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeGetInfoResponse{ + NodeId: nodeSvc.nodeID, + AccessibleTopology: &csi.Topology{ + Segments: map[string]string{ + identifiers.Name + "/" + firstValidIP + "-nfs": "true", + identifiers.Name + "/" + secondValidIP + "-nfs": "true", + }, + }, + MaxVolumesPerNode: 0, + })) + }) + }) + + ginkgo.When("target can not be discovered", func() { + ginkgo.It("should not return iscsi topology key", func() { + goiscsi.GOISCSIMock.InduceDiscoveryError = true + gonvme.GONVMEMock.InduceDiscoveryError = true + clientMock.On("GetNASServers", mock.Anything). + Return(nasData, nil) + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn"}, + }, + }, nil) + conn, _ := net.Dial("udp", "127.0.0.1:80") + fsMock.On("NetDial", mock.Anything).Return( + conn, + nil, + ) + setDefaultNodeLabelsMock() + + res, err := nodeSvc.NodeGetInfo(context.Background(), &csi.NodeGetInfoRequest{}) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeGetInfoResponse{ + NodeId: nodeSvc.nodeID, + AccessibleTopology: &csi.Topology{ + Segments: map[string]string{ + identifiers.Name + "/" + firstValidIP + "-nfs": "true", + identifiers.Name + "/" + secondValidIP + "-nfs": "true", + }, + }, + MaxVolumesPerNode: 0, + })) + gonvme.GONVMEMock.InduceDiscoveryError = false + }) + }) + + ginkgo.When("using FC", func() { + ginkgo.It("should return FC topology segments", func() { + nodeSvc.useFC[firstGlobalID] = true + conn, _ := net.Dial("udp", "127.0.0.1:80") + clientMock.On("GetNASServers", mock.Anything). + Return(nasData, nil) + fsMock.On("NetDial", mock.Anything).Return( + conn, + nil, + ) + clientMock.On("GetHostByName", mock.Anything, nodeSvc.nodeID). + Return(gopowerstore.Host{ + ID: "host-id", + Initiators: []gopowerstore.InitiatorInstance{ + { + ActiveSessions: []gopowerstore.ActiveSessionInstance{ { PortName: validFCTargetsWWPN[0], }, @@ -3398,135 +4739,33 @@ var _ = Describe("CSINodeService", func() { }, PortName: validFCTargetsWWPN[1], PortType: gopowerstore.InitiatorProtocolTypeEnumFC, - }}, + }, + }, Name: "host-name", }, nil) - setDefaultNodeLabelsRetrieverMock() + setDefaultNodeLabelsMock() res, err := nodeSvc.NodeGetInfo(context.Background(), &csi.NodeGetInfoRequest{}) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.NodeGetInfoResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeGetInfoResponse{ NodeId: nodeSvc.nodeID, AccessibleTopology: &csi.Topology{ Segments: map[string]string{ - common.Name + "/" + firstValidIP + "-nfs": "true", - common.Name + "/" + firstValidIP + "-fc": "true", - common.Name + "/" + secondValidIP + "-nfs": "true", + identifiers.Name + "/" + firstValidIP + "-nfs": "true", + identifiers.Name + "/" + firstValidIP + "-fc": "true", + identifiers.Name + "/" + secondValidIP + "-nfs": "true", }, }, MaxVolumesPerNode: 0, })) }) - When("reusing host", func() { - It("should properly deal with additional IPs", func() { - nodeSvc.useFC = true - nodeID := nodeSvc.nodeID - nodeSvc.nodeID = nodeID + "-" + "192.168.0.1" - nodeSvc.reusedHost = true - conn, _ := net.Dial("udp", "127.0.0.1:80") - fsMock.On("NetDial", mock.Anything).Return( - conn, - nil, - ) - clientMock.On("GetHostByName", mock.Anything, nodeID). - Return(gopowerstore.Host{ - ID: "host-id", - Initiators: []gopowerstore.InitiatorInstance{ - { - ActiveSessions: []gopowerstore.ActiveSessionInstance{ - { - PortName: validFCTargetsWWPN[0], - }, - }, - PortName: validFCTargetsWWPN[0], - PortType: gopowerstore.InitiatorProtocolTypeEnumFC, - }, - { - ActiveSessions: []gopowerstore.ActiveSessionInstance{ - { - PortName: validFCTargetsWWPN[1], - }, - }, - PortName: validFCTargetsWWPN[1], - PortType: gopowerstore.InitiatorProtocolTypeEnumFC, - }}, - Name: "host-name", - }, nil) - setDefaultNodeLabelsRetrieverMock() - - res, err := nodeSvc.NodeGetInfo(context.Background(), &csi.NodeGetInfoRequest{}) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.NodeGetInfoResponse{ - NodeId: nodeSvc.nodeID, - AccessibleTopology: &csi.Topology{ - Segments: map[string]string{ - common.Name + "/" + firstValidIP + "-nfs": "true", - common.Name + "/" + firstValidIP + "-fc": "true", - common.Name + "/" + secondValidIP + "-nfs": "true", - }, - }, - MaxVolumesPerNode: 0, - })) - }) - - When("there is no ip in nodeID", func() { - It("should not return FC topology key", func() { - nodeSvc.useFC = true - nodeID := nodeSvc.nodeID - nodeSvc.nodeID = "nodeid-with-no-ip" - nodeSvc.reusedHost = true - conn, _ := net.Dial("udp", "127.0.0.1:80") - fsMock.On("NetDial", mock.Anything).Return( - conn, - nil, - ) - clientMock.On("GetHostByName", mock.Anything, nodeID). - Return(gopowerstore.Host{ - ID: "host-id", - Initiators: []gopowerstore.InitiatorInstance{ - { - ActiveSessions: []gopowerstore.ActiveSessionInstance{ - { - PortName: validFCTargetsWWPN[0], - }, - }, - PortName: validFCTargetsWWPN[0], - PortType: gopowerstore.InitiatorProtocolTypeEnumFC, - }, - { - ActiveSessions: []gopowerstore.ActiveSessionInstance{ - { - PortName: validFCTargetsWWPN[1], - }, - }, - PortName: validFCTargetsWWPN[1], - PortType: gopowerstore.InitiatorProtocolTypeEnumFC, - }}, - Name: "host-name", - }, nil) - setDefaultNodeLabelsRetrieverMock() - - res, err := nodeSvc.NodeGetInfo(context.Background(), &csi.NodeGetInfoRequest{}) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.NodeGetInfoResponse{ - NodeId: nodeSvc.nodeID, - AccessibleTopology: &csi.Topology{ - Segments: map[string]string{ - common.Name + "/" + firstValidIP + "-nfs": "true", - common.Name + "/" + secondValidIP + "-nfs": "true", - }, - }, - MaxVolumesPerNode: 0, - })) - }) - }) - }) - - When("we can not get info about hosts from array", func() { - It("should not return FC topology key", func() { - nodeSvc.useFC = true + ginkgo.When("we can not get info about hosts from array", func() { + ginkgo.It("should not return FC topology key", func() { + nodeSvc.useFC[firstGlobalID] = true e := "internal error" + clientMock.On("GetNASServers", mock.Anything). + Return(nasData, nil) clientMock.On("GetHostByName", mock.Anything, nodeSvc.nodeID). Return(gopowerstore.Host{}, errors.New(e)) conn, _ := net.Dial("udp", "127.0.0.1:80") @@ -3534,16 +4773,16 @@ var _ = Describe("CSINodeService", func() { conn, nil, ) - setDefaultNodeLabelsRetrieverMock() + setDefaultNodeLabelsMock() res, err := nodeSvc.NodeGetInfo(context.Background(), &csi.NodeGetInfoRequest{}) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.NodeGetInfoResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeGetInfoResponse{ NodeId: nodeSvc.nodeID, AccessibleTopology: &csi.Topology{ Segments: map[string]string{ - common.Name + "/" + firstValidIP + "-nfs": "true", - common.Name + "/" + secondValidIP + "-nfs": "true", + identifiers.Name + "/" + firstValidIP + "-nfs": "true", + identifiers.Name + "/" + secondValidIP + "-nfs": "true", }, }, MaxVolumesPerNode: 0, @@ -3551,9 +4790,11 @@ var _ = Describe("CSINodeService", func() { }) }) - When("host initiators is empty", func() { - It("should not return FC topology key", func() { - nodeSvc.useFC = true + ginkgo.When("host initiators is empty", func() { + ginkgo.It("should not return FC topology key", func() { + nodeSvc.useFC[firstGlobalID] = true + clientMock.On("GetNASServers", mock.Anything). + Return(nasData, nil) clientMock.On("GetHostByName", mock.Anything, nodeSvc.nodeID). Return(gopowerstore.Host{ ID: "host-id", @@ -3565,16 +4806,16 @@ var _ = Describe("CSINodeService", func() { conn, nil, ) - setDefaultNodeLabelsRetrieverMock() + setDefaultNodeLabelsMock() res, err := nodeSvc.NodeGetInfo(context.Background(), &csi.NodeGetInfoRequest{}) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.NodeGetInfoResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeGetInfoResponse{ NodeId: nodeSvc.nodeID, AccessibleTopology: &csi.Topology{ Segments: map[string]string{ - common.Name + "/" + firstValidIP + "-nfs": "true", - common.Name + "/" + secondValidIP + "-nfs": "true", + identifiers.Name + "/" + firstValidIP + "-nfs": "true", + identifiers.Name + "/" + secondValidIP + "-nfs": "true", }, }, MaxVolumesPerNode: 0, @@ -3582,10 +4823,12 @@ var _ = Describe("CSINodeService", func() { }) }) - When("there is no active sessions", func() { - It("should not return FC topology key", func() { - nodeSvc.useFC = true + ginkgo.When("there is no active sessions", func() { + ginkgo.It("should not return FC topology key", func() { + nodeSvc.useFC[firstGlobalID] = true conn, _ := net.Dial("udp", "127.0.0.1:80") + clientMock.On("GetNASServers", mock.Anything). + Return(nasData, nil) fsMock.On("NetDial", mock.Anything).Return( conn, nil, @@ -3601,19 +4844,20 @@ var _ = Describe("CSINodeService", func() { { PortName: validFCTargetsWWPN[1], PortType: gopowerstore.InitiatorProtocolTypeEnumFC, - }}, + }, + }, Name: "host-name", }, nil) - setDefaultNodeLabelsRetrieverMock() + setDefaultNodeLabelsMock() res, err := nodeSvc.NodeGetInfo(context.Background(), &csi.NodeGetInfoRequest{}) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.NodeGetInfoResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeGetInfoResponse{ NodeId: nodeSvc.nodeID, AccessibleTopology: &csi.Topology{ Segments: map[string]string{ - common.Name + "/" + firstValidIP + "-nfs": "true", - common.Name + "/" + secondValidIP + "-nfs": "true", + identifiers.Name + "/" + firstValidIP + "-nfs": "true", + identifiers.Name + "/" + secondValidIP + "-nfs": "true", }, }, MaxVolumesPerNode: 0, @@ -3622,10 +4866,12 @@ var _ = Describe("CSINodeService", func() { }) }) - When("using NVMeFC", func() { - It("should return NVMeFC topology segments", func() { - nodeSvc.useNVME = true - nodeSvc.useFC = true + ginkgo.When("using NVMeFC", func() { + ginkgo.It("should return NVMeFC topology segments", func() { + nodeSvc.useNVME[firstGlobalID] = true + nodeSvc.useFC[firstGlobalID] = true + clientMock.On("GetNASServers", mock.Anything). + Return(nasData, nil) clientMock.On("GetCluster", mock.Anything). Return(gopowerstore.Cluster{ Name: validClusterName, @@ -3644,27 +4890,29 @@ var _ = Describe("CSINodeService", func() { conn, nil, ) - setDefaultNodeLabelsRetrieverMock() + setDefaultNodeLabelsMock() res, err := nodeSvc.NodeGetInfo(context.Background(), &csi.NodeGetInfoRequest{}) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.NodeGetInfoResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeGetInfoResponse{ NodeId: nodeSvc.nodeID, AccessibleTopology: &csi.Topology{ Segments: map[string]string{ - common.Name + "/" + firstValidIP + "-nfs": "true", - common.Name + "/" + firstValidIP + "-nvmefc": "true", - common.Name + "/" + secondValidIP + "-nfs": "true", + identifiers.Name + "/" + firstValidIP + "-nfs": "true", + identifiers.Name + "/" + firstValidIP + "-nvmefc": "true", + identifiers.Name + "/" + secondValidIP + "-nfs": "true", }, }, MaxVolumesPerNode: 0, })) }) - When("NVMeFC targets cannot be discovered", func() { - It("should not return NVMeFC topology segments", func() { - nodeSvc.useNVME = true - nodeSvc.useFC = true + ginkgo.When("NVMeFC targets cannot be discovered", func() { + ginkgo.It("should not return NVMeFC topology segments", func() { + nodeSvc.useNVME[firstGlobalID] = true + nodeSvc.useFC[firstGlobalID] = true + clientMock.On("GetNASServers", mock.Anything). + Return(nasData, nil) clientMock.On("GetCluster", mock.Anything). Return(gopowerstore.Cluster{ Name: validClusterName, @@ -3682,16 +4930,16 @@ var _ = Describe("CSINodeService", func() { conn, nil, ) - setDefaultNodeLabelsRetrieverMock() + setDefaultNodeLabelsMock() res, err := nodeSvc.NodeGetInfo(context.Background(), &csi.NodeGetInfoRequest{}) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.NodeGetInfoResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeGetInfoResponse{ NodeId: nodeSvc.nodeID, AccessibleTopology: &csi.Topology{ Segments: map[string]string{ - common.Name + "/" + firstValidIP + "-nfs": "true", - common.Name + "/" + secondValidIP + "-nfs": "true", + identifiers.Name + "/" + firstValidIP + "-nfs": "true", + identifiers.Name + "/" + secondValidIP + "-nfs": "true", }, }, MaxVolumesPerNode: 0, @@ -3700,67 +4948,223 @@ var _ = Describe("CSINodeService", func() { }) }) - When("using NVMeTCP", func() { - It("should return NVMeTCP topology segments", func() { - nodeSvc.useNVME = true - nodeSvc.useFC = false - clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). + ginkgo.When("using NVMeTCP", func() { + ginkgo.It("should return NVMeTCP topology segments", func() { + nodeSvc.useNVME[firstGlobalID] = true + nodeSvc.useFC[firstGlobalID] = false + clientMock.On("GetNASServers", mock.Anything). + Return(nasData, nil) + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn"}, + }, + }, nil) + clientMock.On("GetStorageNVMETCPTargetAddresses", mock.Anything). Return([]gopowerstore.IPPoolAddress{ { Address: "192.168.1.1", IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn"}, }, }, nil) + clientMock.On("GetCluster", mock.Anything). + Return(gopowerstore.Cluster{ + Name: validClusterName, + NVMeNQN: validNVMEInitiators[0], + }, nil) conn, _ := net.Dial("udp", "127.0.0.1:80") fsMock.On("NetDial", mock.Anything).Return( conn, nil, ) - setDefaultNodeLabelsRetrieverMock() + setDefaultNodeLabelsMock() res, err := nodeSvc.NodeGetInfo(context.Background(), &csi.NodeGetInfoRequest{}) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.NodeGetInfoResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeGetInfoResponse{ NodeId: nodeSvc.nodeID, AccessibleTopology: &csi.Topology{ Segments: map[string]string{ - common.Name + "/" + firstValidIP + "-nfs": "true", - common.Name + "/" + firstValidIP + "-nvmetcp": "true", - common.Name + "/" + secondValidIP + "-nfs": "true", + identifiers.Name + "/" + firstValidIP + "-nfs": "true", + identifiers.Name + "/" + firstValidIP + "-nvmetcp": "true", + identifiers.Name + "/" + secondValidIP + "-nfs": "true", }, }, MaxVolumesPerNode: 0, })) }) - When("target can not be discovered", func() { - It("should not return nvme topology key", func() { + ginkgo.When("using iSCSI on multiple networks", func() { + ginkgo.BeforeEach(func() { + options = []variableOption{withMockNumberOfISCSITargets(4)} + }) + ginkgo.AfterEach(func() { + options = []variableOption{} + }) + ginkgo.It("should return iSCSI topology segments", func() { + goiscsi.GOISCSIMock.InduceDiscoveryError = false + nodeSvc.useNVME[firstGlobalID] = false + nodeSvc.useNFS = false + clientMock.On("GetNASServers", mock.Anything). + Return([]gopowerstore.NAS{}, nil) + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "nqn"}, + NetworkID: "NW1", + }, + { + Address: "192.168.1.2", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "nqn"}, + NetworkID: "NW1", + }, + { + Address: "192.168.2.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "nqn"}, + NetworkID: "NW2", + }, + { + Address: "192.168.2.2", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "nqn"}, + NetworkID: "NW2", + }, + }, nil) + clientMock.On("GetCluster", mock.Anything). + Return(gopowerstore.Cluster{ + Name: validClusterName, + NVMeNQN: validNVMEInitiators[0], + }, nil) + conn, _ := net.Dial("udp", "127.0.0.1:80") + fsMock.On("NetDial", mock.Anything).Return( + conn, + nil, + ) + setDefaultNodeLabelsMock() + + res, err := nodeSvc.NodeGetInfo(context.Background(), &csi.NodeGetInfoRequest{}) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeGetInfoResponse{ + NodeId: nodeSvc.nodeID, + AccessibleTopology: &csi.Topology{ + Segments: map[string]string{ + identifiers.Name + "/" + firstValidIP + "-iscsi": "true", + }, + }, + MaxVolumesPerNode: 0, + })) + }) + }) + + ginkgo.When("using NVMeTCP on multiple networks", func() { + ginkgo.BeforeEach(func() { + options = []variableOption{withMockNumberOfNVMeTCPTargets(2)} + }) + ginkgo.AfterEach(func() { + options = []variableOption{} + }) + ginkgo.It("should return NVMeTCP topology segments", func() { + nodeSvc.useNVME[firstGlobalID] = true + nodeSvc.useFC[firstGlobalID] = false + clientMock.On("GetNASServers", mock.Anything). + Return(nasData, nil) + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn"}, + }, + }, nil) + clientMock.On("GetStorageNVMETCPTargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "nqn"}, + NetworkID: "NW1", + }, + { + Address: "192.168.1.2", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "nqn"}, + NetworkID: "NW1", + }, + { + Address: "192.168.2.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "nqn"}, + NetworkID: "NW2", + }, + { + Address: "192.168.2.2", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "nqn"}, + NetworkID: "NW2", + }, + }, nil) + clientMock.On("GetCluster", mock.Anything). + Return(gopowerstore.Cluster{ + Name: validClusterName, + NVMeNQN: validNVMEInitiators[0], + }, nil) + conn, _ := net.Dial("udp", "127.0.0.1:80") + fsMock.On("NetDial", mock.Anything).Return( + conn, + nil, + ) + setDefaultNodeLabelsMock() + + res, err := nodeSvc.NodeGetInfo(context.Background(), &csi.NodeGetInfoRequest{}) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeGetInfoResponse{ + NodeId: nodeSvc.nodeID, + AccessibleTopology: &csi.Topology{ + Segments: map[string]string{ + identifiers.Name + "/" + firstValidIP + "-nfs": "true", + identifiers.Name + "/" + firstValidIP + "-nvmetcp": "true", + identifiers.Name + "/" + secondValidIP + "-nfs": "true", + }, + }, + MaxVolumesPerNode: 0, + })) + }) + }) + + ginkgo.When("target can not be discovered", func() { + ginkgo.It("should not return nvme topology key", func() { goiscsi.GOISCSIMock.InduceDiscoveryError = true gonvme.GONVMEMock.InduceDiscoveryError = true - nodeSvc.useNVME = true - nodeSvc.useFC = false - clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). + nodeSvc.useNVME[firstGlobalID] = true + nodeSvc.useFC[firstGlobalID] = false + clientMock.On("GetNASServers", mock.Anything). + Return(nasData, nil) + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn"}, + }, + }, nil) + clientMock.On("GetStorageNVMETCPTargetAddresses", mock.Anything). Return([]gopowerstore.IPPoolAddress{ { Address: "192.168.1.1", IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn"}, }, }, nil) + clientMock.On("GetCluster", mock.Anything). + Return(gopowerstore.Cluster{ + Name: validClusterName, + NVMeNQN: validNVMEInitiators[0], + }, nil) conn, _ := net.Dial("udp", "127.0.0.1:80") fsMock.On("NetDial", mock.Anything).Return( conn, nil, ) - setDefaultNodeLabelsRetrieverMock() + setDefaultNodeLabelsMock() res, err := nodeSvc.NodeGetInfo(context.Background(), &csi.NodeGetInfoRequest{}) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.NodeGetInfoResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeGetInfoResponse{ NodeId: nodeSvc.nodeID, AccessibleTopology: &csi.Topology{ Segments: map[string]string{ - common.Name + "/" + firstValidIP + "-nfs": "true", - common.Name + "/" + secondValidIP + "-nfs": "true", + identifiers.Name + "/" + firstValidIP + "-nfs": "true", + identifiers.Name + "/" + secondValidIP + "-nfs": "true", }, }, MaxVolumesPerNode: 0, @@ -3769,28 +5173,41 @@ var _ = Describe("CSINodeService", func() { }) }) - When("we cannot get NVMeTCP targets from the array", func() { - It("should not return NVMeTCP topology segments", func() { - nodeSvc.useNVME = true - nodeSvc.useFC = false + ginkgo.When("we cannot get NVMeTCP targets from the array", func() { + ginkgo.It("should not return NVMeTCP topology segments", func() { + nodeSvc.useNVME[firstGlobalID] = true + nodeSvc.useFC[firstGlobalID] = false e := "internalerror" - clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). + clientMock.On("GetNASServers", mock.Anything). + Return(nasData, nil) + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn"}, + }, + }, nil) + clientMock.On("GetStorageNVMETCPTargetAddresses", mock.Anything). Return([]gopowerstore.IPPoolAddress{}, errors.New(e)) + clientMock.On("GetCluster", mock.Anything). + Return(gopowerstore.Cluster{ + Name: validClusterName, + NVMeNQN: validNVMEInitiators[0], + }, nil) conn, _ := net.Dial("udp", "127.0.0.1:80") fsMock.On("NetDial", mock.Anything).Return( conn, nil, ) - setDefaultNodeLabelsRetrieverMock() + setDefaultNodeLabelsMock() res, err := nodeSvc.NodeGetInfo(context.Background(), &csi.NodeGetInfoRequest{}) - Expect(err).To(BeNil()) - Expect(res).To(Equal(&csi.NodeGetInfoResponse{ + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeGetInfoResponse{ NodeId: nodeSvc.nodeID, AccessibleTopology: &csi.Topology{ Segments: map[string]string{ - common.Name + "/" + firstValidIP + "-nfs": "true", - common.Name + "/" + secondValidIP + "-nfs": "true", + identifiers.Name + "/" + firstValidIP + "-nfs": "true", + identifiers.Name + "/" + secondValidIP + "-nfs": "true", }, }, MaxVolumesPerNode: 0, @@ -3800,9 +5217,9 @@ var _ = Describe("CSINodeService", func() { }) }) - Describe("Calling NodeGetCapabilities()", func() { - It("should return predefined parameters with health monitor", func() { - csictx.Setenv(context.Background(), common.EnvIsHealthMonitorEnabled, "true") + ginkgo.Describe("Calling NodeGetCapabilities()", func() { + ginkgo.It("should return predefined parameters with health monitor", func() { + csictx.Setenv(context.Background(), identifiers.EnvIsHealthMonitorEnabled, "true") nodeSvc.nodeID = "" @@ -3834,24 +5251,27 @@ var _ = Describe("CSINodeService", func() { }}, Name: "host-name", }}, nil) - clientMock.On("GetCustomHTTPHeaders").Return(make(http.Header)) + // clientMock.On("GetStorageISCSITargetAddresses", mock.Anything).Return([]gopowerstore.IPPoolAddress{}, nil) + clientMock.On("GetCustomHTTPHeaders").Return(api.NewSafeHeader().GetHeader()) clientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(3.0), nil) clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) clientMock.On("CreateHost", mock.Anything, mock.Anything). Return(gopowerstore.CreateResponse{ID: validHostID}, nil) + setDefaultNodeLabelsMock() nodeSvc.opts.NodeNamePrefix = "" nodeSvc.Init() res, err := nodeSvc.NodeGetCapabilities(context.Background(), &csi.NodeGetCapabilitiesRequest{}) - Ω(err).To(BeNil()) - Ω(res).To(Equal(&csi.NodeGetCapabilitiesResponse{ + gomega.Ω(err).To(gomega.BeNil()) + gomega.Ω(res).To(gomega.Equal(&csi.NodeGetCapabilitiesResponse{ Capabilities: []*csi.NodeServiceCapability{ - {Type: &csi.NodeServiceCapability_Rpc{ - Rpc: &csi.NodeServiceCapability_RPC{ - Type: csi.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME, + { + Type: &csi.NodeServiceCapability_Rpc{ + Rpc: &csi.NodeServiceCapability_RPC{ + Type: csi.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME, + }, }, }, - }, { Type: &csi.NodeServiceCapability_Rpc{ Rpc: &csi.NodeServiceCapability_RPC{ @@ -3885,169 +5305,3721 @@ var _ = Describe("CSINodeService", func() { }) }) - Describe("Calling getInitiators()", func() { - When("Only iSCSI initiators are on node", func() { - It("should succeed", func() { + ginkgo.Describe("Calling getInitiators()", func() { + ginkgo.When("Only iSCSI initiators are on node", func() { + ginkgo.It("should succeed", func() { iscsiConnectorMock.On("GetInitiatorName", mock.Anything).Return(validISCSIInitiators, nil) nvmeConnectorMock.On("GetInitiatorName", mock.Anything).Return([]string{}, nil) fcConnectorMock.On("GetInitiatorPorts", mock.Anything).Return([]string{}, nil) iinit, fcinit, nvmeinit, err := nodeSvc.getInitiators() - Ω(iinit).To(Equal([]string{ + gomega.Ω(iinit).To(gomega.Equal([]string{ "iqn.1994-05.com.redhat:4db86abbe3c", - "iqn.1994-05.com.redhat:2950c9ca441b"})) - Ω(nvmeinit).To(Equal([]string{})) - Ω(fcinit).To(BeNil()) - Ω(err).To(BeNil()) + "iqn.1994-05.com.redhat:2950c9ca441b", + })) + gomega.Ω(nvmeinit).To(gomega.Equal([]string{})) + gomega.Ω(fcinit).To(gomega.BeNil()) + gomega.Ω(err).To(gomega.BeNil()) }) }) - When("Only NVMe initiators are on node", func() { - It("should succeed", func() { + ginkgo.When("Only NVMe initiators are on node", func() { + ginkgo.It("should succeed", func() { iscsiConnectorMock.On("GetInitiatorName", mock.Anything).Return([]string{}, nil) nvmeConnectorMock.On("GetInitiatorName", mock.Anything).Return(validNVMEInitiators, nil) fcConnectorMock.On("GetInitiatorPorts", mock.Anything).Return([]string{}, nil) iinit, fcinit, nvmeinit, err := nodeSvc.getInitiators() - Ω(nvmeinit).To(Equal([]string{ + gomega.Ω(nvmeinit).To(gomega.Equal([]string{ "nqn.2014-08.org.nvmexpress:uuid:02a08600-57d6-4089-8736-bf1f7326990e", - "nqn.2014-08.org.nvmexpress:uuid:fa363a22-1c74-44f3-9932-1c35d5cf5c4d"})) - Ω(iinit).To(Equal([]string{})) - Ω(fcinit).To(BeNil()) - Ω(err).To(BeNil()) + "nqn.2014-08.org.nvmexpress:uuid:fa363a22-1c74-44f3-9932-1c35d5cf5c4d", + })) + gomega.Ω(iinit).To(gomega.Equal([]string{})) + gomega.Ω(fcinit).To(gomega.BeNil()) + gomega.Ω(err).To(gomega.BeNil()) }) }) - When("NVMe, FC and iSCSI initiators are on node", func() { - It("should succeed", func() { + ginkgo.When("NVMe, FC and iSCSI initiators are on node", func() { + ginkgo.It("should succeed", func() { iscsiConnectorMock.On("GetInitiatorName", mock.Anything).Return(validISCSIInitiators, nil) nvmeConnectorMock.On("GetInitiatorName", mock.Anything).Return(validNVMEInitiators, nil) fcConnectorMock.On("GetInitiatorPorts", mock.Anything).Return(validFCTargetsWWPN, nil) iinit, fcinit, nvmeinit, err := nodeSvc.getInitiators() - Ω(iinit).To(Equal([]string{ + gomega.Ω(iinit).To(gomega.Equal([]string{ "iqn.1994-05.com.redhat:4db86abbe3c", - "iqn.1994-05.com.redhat:2950c9ca441b"})) - Ω(nvmeinit).To(Equal([]string{ + "iqn.1994-05.com.redhat:2950c9ca441b", + })) + gomega.Ω(nvmeinit).To(gomega.Equal([]string{ "nqn.2014-08.org.nvmexpress:uuid:02a08600-57d6-4089-8736-bf1f7326990e", - "nqn.2014-08.org.nvmexpress:uuid:fa363a22-1c74-44f3-9932-1c35d5cf5c4d"})) - Ω(fcinit).To(Equal([]string{ + "nqn.2014-08.org.nvmexpress:uuid:fa363a22-1c74-44f3-9932-1c35d5cf5c4d", + })) + gomega.Ω(fcinit).To(gomega.Equal([]string{ "58:cc:f0:93:48:a0:03:a3", - "58:cc:f0:93:48:a0:02:a3"})) - Ω(err).To(BeNil()) + "58:cc:f0:93:48:a0:02:a3", + })) + gomega.Ω(err).To(gomega.BeNil()) }) }) - When("Neither NVMe nor FC nor iSCSI initiators are found on node", func() { - It("should succeed [NFS only]", func() { + ginkgo.When("Neither NVMe nor FC nor iSCSI initiators are found on node", func() { + ginkgo.It("should succeed [NFS only]", func() { iscsiConnectorMock.On("GetInitiatorName", mock.Anything).Return([]string{}, nil) nvmeConnectorMock.On("GetInitiatorName", mock.Anything).Return([]string{}, nil) fcConnectorMock.On("GetInitiatorPorts", mock.Anything).Return([]string{}, nil) iinit, fcinit, nvmeinit, err := nodeSvc.getInitiators() - Ω(len(iinit)).To(Equal(0)) - Ω(len(nvmeinit)).To(Equal(0)) - Ω(len(fcinit)).To(Equal(0)) - Ω(err).To(BeNil()) + gomega.Ω(len(iinit)).To(gomega.Equal(0)) + gomega.Ω(len(nvmeinit)).To(gomega.Equal(0)) + gomega.Ω(len(fcinit)).To(gomega.Equal(0)) + gomega.Ω(err).To(gomega.BeNil()) }) }) - When("Only FC initiators are on node", func() { - It("should succeed", func() { + ginkgo.When("Only FC initiators are on node", func() { + ginkgo.It("should succeed", func() { iscsiConnectorMock.On("GetInitiatorName", mock.Anything).Return([]string{}, nil) fcConnectorMock.On("GetInitiatorPorts", mock.Anything).Return(validFCTargetsWWPN, nil) nvmeConnectorMock.On("GetInitiatorName", mock.Anything).Return([]string{}, nil) iinit, fcinit, nvmeinit, err := nodeSvc.getInitiators() - Ω(iinit).To(Equal([]string{})) - Ω(nvmeinit).To(Equal([]string{})) - Ω(fcinit).To(Equal([]string{ + gomega.Ω(iinit).To(gomega.Equal([]string{})) + gomega.Ω(nvmeinit).To(gomega.Equal([]string{})) + gomega.Ω(fcinit).To(gomega.Equal([]string{ "58:cc:f0:93:48:a0:03:a3", - "58:cc:f0:93:48:a0:02:a3"})) - Ω(err).To(BeNil()) + "58:cc:f0:93:48:a0:02:a3", + })) + gomega.Ω(err).To(gomega.BeNil()) }) }) }) - Describe("calling Node Get Volume Stats", func() { - When("volume path missing", func() { - It("should fail", func() { - clientMock.On("GetVolume", mock.Anything, validBaseVolumeID). - Return(gopowerstore.Volume{ID: validBaseVolumeID, State: gopowerstore.VolumeStateEnumReady}, nil) - - clientMock.On("GetFS", mock.Anything, validBaseVolumeID). - Return(gopowerstore.FileSystem{ID: validBaseVolumeID}, nil) - - req := &csi.NodeGetVolumeStatsRequest{VolumeId: validBlockVolumeID, VolumePath: ""} - - res, err := nodeSvc.NodeGetVolumeStats(context.Background(), req) - - Expect(res).To(BeNil()) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To( - ContainSubstring("no volume Path provided"), + ginkgo.Describe("calling NodeExpandRawBlockVolume() offline", func() { + ginkgo.When("Error is encountered", func() { + ginkgo.It("should return error ", func() { + fsMock.On("GetUtil").Return(utilMock) + utilMock.On("GetSysBlockDevicesForVolumeWWN", mock.Anything, mock.Anything).Return( + []string{"nvme0n1,nvme0n2"}, + errors.New("Error"), ) + _, err := nodeSvc.nodeExpandRawBlockVolume(context.Background(), "") + gomega.Expect(err).ToNot(gomega.BeNil()) }) }) - }) -}) - -func TestInitConnectors(t *testing.T) { - arrays := getTestArrays() - nodeSvc = &Service{ - Fs: fsMock, - ctrlSvc: nil, - iscsiConnector: nil, - fcConnector: nil, - iscsiLib: nil, - nodeID: validNodeID, - useFC: false, - initialized: true, - } - nodeSvc.SetArrays(arrays) - nodeSvc.SetDefaultArray(arrays[firstValidIP]) - t.Run("success test", func(t *testing.T) { - nodeSvc.initConnectors() - }) -} - -func TestGetNodeOptions(t *testing.T) { - arrays := getTestArrays() - nodeSvc = &Service{ - Fs: fsMock, - ctrlSvc: nil, - iscsiConnector: nil, - fcConnector: nil, - iscsiLib: nil, + ginkgo.When("Devicenames is empty", func() { + ginkgo.It("should return error ", func() { + fsMock.On("GetUtil").Return(utilMock) + utilMock.On("GetSysBlockDevicesForVolumeWWN", mock.Anything, mock.Anything).Return( + []string{}, + nil, + ) + _, err := nodeSvc.nodeExpandRawBlockVolume(context.Background(), "") + gomega.Expect(err).ToNot(gomega.BeNil()) + }) + }) + ginkgo.When("error encountered in getnvmecontroller", func() { + ginkgo.It("should return error ", func() { + fsMock.On("GetUtil").Return(utilMock) + utilMock.On("GetSysBlockDevicesForVolumeWWN", mock.Anything, mock.Anything).Return( + []string{"nvme0n1,nvme0n2"}, + nil, + ) + utilMock.On("GetNVMeController", mock.Anything).Return( + "nvmecontroller-dev1", + errors.New("Error"), + ) + _, err := nodeSvc.nodeExpandRawBlockVolume(context.Background(), "") + gomega.Expect(err).ToNot(gomega.BeNil()) + }) + }) + ginkgo.When("DeviceRescan fail", func() { + ginkgo.It("should return error", func() { + fsMock.On("GetUtil").Return(utilMock) + utilMock.On("GetSysBlockDevicesForVolumeWWN", mock.Anything, mock.Anything).Return( + []string{"fcnvme0n1,fcnvme0n2"}, + nil, + ) + utilMock.On("DeviceRescan", mock.Anything, mock.Anything).Return( + errors.New("Error"), + ) + utilMock.On("GetMpathNameFromDevice", mock.Anything, mock.Anything).Return( + "", + errors.New("Error"), + ) + _, err := nodeSvc.nodeExpandRawBlockVolume(context.Background(), "") + gomega.Expect(err).ToNot(gomega.BeNil()) + }) + }) + ginkgo.When("GetMpathNameFromDevice fail", func() { + ginkgo.It("should return error", func() { + fsMock.On("GetUtil").Return(utilMock) + utilMock.On("GetSysBlockDevicesForVolumeWWN", mock.Anything, mock.Anything).Return( + []string{"fcnvme0n1,fcnvme0n2"}, + nil, + ) + utilMock.On("DeviceRescan", mock.Anything, mock.Anything).Return( + nil, + ) + utilMock.On("GetMpathNameFromDevice", mock.Anything, mock.Anything).Return( + "", + errors.New("Error"), + ) + _, err := nodeSvc.nodeExpandRawBlockVolume(context.Background(), "") + gomega.Expect(err).ToNot(gomega.BeNil()) + }) + }) + ginkgo.When("ResizeMultipath fail", func() { + ginkgo.It("should return error", func() { + fsMock.On("GetUtil").Return(utilMock) + utilMock.On("GetSysBlockDevicesForVolumeWWN", mock.Anything, mock.Anything).Return( + []string{"fcnvme0n1,fcnvme0n2"}, + nil, + ) + utilMock.On("DeviceRescan", mock.Anything, mock.Anything).Return( + nil, + ) + utilMock.On("GetMpathNameFromDevice", mock.Anything, mock.Anything).Return( + "mpath", + nil, + ) + utilMock.On("ResizeMultipath", mock.Anything, mock.Anything).Return( + errors.New("Error"), + ) + _, err := nodeSvc.nodeExpandRawBlockVolume(context.Background(), "") + gomega.Expect(err).ToNot(gomega.BeNil()) + }) + }) + }) + ginkgo.Describe("calling NodeGetVolumeStats()", func() { + ginkgo.When("volume ID is missing", func() { + ginkgo.It("should fail", func() { + req := &csi.NodeGetVolumeStatsRequest{VolumeId: "", VolumePath: ""} + res, err := nodeSvc.NodeGetVolumeStats(context.Background(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("no volume ID provided")) + }) + }) + + ginkgo.When("volume path is missing", func() { + ginkgo.It("should fail", func() { + req := &csi.NodeGetVolumeStatsRequest{VolumeId: validBlockVolumeHandle, VolumePath: ""} + res, err := nodeSvc.NodeGetVolumeStats(context.Background(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("no volume Path provided")) + }) + }) + + ginkgo.When("array ID is invalid", func() { + ginkgo.It("should fail", func() { + req := &csi.NodeGetVolumeStatsRequest{VolumeId: invalidBlockVolumeID, VolumePath: validTargetPath} + res, err := nodeSvc.NodeGetVolumeStats(context.Background(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("failed to find array with given ID")) + }) + }) + + ginkgo.When("API call fails [block]", func() { + ginkgo.It("should fail [GetVolume]", func() { + clientMock.On("GetVolume", mock.Anything, validBaseVolumeID). + Return(gopowerstore.Volume{}, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{StatusCode: http.StatusBadRequest}, + }) + + req := &csi.NodeGetVolumeStatsRequest{VolumeId: validBlockVolumeHandle, VolumePath: validTargetPath} + res, err := nodeSvc.NodeGetVolumeStats(context.Background(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("failed to find volume")) + }) + + ginkgo.It("should fail [GetHostVolumeMappingByVolumeID]", func() { + clientMock.On("GetVolume", mock.Anything, validBaseVolumeID). + Return(gopowerstore.Volume{ID: validBaseVolumeID, Size: controller.MaxVolumeSizeBytes / 200}, nil) + clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, validBaseVolumeID). + Return([]gopowerstore.HostVolumeMapping{}, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{StatusCode: http.StatusBadRequest}, + }) + + req := &csi.NodeGetVolumeStatsRequest{VolumeId: validBlockVolumeHandle, VolumePath: validTargetPath} + res, err := nodeSvc.NodeGetVolumeStats(context.Background(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("failed to get host volume mapping for volume")) + }) + + ginkgo.It("should fail [GetHost]", func() { + clientMock.On("GetVolume", mock.Anything, validBaseVolumeID). + Return(gopowerstore.Volume{ID: validBaseVolumeID, Size: controller.MaxVolumeSizeBytes / 200}, nil) + clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, validBaseVolumeID). + Return([]gopowerstore.HostVolumeMapping{{HostID: validHostID, LogicalUnitNumber: 1}}, nil) + clientMock.On("GetHost", mock.Anything, validHostID). + Return(gopowerstore.Host{}, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{StatusCode: http.StatusBadRequest}, + }) + + req := &csi.NodeGetVolumeStatsRequest{VolumeId: validBlockVolumeHandle, VolumePath: validTargetPath} + res, err := nodeSvc.NodeGetVolumeStats(context.Background(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("failed to get host")) + }) + }) + + ginkgo.When("API call fails with not found error [block]", func() { + ginkgo.It("should return stats as abnormal [GetVolume]", func() { + clientMock.On("GetVolume", mock.Anything, validBaseVolumeID). + Return(gopowerstore.Volume{}, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{StatusCode: http.StatusNotFound}, + }) + + req := &csi.NodeGetVolumeStatsRequest{VolumeId: validBlockVolumeHandle, VolumePath: validTargetPath} + res, err := nodeSvc.NodeGetVolumeStats(context.Background(), req) + + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeGetVolumeStatsResponse{ + Usage: usage, + VolumeCondition: &csi.VolumeCondition{ + Abnormal: true, + Message: fmt.Sprintf("Volume %s is not found", validBaseVolumeID), + }, + })) + }) + + ginkgo.It("should return stats as abnormal [GetHost]", func() { + clientMock.On("GetVolume", mock.Anything, validBaseVolumeID). + Return(gopowerstore.Volume{ID: validBaseVolumeID, Size: controller.MaxVolumeSizeBytes / 200}, nil) + clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, validBaseVolumeID). + Return([]gopowerstore.HostVolumeMapping{{HostID: validHostID, LogicalUnitNumber: 1}}, nil) + clientMock.On("GetHost", mock.Anything, validHostID). + Return(gopowerstore.Host{}, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{StatusCode: http.StatusNotFound}, + }) + + req := &csi.NodeGetVolumeStatsRequest{VolumeId: validBlockVolumeHandle, VolumePath: validTargetPath} + res, err := nodeSvc.NodeGetVolumeStats(context.Background(), req) + + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeGetVolumeStatsResponse{ + Usage: usage, + VolumeCondition: &csi.VolumeCondition{ + Abnormal: true, + Message: fmt.Sprintf("host %s is not attached to volume %s", validNodeID, validBaseVolumeID), + }, + })) + }) + }) + + ginkgo.When("host mapping not found as expected [block]", func() { + ginkgo.It("should return stats as abnormal [no active initiator]", func() { + clientMock.On("GetVolume", mock.Anything, validBaseVolumeID). + Return(gopowerstore.Volume{ID: validBaseVolumeID, Size: controller.MaxVolumeSizeBytes / 200}, nil) + clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, validBaseVolumeID). + Return([]gopowerstore.HostVolumeMapping{{HostID: validHostID, LogicalUnitNumber: 1}}, nil) + clientMock.On("GetHost", mock.Anything, validHostID). + Return(gopowerstore.Host{ID: validHostID, Name: validHostName}, nil) + + req := &csi.NodeGetVolumeStatsRequest{VolumeId: validBlockVolumeHandle, VolumePath: validTargetPath} + res, err := nodeSvc.NodeGetVolumeStats(context.Background(), req) + + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeGetVolumeStatsResponse{ + VolumeCondition: &csi.VolumeCondition{ + Abnormal: true, + Message: fmt.Sprintf("host %s has no active initiator connection", validNodeID), + }, + })) + }) + + ginkgo.It("should return stats as abnormal [not attached]", func() { + clientMock.On("GetVolume", mock.Anything, validBaseVolumeID). + Return(gopowerstore.Volume{ID: validBaseVolumeID, Size: controller.MaxVolumeSizeBytes / 200}, nil) + clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, validBaseVolumeID). + Return([]gopowerstore.HostVolumeMapping{{HostID: validHostID, LogicalUnitNumber: 1}}, nil) + clientMock.On("GetHost", mock.Anything, validHostID). + Return(gopowerstore.Host{ + ID: validHostID, + Name: validHostName, + Initiators: []gopowerstore.InitiatorInstance{ + { + ActiveSessions: []gopowerstore.ActiveSessionInstance{ + { + PortName: validISCSITargets[0], + }, + }, + PortName: validISCSIPortals[0], + PortType: gopowerstore.InitiatorProtocolTypeEnumISCSI, + }, + }, + }, nil) + + req := &csi.NodeGetVolumeStatsRequest{VolumeId: validBlockVolumeHandle, VolumePath: validTargetPath} + res, err := nodeSvc.NodeGetVolumeStats(context.Background(), req) + + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeGetVolumeStatsResponse{ + Usage: usage, + VolumeCondition: &csi.VolumeCondition{ + Abnormal: true, + Message: fmt.Sprintf("host %s is not attached to volume %s", validNodeID, validBaseVolumeID), + }, + })) + }) + }) + + ginkgo.When("API call fails [NFS]", func() { + ginkgo.It("should fail [GetFS]", func() { + clientMock.On("GetFS", mock.Anything, validBaseVolumeID). + Return(gopowerstore.FileSystem{}, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{StatusCode: http.StatusBadRequest}, + }) + + req := &csi.NodeGetVolumeStatsRequest{VolumeId: validNfsVolumeID, VolumePath: validTargetPath} + res, err := nodeSvc.NodeGetVolumeStats(context.Background(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("failed to find filesystem ")) + }) + + ginkgo.It("should fail [GetNFSExportByFileSystemID]", func() { + clientMock.On("GetFS", mock.Anything, validBaseVolumeID). + Return(gopowerstore.FileSystem{ID: validBaseVolumeID}, nil) + clientMock.On("GetNFSExportByFileSystemID", mock.Anything, validBaseVolumeID). + Return(gopowerstore.NFSExport{}, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{StatusCode: http.StatusBadRequest}, + }) + + req := &csi.NodeGetVolumeStatsRequest{VolumeId: validNfsVolumeID, VolumePath: validTargetPath} + res, err := nodeSvc.NodeGetVolumeStats(context.Background(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("failed to find nfs export for filesystem")) + }) + }) + + ginkgo.When("API call fails with not found error [NFS]", func() { + ginkgo.It("should return stats as abnormal [GetFS]", func() { + clientMock.On("GetFS", mock.Anything, validBaseVolumeID). + Return(gopowerstore.FileSystem{}, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{StatusCode: http.StatusNotFound}, + }) + + req := &csi.NodeGetVolumeStatsRequest{VolumeId: validNfsVolumeID, VolumePath: validTargetPath} + res, err := nodeSvc.NodeGetVolumeStats(context.Background(), req) + + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeGetVolumeStatsResponse{ + Usage: usage, + VolumeCondition: &csi.VolumeCondition{ + Abnormal: true, + Message: fmt.Sprintf("Filesystem %s is not found", validBaseVolumeID), + }, + })) + }) + + ginkgo.It("should return stats as abnormal [GetNFSExportByFileSystemID]", func() { + clientMock.On("GetFS", mock.Anything, validBaseVolumeID). + Return(gopowerstore.FileSystem{ID: validBaseVolumeID}, nil) + clientMock.On("GetNFSExportByFileSystemID", mock.Anything, validBaseVolumeID). + Return(gopowerstore.NFSExport{}, gopowerstore.APIError{ + ErrorMsg: &api.ErrorMsg{StatusCode: http.StatusNotFound}, + }) + + req := &csi.NodeGetVolumeStatsRequest{VolumeId: validNfsVolumeID, VolumePath: validTargetPath} + res, err := nodeSvc.NodeGetVolumeStats(context.Background(), req) + + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeGetVolumeStatsResponse{ + Usage: usage, + VolumeCondition: &csi.VolumeCondition{ + Abnormal: true, + Message: fmt.Sprintf("NFS export for volume %s is not found", validBaseVolumeID), + }, + })) + }) + }) + + ginkgo.When("NFS export not found as expected [NFS]", func() { + ginkgo.It("should return stats as abnormal [not attached]", func() { + clientMock.On("GetFS", mock.Anything, validBaseVolumeID). + Return(gopowerstore.FileSystem{ID: validBaseVolumeID}, nil) + clientMock.On("GetNFSExportByFileSystemID", mock.Anything, validBaseVolumeID). + Return(gopowerstore.NFSExport{ + ID: "some-export-id", + ROHosts: []string{}, + }, nil) + + req := &csi.NodeGetVolumeStatsRequest{VolumeId: validNfsVolumeID, VolumePath: validTargetPath} + res, err := nodeSvc.NodeGetVolumeStats(context.Background(), req) + + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeGetVolumeStatsResponse{ + Usage: usage, + VolumeCondition: &csi.VolumeCondition{ + Abnormal: true, + Message: fmt.Sprintf("host %s is not attached to NFS export for filesystem %s", validNodeID, validBaseVolumeID), + }, + })) + }) + }) + + ginkgo.When("NFS export found as expected [NFS]", func() { + ginkgo.It("should return stats as abnormal with ReadDir() error", func() { + clientMock.On("GetFS", mock.Anything, validBaseVolumeID). + Return(gopowerstore.FileSystem{ID: validBaseVolumeID}, nil) + clientMock.On("GetNFSExportByFileSystemID", mock.Anything, validBaseVolumeID). + Return(gopowerstore.NFSExport{ + ID: "some-export-id", + ROHosts: []string{"127.0.0.1/255.255.255.0"}, + }, nil) + fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil) + fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return([]gofsutil.Info{ + { + Device: validDevName, + Path: validTargetPath, + }, + }, nil) + + req := &csi.NodeGetVolumeStatsRequest{ + VolumeId: validNfsVolumeID, + VolumePath: validTargetPath, + StagingTargetPath: "", + } + res, err := nodeSvc.NodeGetVolumeStats(context.Background(), req) + + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeGetVolumeStatsResponse{ + Usage: usage, + VolumeCondition: &csi.VolumeCondition{ + Abnormal: true, + Message: fmt.Sprintf("volume path %s not accessible for volume %s", validTargetPath, validBaseVolumeID), + }, + })) + }) + }) + + ginkgo.When("there are issues with mount paths", func() { + ginkgo.BeforeEach(func() { + clientMock.On("GetVolume", mock.Anything, validBaseVolumeID). + Return(gopowerstore.Volume{ID: validBaseVolumeID, Size: controller.MaxVolumeSizeBytes / 200}, nil) + clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, validBaseVolumeID). + Return([]gopowerstore.HostVolumeMapping{{HostID: validHostID, LogicalUnitNumber: 1}}, nil) + clientMock.On("GetHost", mock.Anything, validHostID). + Return(gopowerstore.Host{ + ID: validHostID, + Name: validNodeID, + Initiators: []gopowerstore.InitiatorInstance{ + { + ActiveSessions: []gopowerstore.ActiveSessionInstance{ + { + PortName: validISCSITargets[0], + }, + }, + PortName: validISCSIPortals[0], + PortType: gopowerstore.InitiatorProtocolTypeEnumISCSI, + }, + }, + }, nil) + }) + + ginkgo.It("should fail for getTargetMount() error [stagingPath]", func() { + fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, errors.New("fail")) + + req := &csi.NodeGetVolumeStatsRequest{ + VolumeId: validBlockVolumeHandle, + VolumePath: validTargetPath, + StagingTargetPath: nodeStagePrivateDir, + } + res, err := nodeSvc.NodeGetVolumeStats(context.Background(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("can't check mounts for path")) + }) + + ginkgo.It("should return stats as abnormal for getTargetMount() error [stagingPath]", func() { + fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil) + fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return([]gofsutil.Info{}, nil) + + req := &csi.NodeGetVolumeStatsRequest{ + VolumeId: validBlockVolumeHandle, + VolumePath: validTargetPath, + StagingTargetPath: nodeStagePrivateDir, + } + res, err := nodeSvc.NodeGetVolumeStats(context.Background(), req) + + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeGetVolumeStatsResponse{ + Usage: usage, + VolumeCondition: &csi.VolumeCondition{ + Abnormal: true, + Message: fmt.Sprintf("staging target path %s not mounted for volume %s", nodeStagePrivateDir, validBaseVolumeID), + }, + })) + }) + + ginkgo.It("should fail for getTargetMount() error [volumePath]", func() { + fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, errors.New("fail")) + + req := &csi.NodeGetVolumeStatsRequest{ + VolumeId: validBlockVolumeHandle, + VolumePath: validTargetPath, + StagingTargetPath: "", + } + res, err := nodeSvc.NodeGetVolumeStats(context.Background(), req) + + gomega.Expect(res).To(gomega.BeNil()) + gomega.Expect(err).ToNot(gomega.BeNil()) + gomega.Expect(err.Error()).To(gomega.ContainSubstring("can't check mounts for path")) + }) + + ginkgo.It("should return stats as abnormal for getTargetMount() error [volumePath]", func() { + fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil) + fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return([]gofsutil.Info{}, nil) + + req := &csi.NodeGetVolumeStatsRequest{ + VolumeId: validBlockVolumeHandle, + VolumePath: validTargetPath, + StagingTargetPath: "", + } + res, err := nodeSvc.NodeGetVolumeStats(context.Background(), req) + + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeGetVolumeStatsResponse{ + Usage: usage, + VolumeCondition: &csi.VolumeCondition{ + Abnormal: true, + Message: fmt.Sprintf("volume path %s not mounted for volume %s", validTargetPath, validBaseVolumeID), + }, + })) + }) + + ginkgo.It("should return stats as abnormal for ReadDir() error", func() { + fsMock.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil) + fsMock.On("ParseProcMounts", context.Background(), mock.Anything).Return([]gofsutil.Info{ + { + Device: validDevName, + Path: validTargetPath, + }, + }, nil) + + req := &csi.NodeGetVolumeStatsRequest{ + VolumeId: validBlockVolumeHandle, + VolumePath: validTargetPath, + StagingTargetPath: "", + } + res, err := nodeSvc.NodeGetVolumeStats(context.Background(), req) + + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(res).To(gomega.Equal(&csi.NodeGetVolumeStatsResponse{ + Usage: usage, + VolumeCondition: &csi.VolumeCondition{ + Abnormal: true, + Message: fmt.Sprintf("volume path %s not accessible for volume %s", validTargetPath, validBaseVolumeID), + }, + })) + }) + }) + }) +}) + +func TestInitConnectors(t *testing.T) { + arrays := getTestArrays() + nodeSvc = &Service{ + Fs: fsMock, + ctrlSvc: nil, + iscsiConnector: nil, + fcConnector: nil, + iscsiLib: nil, + nodeID: validNodeID, + initialized: true, + } + nodeSvc.SetArrays(arrays) + nodeSvc.SetDefaultArray(arrays[firstValidIP]) + t.Run("success test", func(_ *testing.T) { + nodeSvc.initConnectors() + }) +} + +func TestGetNodeOptions(t *testing.T) { + arrays := getTestArrays() + nodeSvc = &Service{ + Fs: fsMock, + ctrlSvc: nil, + iscsiConnector: nil, + fcConnector: nil, + iscsiLib: nil, nodeID: validNodeID, - useFC: false, initialized: true, } - nodeSvc.SetArrays(arrays) - nodeSvc.SetDefaultArray(arrays[firstValidIP]) - t.Run("success test", func(t *testing.T) { - csictx.Setenv(context.Background(), common.EnvNodeIDFilePath, "") - csictx.Setenv(context.Background(), common.EnvNodeNamePrefix, "") - csictx.Setenv(context.Background(), common.EnvKubeNodeName, "") - csictx.Setenv(context.Background(), common.EnvNodeChrootPath, "") - csictx.Setenv(context.Background(), common.EnvTmpDir, "") - csictx.Setenv(context.Background(), common.EnvFCPortsFilterFilePath, "") - csictx.Setenv(context.Background(), common.EnvEnableCHAP, "") - getNodeOptions() + nodeSvc.SetArrays(arrays) + nodeSvc.SetDefaultArray(arrays[firstValidIP]) + + t.Run("success test with valid maxVolumesPerNode", func(_ *testing.T) { + ctx := context.Background() + csictx.Setenv(ctx, identifiers.EnvNodeIDFilePath, "") + csictx.Setenv(ctx, identifiers.EnvNodeNamePrefix, "") + csictx.Setenv(ctx, identifiers.EnvKubeNodeName, "") + csictx.Setenv(ctx, identifiers.EnvNodeChrootPath, "") + csictx.Setenv(ctx, identifiers.EnvTmpDir, "") + csictx.Setenv(ctx, identifiers.EnvFCPortsFilterFilePath, "") + csictx.Setenv(ctx, identifiers.EnvEnableCHAP, "") + csictx.Setenv(ctx, identifiers.EnvMaxVolumesPerNode, "42") // ✅ valid value + csictx.Setenv(ctx, identifiers.EnvKubeConfigPath, "myConfigPath") + + opts := getNodeOptions() + if opts.MaxVolumesPerNode != 42 { + t.Errorf("expected MaxVolumesPerNode to be 42, got %d", opts.MaxVolumesPerNode) + } + }) + + t.Run("fallback test with invalid maxVolumesPerNode", func(_ *testing.T) { + ctx := context.Background() + csictx.Setenv(ctx, identifiers.EnvMaxVolumesPerNode, "invalid") // ❌ invalid value + + opts := getNodeOptions() + if opts.MaxVolumesPerNode != 0 { + t.Errorf("expected MaxVolumesPerNode to default to 0, got %d", opts.MaxVolumesPerNode) + } + }) +} + +func getNodeVolumeExpandValidRequest(volid string, isBlock bool) *csi.NodeExpandVolumeRequest { + var size int64 = controller.MaxVolumeSizeBytes / 100 + if !isBlock { + req := csi.NodeExpandVolumeRequest{ + VolumeId: volid, + VolumePath: validTargetPath, + CapacityRange: &csi.CapacityRange{ + RequiredBytes: size, + LimitBytes: controller.MaxVolumeSizeBytes, + }, + } + return &req + } + req := csi.NodeExpandVolumeRequest{ + VolumeId: volid, + VolumePath: validTargetPath + "/csi/volumeDevices/publish/", + CapacityRange: &csi.CapacityRange{ + RequiredBytes: size, + LimitBytes: controller.MaxVolumeSizeBytes, + }, + } + return &req +} + +type MockService struct { + // Add fields to mock dependencies if needed + *Service +} + +func TestIsRemoteToOtherArray(t *testing.T) { + originalGetAllRemoteSystemsFunc := getAllRemoteSystemsFunc + defer func() { + getAllRemoteSystemsFunc = originalGetAllRemoteSystemsFunc + }() + tests := []struct { + name string + s *Service + arrA *array.PowerStoreArray + arrB *array.PowerStoreArray + setupMocks func() + wantErr bool + want bool + }{ + { + name: "Array B is not remote to Array A", + arrA: &array.PowerStoreArray{GlobalID: "arrayA"}, + arrB: &array.PowerStoreArray{GlobalID: "arrayB"}, + setupMocks: func() { + getAllRemoteSystemsFunc = func(arr *array.PowerStoreArray, _ context.Context) ([]gopowerstore.RemoteSystem, error) { + if arr.GlobalID == "arrayA" { + return []gopowerstore.RemoteSystem{ + { + ID: "arrayid1", + Name: "Pstore1", + Description: "", + SerialNumber: "arrayC", + ManagementAddress: "10.198.0.1", + DataConnectionState: "OK", + Capabilities: []string{"Synchronous_Block_Replication"}, + }, + }, nil + } + + return []gopowerstore.RemoteSystem{ + { + ID: "arrayid2", + Name: "Pstore2", + Description: "", + SerialNumber: "arrayD", + ManagementAddress: "10.198.0.2", + DataConnectionState: "OK", + Capabilities: []string{"Synchronous_Block_Replication"}, + }, + }, nil + } + }, + wantErr: false, + want: false, + }, + { + name: "Error fetching remotes for Array A", + arrA: &array.PowerStoreArray{GlobalID: "arrayA"}, + arrB: &array.PowerStoreArray{GlobalID: "arrayB"}, + setupMocks: func() { + getAllRemoteSystemsFunc = func(arr *array.PowerStoreArray, _ context.Context) ([]gopowerstore.RemoteSystem, error) { + if arr.GlobalID == "arrayA" { + return nil, fmt.Errorf("failed to get remoteSystem") + } + + return nil, nil + } + }, + wantErr: true, + want: false, + }, + { + name: "Error fetching remotes for Array B", + arrA: &array.PowerStoreArray{GlobalID: "arrayA"}, + arrB: &array.PowerStoreArray{GlobalID: "arrayB"}, + setupMocks: func() { + getAllRemoteSystemsFunc = func(arr *array.PowerStoreArray, _ context.Context) ([]gopowerstore.RemoteSystem, error) { + if arr.GlobalID == "arrayB" { + return nil, fmt.Errorf("failed to get remoteSystem") + } + + return nil, nil + } + }, + wantErr: true, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.setupMocks() + got := tt.s.isRemoteToOtherArray(context.Background(), tt.arrA, tt.arrB) + + if got == tt.want { + log.Info("Success") + } else { + t.Errorf("Service.isRemoteToOtherArray() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestHandleNoLabelMatchRegistration(t *testing.T) { + originalGetArrayfn := getArrayfn + originalGetIsHostAlreadyRegistered := getIsHostAlreadyRegistered + originalGetAllRemoteSystemsFunc := getAllRemoteSystemsFunc + originalGetIsRemoteToOtherArray := getIsRemoteToOtherArray + originalRegisterHostFunc := registerHostFunc + + defer func() { + getArrayfn = originalGetArrayfn + getIsHostAlreadyRegistered = originalGetIsHostAlreadyRegistered + getAllRemoteSystemsFunc = originalGetAllRemoteSystemsFunc + getIsRemoteToOtherArray = originalGetIsRemoteToOtherArray + registerHostFunc = originalRegisterHostFunc + }() + + tests := []struct { + s *MockService + name string + initiators []string + nodeLabels map[string]string + arrayAddedList map[string]bool + arr *array.PowerStoreArray + setupMocks func() + wantErr bool + want bool + }{ + { + name: "No array labels match node labels", + initiators: []string{"init1"}, + nodeLabels: map[string]string{"topology.kubernetes.io/zone1": "zone1"}, + arrayAddedList: map[string]bool{}, + arr: &array.PowerStoreArray{ + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone2": "zone2"}, + IP: "10.198.0.1", + }, + setupMocks: func() { + getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { + log.Info("InsideGetArray") + + return map[string]*array.PowerStoreArray{ + "Array1": { + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone2"}, + IP: "10.198.0.1", + }, + "Array2": { + Endpoint: "https://10.198.0.2/api/rest", + GlobalID: "Array2", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone2"}, + IP: "10.198.0.2", + }, + } + } + + getIsHostAlreadyRegistered = func(_ *Service, _ context.Context, _ gopowerstore.Client, _ []string) bool { + return false + } + + getIsRemoteToOtherArray = func(_ *Service, _ context.Context, _, _ *array.PowerStoreArray) bool { + return true + } + }, + wantErr: false, + want: true, + }, + { + name: "Success Host Registration", + initiators: []string{"init1"}, + nodeLabels: map[string]string{"topology.kubernetes.io/zone1": "zone1", "topology.kubernetes.io/zone2": "zone2"}, + arrayAddedList: map[string]bool{}, + arr: &array.PowerStoreArray{ + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone3"}, + IP: "10.198.0.1", + }, + setupMocks: func() { + getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { + log.Info("InsideGetArray") + + return map[string]*array.PowerStoreArray{ + "Array1": { + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone1"}, + IP: "10.198.0.1", + }, + "Array2": { + Endpoint: "https://10.198.0.2/api/rest", + GlobalID: "Array2", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone2": "zone2"}, + IP: "10.198.0.2", + }, + } + } + + getAllRemoteSystemsFunc = func(arr *array.PowerStoreArray, _ context.Context) ([]gopowerstore.RemoteSystem, error) { + if arr.GlobalID == "Array2" { + return []gopowerstore.RemoteSystem{ + { + ID: "arrayid1", + Name: "Pstore1", + Description: "", + SerialNumber: "Array1", + ManagementAddress: "10.198.0.1", + DataConnectionState: "OK", + Capabilities: []string{"Synchronous_Block_Replication"}, + }, + }, nil + } + + return []gopowerstore.RemoteSystem{ + { + ID: "arrayid2", + Name: "Pstore2", + Description: "", + SerialNumber: "Array2", + ManagementAddress: "10.198.0.2", + DataConnectionState: "OK", + Capabilities: []string{"Synchronous_Block_Replication"}, + }, + }, nil + } + + getIsHostAlreadyRegistered = func(_ *Service, _ context.Context, _ gopowerstore.Client, _ []string) bool { + return false + } + + getIsRemoteToOtherArray = func(_ *Service, _ context.Context, _, _ *array.PowerStoreArray) bool { + return true + } + + registerHostFunc = func(_ *Service, _ context.Context, _ gopowerstore.Client, _ string, _ []string, _ gopowerstore.HostConnectivityEnum) error { + log.Info("Inside RegisterHost") + return nil + } + }, + wantErr: false, + want: true, + }, + { + name: "Host Already register", + initiators: []string{"init1"}, + nodeLabels: map[string]string{"topology.kubernetes.io/zone1": "zone1", "topology.kubernetes.io/zone2": "zone2"}, + arrayAddedList: map[string]bool{}, + arr: &array.PowerStoreArray{ + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone3"}, + IP: "10.198.0.1", + }, + setupMocks: func() { + getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { + log.Info("InsideGetArray") + + return map[string]*array.PowerStoreArray{ + "Array1": { + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone1"}, + IP: "10.198.0.1", + }, + "Array2": { + Endpoint: "https://10.198.0.2/api/rest", + GlobalID: "Array2", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone2": "zone2"}, + IP: "10.198.0.2", + }, + } + } + + getAllRemoteSystemsFunc = func(arr *array.PowerStoreArray, _ context.Context) ([]gopowerstore.RemoteSystem, error) { + if arr.GlobalID == "Array2" { + return []gopowerstore.RemoteSystem{ + { + ID: "arrayid1", + Name: "Pstore1", + Description: "", + SerialNumber: "Array1", + ManagementAddress: "10.198.0.1", + DataConnectionState: "OK", + Capabilities: []string{"Synchronous_Block_Replication"}, + }, + }, nil + } + + return []gopowerstore.RemoteSystem{ + { + ID: "arrayid2", + Name: "Pstore2", + Description: "", + SerialNumber: "Array2", + ManagementAddress: "10.198.0.2", + DataConnectionState: "OK", + Capabilities: []string{"Synchronous_Block_Replication"}, + }, + }, nil + } + + getIsHostAlreadyRegistered = func(_ *Service, _ context.Context, _ gopowerstore.Client, _ []string) bool { + return true + } + + getIsRemoteToOtherArray = func(_ *Service, _ context.Context, _, _ *array.PowerStoreArray) bool { + return true + } + + registerHostFunc = func(_ *Service, _ context.Context, _ gopowerstore.Client, _ string, _ []string, _ gopowerstore.HostConnectivityEnum) error { + log.Info("Inside RegisterHost") + return nil + } + }, + wantErr: false, + want: true, + }, + { + name: "Local connectivity", + initiators: []string{"init1"}, + nodeLabels: map[string]string{"topology.kubernetes.io/zone1": "zone1", "topology.kubernetes.io/zone2": "zone2"}, + arrayAddedList: map[string]bool{}, + arr: &array.PowerStoreArray{ + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone1"}, + IP: "10.198.0.1", + }, + setupMocks: func() { + getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { + log.Info("InsideGetArray") + + return map[string]*array.PowerStoreArray{ + "Array1": { + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone1"}, + IP: "10.198.0.1", + }, + "Array2": { + Endpoint: "https://10.198.0.2/api/rest", + GlobalID: "Array2", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone1"}, + IP: "10.198.0.2", + }, + } + } + + getAllRemoteSystemsFunc = func(arr *array.PowerStoreArray, _ context.Context) ([]gopowerstore.RemoteSystem, error) { + if arr.GlobalID == "Array2" { + return []gopowerstore.RemoteSystem{ + { + ID: "arrayid1", + Name: "Pstore1", + Description: "", + SerialNumber: "Array1", + ManagementAddress: "10.198.0.1", + DataConnectionState: "OK", + Capabilities: []string{"Synchronous_Block_Replication"}, + }, + }, nil + } + + return []gopowerstore.RemoteSystem{ + { + ID: "arrayid2", + Name: "Pstore2", + Description: "", + SerialNumber: "Array2", + ManagementAddress: "10.198.0.2", + DataConnectionState: "OK", + Capabilities: []string{"Synchronous_Block_Replication"}, + }, + }, nil + } + + getIsHostAlreadyRegistered = func(_ *Service, _ context.Context, _ gopowerstore.Client, _ []string) bool { + return false + } + + getIsRemoteToOtherArray = func(_ *Service, _ context.Context, _, _ *array.PowerStoreArray) bool { + return true + } + + registerHostFunc = func(_ *Service, _ context.Context, _ gopowerstore.Client, _ string, _ []string, _ gopowerstore.HostConnectivityEnum) error { + log.Info("Inside RegisterHost") + return nil + } + }, + wantErr: false, + want: true, + }, + { + name: "Failed to Register host", + initiators: []string{"init1"}, + nodeLabels: map[string]string{"topology.kubernetes.io/zone1": "zone1", "topology.kubernetes.io/zone2": "zone2"}, + arrayAddedList: map[string]bool{}, + arr: &array.PowerStoreArray{ + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone1"}, + IP: "10.198.0.1", + }, + setupMocks: func() { + getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { + log.Info("InsideGetArray") + + return map[string]*array.PowerStoreArray{ + "Array1": { + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone1"}, + IP: "10.198.0.1", + }, + "Array2": { + Endpoint: "https://10.198.0.2/api/rest", + GlobalID: "Array2", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone2": "zone2"}, + IP: "10.198.0.2", + }, + } + } + + getAllRemoteSystemsFunc = func(arr *array.PowerStoreArray, _ context.Context) ([]gopowerstore.RemoteSystem, error) { + if arr.GlobalID == "Array2" { + return []gopowerstore.RemoteSystem{ + { + ID: "arrayid1", + Name: "Pstore1", + Description: "", + SerialNumber: "Array1", + ManagementAddress: "10.198.0.1", + DataConnectionState: "OK", + Capabilities: []string{"Synchronous_Block_Replication"}, + }, + }, nil + } + + return []gopowerstore.RemoteSystem{ + { + ID: "arrayid2", + Name: "Pstore2", + Description: "", + SerialNumber: "Array2", + ManagementAddress: "10.198.0.2", + DataConnectionState: "OK", + Capabilities: []string{"Synchronous_Block_Replication"}, + }, + }, nil + } + + getIsHostAlreadyRegistered = func(_ *Service, _ context.Context, _ gopowerstore.Client, _ []string) bool { + return false + } + + getIsRemoteToOtherArray = func(_ *Service, _ context.Context, _, _ *array.PowerStoreArray) bool { + return true + } + + registerHostFunc = func(_ *Service, _ context.Context, _ gopowerstore.Client, _ string, _ []string, _ gopowerstore.HostConnectivityEnum) error { + log.Info("Inside RegisterHost") + return fmt.Errorf("failed to registerHost") + } + }, + wantErr: true, + want: false, + }, + { + name: "Fail to get remote system", + initiators: []string{"init1"}, + nodeLabels: map[string]string{"topology.kubernetes.io/zone1": "zone1", "topology.kubernetes.io/zone2": "zone2"}, + arrayAddedList: map[string]bool{}, + arr: &array.PowerStoreArray{ + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone1"}, + IP: "10.198.0.1", + }, + setupMocks: func() { + getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { + log.Info("InsideGetArray") + + return map[string]*array.PowerStoreArray{ + "Array1": { + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone1"}, + IP: "10.198.0.1", + }, + "Array2": { + Endpoint: "https://10.198.0.2/api/rest", + GlobalID: "Array2", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone2": "zone2"}, + IP: "10.198.0.2", + }, + } + } + + getAllRemoteSystemsFunc = func(_ *array.PowerStoreArray, _ context.Context) ([]gopowerstore.RemoteSystem, error) { + log.Info("Inside Remote Systems") + return nil, fmt.Errorf("failed to get remoteSystem") + } + + getIsHostAlreadyRegistered = func(_ *Service, _ context.Context, _ gopowerstore.Client, _ []string) bool { + return false + } + + getIsRemoteToOtherArray = func(_ *Service, _ context.Context, _, _ *array.PowerStoreArray) bool { + return true + } + + registerHostFunc = func(_ *Service, _ context.Context, _ gopowerstore.Client, _ string, _ []string, _ gopowerstore.HostConnectivityEnum) error { + log.Info("Inside RegisterHost") + return fmt.Errorf("failed to registerHost") + } + }, + wantErr: true, + want: false, + }, + // Add more test cases here + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mockService := new(MockService) + tt.setupMocks() + + log.Info("Test") + got, err := mockService.handleNoLabelMatchRegistration(context.Background(), tt.arr, tt.initiators, tt.nodeLabels, tt.arrayAddedList) + + if (err != nil) != tt.wantErr { + t.Errorf("Service.handleLabelMatchRegistration() error = %v, wantErr %v", err, tt.wantErr) + return + } + + if got == tt.want { + log.Info("Success") + } + }) + } +} + +func TestHandleLabelMatchRegistration(t *testing.T) { + originalGetArrayfn := getArrayfn + originalGetIsHostAlreadyRegistered := getIsHostAlreadyRegistered + originalGetAllRemoteSystemsFunc := getAllRemoteSystemsFunc + originalGetIsRemoteToOtherArray := getIsRemoteToOtherArray + originalRegisterHostFunc := registerHostFunc + + defer func() { + getArrayfn = originalGetArrayfn + getIsHostAlreadyRegistered = originalGetIsHostAlreadyRegistered + getAllRemoteSystemsFunc = originalGetAllRemoteSystemsFunc + getIsRemoteToOtherArray = originalGetIsRemoteToOtherArray + registerHostFunc = originalRegisterHostFunc + }() + + tests := []struct { + s *MockService + name string + initiators []string + nodeLabels map[string]string + arrayAddedList map[string]bool + arr *array.PowerStoreArray + setupMocks func() + wantErr bool + want bool + }{ + { + name: "No array labels match node labels", + initiators: []string{"init1"}, + nodeLabels: map[string]string{"topology.kubernetes.io/zone1": "zone1"}, + arrayAddedList: map[string]bool{}, + arr: &array.PowerStoreArray{ + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone2": "zone2"}, + IP: "10.198.0.1", + }, + setupMocks: func() { + getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { + log.Info("InsideGetArray") + + return map[string]*array.PowerStoreArray{ + "Array1": { + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone2"}, + IP: "10.198.0.1", + }, + "Array2": { + Endpoint: "https://10.198.0.2/api/rest", + GlobalID: "Array2", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone2"}, + IP: "10.198.0.2", + }, + } + } + + getIsHostAlreadyRegistered = func(_ *Service, _ context.Context, _ gopowerstore.Client, _ []string) bool { + return false + } + + getIsRemoteToOtherArray = func(_ *Service, _ context.Context, _, _ *array.PowerStoreArray) bool { + return true + } + }, + wantErr: false, + want: false, + }, + { + name: "No array labels match node labels -2", + initiators: []string{"init1"}, + nodeLabels: map[string]string{"topology.kubernetes.io/zone1": "zone1", "topology.kubernetes.io/zone2": "zone2"}, + arrayAddedList: map[string]bool{}, + arr: &array.PowerStoreArray{ + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone3"}, + IP: "10.198.0.1", + }, + setupMocks: func() { + getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { + log.Info("InsideGetArray") + + return map[string]*array.PowerStoreArray{ + "Array1": { + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone1"}, + IP: "10.198.0.1", + }, + "Array2": { + Endpoint: "https://10.198.0.2/api/rest", + GlobalID: "Array2", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone2": "zone2"}, + IP: "10.198.0.2", + }, + } + } + + getAllRemoteSystemsFunc = func(arr *array.PowerStoreArray, _ context.Context) ([]gopowerstore.RemoteSystem, error) { + if arr.GlobalID == "Array2" { + return []gopowerstore.RemoteSystem{ + { + ID: "arrayid1", + Name: "Pstore1", + Description: "", + SerialNumber: "Array1", + ManagementAddress: "10.198.0.1", + DataConnectionState: "OK", + Capabilities: []string{"Synchronous_Block_Replication"}, + }, + }, nil + } + + return []gopowerstore.RemoteSystem{ + { + ID: "arrayid2", + Name: "Pstore2", + Description: "", + SerialNumber: "Array2", + ManagementAddress: "10.198.0.2", + DataConnectionState: "OK", + Capabilities: []string{"Synchronous_Block_Replication"}, + }, + }, nil + } + + getIsHostAlreadyRegistered = func(_ *Service, _ context.Context, _ gopowerstore.Client, _ []string) bool { + return false + } + + getIsRemoteToOtherArray = func(_ *Service, _ context.Context, _, _ *array.PowerStoreArray) bool { + return true + } + + registerHostFunc = func(_ *Service, _ context.Context, _ gopowerstore.Client, _ string, _ []string, _ gopowerstore.HostConnectivityEnum) error { + log.Info("Inside RegisterHost") + return nil + } + }, + wantErr: false, + want: false, + }, + { + name: "No array labels match node labels - 3", + initiators: []string{"init1"}, + nodeLabels: map[string]string{"topology.kubernetes.io/zone1": "zone1", "topology.kubernetes.io/zone2": "zone2"}, + arrayAddedList: map[string]bool{}, + arr: &array.PowerStoreArray{ + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone3"}, + IP: "10.198.0.1", + }, + setupMocks: func() { + getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { + log.Info("InsideGetArray") + + return map[string]*array.PowerStoreArray{ + "Array1": { + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone1"}, + IP: "10.198.0.1", + }, + "Array2": { + Endpoint: "https://10.198.0.2/api/rest", + GlobalID: "Array2", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone2": "zone2"}, + IP: "10.198.0.2", + }, + } + } + + getAllRemoteSystemsFunc = func(arr *array.PowerStoreArray, _ context.Context) ([]gopowerstore.RemoteSystem, error) { + if arr.GlobalID == "Array2" { + return []gopowerstore.RemoteSystem{ + { + ID: "arrayid1", + Name: "Pstore1", + Description: "", + SerialNumber: "Array1", + ManagementAddress: "10.198.0.1", + DataConnectionState: "OK", + Capabilities: []string{"Synchronous_Block_Replication"}, + }, + }, nil + } + + return []gopowerstore.RemoteSystem{ + { + ID: "arrayid2", + Name: "Pstore2", + Description: "", + SerialNumber: "Array2", + ManagementAddress: "10.198.0.2", + DataConnectionState: "OK", + Capabilities: []string{"Synchronous_Block_Replication"}, + }, + }, nil + } + + getIsHostAlreadyRegistered = func(_ *Service, _ context.Context, _ gopowerstore.Client, _ []string) bool { + return false + } + + getIsRemoteToOtherArray = func(_ *Service, _ context.Context, _, _ *array.PowerStoreArray) bool { + return true + } + + registerHostFunc = func(_ *Service, _ context.Context, _ gopowerstore.Client, _ string, _ []string, _ gopowerstore.HostConnectivityEnum) error { + log.Info("Inside RegisterHost") + return fmt.Errorf("failed to registerHost") + } + }, + wantErr: true, + want: true, + }, + { + name: "Host Already Registered", + initiators: []string{"init1"}, + nodeLabels: map[string]string{"topology.kubernetes.io/zone1": "zone1"}, + arrayAddedList: map[string]bool{}, + arr: &array.PowerStoreArray{ + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone1"}, + IP: "10.198.0.1", + }, + setupMocks: func() { + getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { + log.Info("InsideGetArray") + + return map[string]*array.PowerStoreArray{ + "Array1": { + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone1"}, + IP: "10.198.0.1", + }, + "Array2": { + Endpoint: "https://10.198.0.2/api/rest", + GlobalID: "Array2", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone1"}, + IP: "10.198.0.2", + }, + } + } + + getAllRemoteSystemsFunc = func(arr *array.PowerStoreArray, _ context.Context) ([]gopowerstore.RemoteSystem, error) { + if arr.GlobalID == "Array2" { + return []gopowerstore.RemoteSystem{ + { + ID: "arrayid1", + Name: "Pstore1", + Description: "", + SerialNumber: "Array1", + ManagementAddress: "10.198.0.1", + DataConnectionState: "OK", + Capabilities: []string{"Synchronous_Block_Replication"}, + }, + }, nil + } + + return []gopowerstore.RemoteSystem{ + { + ID: "arrayid2", + Name: "Pstore2", + Description: "", + SerialNumber: "Array2", + ManagementAddress: "10.198.0.2", + DataConnectionState: "OK", + Capabilities: []string{"Synchronous_Block_Replication"}, + }, + }, nil + } + + getIsHostAlreadyRegistered = func(_ *Service, _ context.Context, _ gopowerstore.Client, _ []string) bool { + return true + } + + getIsRemoteToOtherArray = func(_ *Service, _ context.Context, _, _ *array.PowerStoreArray) bool { + return true + } + + registerHostFunc = func(_ *Service, _ context.Context, _ gopowerstore.Client, _ string, _ []string, _ gopowerstore.HostConnectivityEnum) error { + log.Info("Inside RegisterHost") + return fmt.Errorf("failed to registerHost") + } + }, + wantErr: false, + want: false, + }, + // Add more test cases here + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mockService := new(MockService) + tt.setupMocks() + + log.Info("Test") + got, err := mockService.handleLabelMatchRegistration(context.Background(), tt.arr, tt.initiators, tt.nodeLabels, tt.arrayAddedList) + + if (err != nil) != tt.wantErr { + t.Errorf("Service.handleLabelMatchRegistration() error = %v, wantErr %v", err, tt.wantErr) + return + } + + if got == tt.want { + log.Info("Test passed") + } + }) + } +} + +// Unit test for createHost +func TestService_createHost(t *testing.T) { + defaultK8sConfigFunc := k8sutils.InClusterConfigFunc + defaultK8sClientsetFunc := k8sutils.NewForConfigFunc + + beforeEach := func() { + k8sutils.InClusterConfigFunc = func() (*rest.Config, error) { + return &rest.Config{}, nil + } + k8sutils.NewForConfigFunc = func(_ *rest.Config) (kubernetes.Interface, error) { + return fake.NewClientset(), nil + } + + // Base initialize k8sclient + k8sutils.Kubeclient = &k8sutils.K8sClient{ + Clientset: fake.NewClientset([]runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone1"}, + }, + }, + }...), + } + } + + afterEach := func() { + k8sutils.InClusterConfigFunc = defaultK8sConfigFunc + k8sutils.NewForConfigFunc = defaultK8sClientsetFunc + } + + originalGetArrayfn := getArrayfn + originalGetIsHostAlreadyRegistered := getIsHostAlreadyRegistered + originalGetAllRemoteSystemsFunc := getAllRemoteSystemsFunc + originalGetIsRemoteToOtherArray := getIsRemoteToOtherArray + originalRegisterHostFunc := registerHostFunc + clientMock = new(gopowerstoremock.Client) + clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) + clientMock.On("CreateHost", mock.Anything, mock.Anything). + Return(gopowerstore.CreateResponse{ID: validHostID}, nil) + clientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(3.0), nil) + clientMock.On("GetCustomHTTPHeaders").Return(api.NewSafeHeader().GetHeader()) + + defer func() { + getArrayfn = originalGetArrayfn + getIsHostAlreadyRegistered = originalGetIsHostAlreadyRegistered + getAllRemoteSystemsFunc = originalGetAllRemoteSystemsFunc + getIsRemoteToOtherArray = originalGetIsRemoteToOtherArray + registerHostFunc = originalRegisterHostFunc + }() + + getIsHostAlreadyRegistered = func(_ *Service, _ context.Context, _ gopowerstore.Client, _ []string) bool { + return false + } + + getAllRemoteSystemsFunc = func(arr *array.PowerStoreArray, _ context.Context) ([]gopowerstore.RemoteSystem, error) { + if arr.GlobalID == "Array2" { + return []gopowerstore.RemoteSystem{ + { + ID: "arrayid1", + Name: "Pstore1", + Description: "", + SerialNumber: "Array1", + ManagementAddress: "10.198.0.1", + DataConnectionState: "OK", + Capabilities: []string{"Synchronous_Block_Replication"}, + }, + }, nil + } + + return []gopowerstore.RemoteSystem{ + { + ID: "arrayid2", + Name: "Pstore2", + Description: "", + SerialNumber: "Array2", + ManagementAddress: "10.198.0.2", + DataConnectionState: "OK", + Capabilities: []string{"Synchronous_Block_Replication"}, + }, + }, nil + } + + type args struct { + ctx context.Context + initiators []string + } + + tests := []struct { + name string + s *MockService + setup func() + args args + want []string + wantErr bool + }{ + { + name: "Successful host creation 1", + s: &MockService{ + Service: &Service{ + opts: Opts{ + KubeNodeName: "node1", + }, + }, + }, + args: args{ + ctx: context.TODO(), + initiators: []string{"initiator1", "initiator2"}, + }, + setup: func() { + getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { + return map[string]*array.PowerStoreArray{ + "Array1": { + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone2"}, + IP: "10.198.0.1", + Client: clientMock, + }, + "Array2": { + Endpoint: "https://10.198.0.2/api/rest", + GlobalID: "Array2", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone1"}, + IP: "10.198.0.2", + Client: clientMock, + }, + } + } + }, + want: []string{"Array1", "Array2"}, + wantErr: false, + }, + { + name: "Successful host creation 2", + s: &MockService{ + Service: &Service{ + opts: Opts{KubeNodeName: "node1"}, + }, + }, + args: args{ + ctx: context.TODO(), + initiators: []string{"initiator1", "initiator2"}, + }, + setup: func() { + getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { + return map[string]*array.PowerStoreArray{ + "Array1": { + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone1"}, + IP: "10.198.0.1", + Client: clientMock, + }, + "Array2": { + Endpoint: "https://10.198.0.2/api/rest", + GlobalID: "Array2", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone2"}, + IP: "10.198.0.2", + Client: clientMock, + }, + } + } + }, + want: []string{"Array1", "Array2"}, + wantErr: false, + }, + { + name: "Successful host creation 3", + s: &MockService{ + Service: &Service{ + opts: Opts{KubeNodeName: "node1"}, + }, + }, + args: args{ + ctx: context.TODO(), + initiators: []string{"initiator1", "initiator2"}, + }, + setup: func() { + getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { + log.Info("InsideGetArray") + + return map[string]*array.PowerStoreArray{ + "Array1": { + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone1"}, + IP: "10.198.0.1", + Client: clientMock, + }, + "Array2": { + Endpoint: "https://10.198.0.2/api/rest", + GlobalID: "Array2", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone1"}, + IP: "10.198.0.2", + Client: clientMock, + }, + } + } + }, + want: []string{"Array1", "Array2"}, + wantErr: false, + }, + { + name: "Host Registration Success - For New HostConnectivity Secret - LocalOnly", + s: &MockService{ + Service: &Service{ + opts: Opts{KubeNodeName: "node1"}, + }, + }, + args: args{ + ctx: context.TODO(), + initiators: []string{"initiator1", "initiator2"}, + }, + setup: func() { + // Override the k8s client + k8sutils.Kubeclient = &k8sutils.K8sClient{ + Clientset: fake.NewClientset([]runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Labels: map[string]string{"topology.kubernetes.io/zone": "zone1"}, + }, + }, + }...), + } + getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { + return map[string]*array.PowerStoreArray{ + "Array1": { + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + HostConnectivity: &array.HostConnectivity{ + Local: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"zone1"}, + }, + }, + }, + }, + }, + }, + IP: "10.198.0.1", + Client: clientMock, + }, + } + } + + getIsHostAlreadyRegistered = func(_ *Service, _ context.Context, _ gopowerstore.Client, _ []string) bool { + return false + } + }, + want: []string{"Array1"}, + wantErr: false, + }, + { + name: "Host Registration Success - For New HostConnectivity Secret - Metro ColocatedLocal and Remote", + s: &MockService{ + Service: &Service{ + opts: Opts{KubeNodeName: "node1"}, + }, + }, + args: args{ + ctx: context.TODO(), + initiators: []string{"initiator1", "initiator2"}, + }, + setup: func() { + // Override the k8s client + k8sutils.Kubeclient = &k8sutils.K8sClient{ + Clientset: fake.NewClientset([]runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Labels: map[string]string{"topology.kubernetes.io/zone": "zone1"}, + }, + }, + }...), + } + getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { + return map[string]*array.PowerStoreArray{ + "Array1": { + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + HostConnectivity: &array.HostConnectivity{ + Metro: array.MetroConnectivityOptions{ + ColocatedLocal: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"zone1"}, + }, + }, + }, + }, + }, + }, + }, + IP: "10.198.0.1", + Client: clientMock, + }, + "Array2": { + Endpoint: "https://10.198.0.2/api/rest", + GlobalID: "Array2", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + HostConnectivity: &array.HostConnectivity{ + Metro: array.MetroConnectivityOptions{ + ColocatedRemote: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"zone1"}, + }, + }, + }, + }, + }, + }, + }, + IP: "10.198.0.2", + Client: clientMock, + }, + } + } + + getIsHostAlreadyRegistered = func(_ *Service, _ context.Context, _ gopowerstore.Client, _ []string) bool { + return false + } + }, + want: []string{"Array1", "Array2"}, + wantErr: false, + }, + { + name: "Host Registration Success - For New HostConnectivity Secret - Metro ColocatedBoth", + s: &MockService{ + Service: &Service{ + opts: Opts{KubeNodeName: "node1"}, + }, + }, + args: args{ + ctx: context.TODO(), + initiators: []string{"initiator1", "initiator2"}, + }, + setup: func() { + // Override the k8s client + k8sutils.Kubeclient = &k8sutils.K8sClient{ + Clientset: fake.NewClientset([]runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Labels: map[string]string{"topology.kubernetes.io/zone": "zone1"}, + }, + }, + }...), + } + getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { + return map[string]*array.PowerStoreArray{ + "Array1": { + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + HostConnectivity: &array.HostConnectivity{ + Metro: array.MetroConnectivityOptions{ + ColocatedBoth: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"zone1"}, + }, + }, + }, + }, + }, + }, + }, + IP: "10.198.0.1", + Client: clientMock, + }, + } + } + + getIsHostAlreadyRegistered = func(_ *Service, _ context.Context, _ gopowerstore.Client, _ []string) bool { + return false + } + }, + want: []string{"Array1"}, + wantErr: false, + }, + { + name: "Failure host creation - Label don't match", + s: &MockService{ + Service: &Service{ + opts: Opts{KubeNodeName: "node1"}, + }, + }, + args: args{ + ctx: context.TODO(), + initiators: []string{"initiator1", "initiator2"}, + }, + setup: func() { + getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { + log.Info("InsideGetArray") + + return map[string]*array.PowerStoreArray{ + "Array1": { + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone2"}, + IP: "10.198.0.1", + Client: clientMock, + }, + "Array2": { + Endpoint: "https://10.198.0.2/api/rest", + GlobalID: "Array2", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone2"}, + IP: "10.198.0.2", + Client: clientMock, + }, + } + } + }, + want: []string{}, + wantErr: true, + }, + { + name: "Host Registration Failure - For New HostConnectivity Secret - Label don't match", + s: &MockService{ + Service: &Service{ + opts: Opts{KubeNodeName: "node1"}, + }, + }, + args: args{ + ctx: context.TODO(), + initiators: []string{"initiator1", "initiator2"}, + }, + setup: func() { + // Override k8sclient + k8sutils.Kubeclient = &k8sutils.K8sClient{ + Clientset: fake.NewClientset([]runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Labels: map[string]string{"topology.kubernetes.io/zone": "zoneX"}, + }, + }, + }...), + } + getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { + return map[string]*array.PowerStoreArray{ + "Array1": { + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + HostConnectivity: &array.HostConnectivity{ + Local: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"nomatch"}, + }, + }, + }, + }, + }, + }, + IP: "10.198.0.1", + Client: clientMock, + }, + } + } + }, + want: []string{""}, + wantErr: true, + }, + { + name: "Host Registration Failure - For New HostConnectivity Secret - Label duplicated match", + s: &MockService{ + Service: &Service{ + opts: Opts{KubeNodeName: "node1"}, + }, + }, + args: args{ + ctx: context.TODO(), + initiators: []string{"initiator1", "initiator2"}, + }, + setup: func() { + // Override k8sclient + k8sutils.Kubeclient = &k8sutils.K8sClient{ + Clientset: fake.NewClientset([]runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Labels: map[string]string{"topology.kubernetes.io/zone": "zone1"}, + }, + }, + }...), + } + getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { + return map[string]*array.PowerStoreArray{ + "Array1": { + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + HostConnectivity: &array.HostConnectivity{ + Metro: array.MetroConnectivityOptions{ + ColocatedLocal: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"zone1"}, + }, + }, + }, + }, + }, + ColocatedBoth: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"zone1"}, + }, + }, + }, + }, + }, + }, + }, + IP: "10.198.0.1", + Client: clientMock, + }, + } + } + }, + want: []string{""}, + wantErr: true, + }, + { + name: "Host Registration Failure - For New HostConnectivity Secret - Metrotopology set", + s: &MockService{ + Service: &Service{ + opts: Opts{KubeNodeName: "node1"}, + }, + }, + args: args{ + ctx: context.TODO(), + initiators: []string{"initiator1", "initiator2"}, + }, + setup: func() { + // Override k8sclient + k8sutils.Kubeclient = &k8sutils.K8sClient{ + Clientset: fake.NewClientset([]runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Labels: map[string]string{"topology.kubernetes.io/zone": "zoneX"}, + }, + }, + }...), + } + getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { + return map[string]*array.PowerStoreArray{ + "Array1": { + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + HostConnectivity: &array.HostConnectivity{ + Local: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"nomatch"}, + }, + }, + }, + }, + }, + }, + IP: "10.198.0.1", + Client: clientMock, + }, + } + } + }, + want: []string{""}, + wantErr: true, + }, + { + name: "Failed to get node labels", + s: &MockService{ + Service: &Service{ + opts: Opts{KubeNodeName: "node1"}, + }, + }, + args: args{ + ctx: context.TODO(), + initiators: []string{"initiator1", "initiator2"}, + }, + setup: func() { + // Override the k8s client + k8sutils.Kubeclient = &k8sutils.K8sClient{ + Clientset: fake.NewClientset(), + } + getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { + return map[string]*array.PowerStoreArray{ + "Array1": { + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone2"}, + IP: "10.198.0.1", + Client: clientMock, + }, + "Array2": { + Endpoint: "https://10.198.0.2/api/rest", + GlobalID: "Array2", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone1"}, + IP: "10.198.0.2", + Client: clientMock, + }, + } + } + }, + want: []string{}, + wantErr: true, + }, + { + name: "Host Registration Failure: Both array more than one labels", + s: &MockService{ + Service: &Service{ + opts: Opts{KubeNodeName: "node1"}, + }, + }, + args: args{ + ctx: context.TODO(), + initiators: []string{"initiator1", "initiator2"}, + }, + setup: func() { + getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { + return map[string]*array.PowerStoreArray{ + "Array1": { + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone1", "topology.kubernetes.io/zone2": "zone2"}, + IP: "10.198.0.1", + Client: clientMock, + }, + "Array2": { + Endpoint: "https://10.198.0.2/api/rest", + GlobalID: "Array2", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone1", "topology.kubernetes.io/zone2": "zone2"}, + IP: "10.198.0.2", + Client: clientMock, + }, + } + } + }, + want: []string{}, + wantErr: true, + }, + { + name: "Host Registration Failure: One array has more than one labels", + s: &MockService{ + Service: &Service{ + opts: Opts{KubeNodeName: "node1"}, + }, + }, + args: args{ + ctx: context.TODO(), + initiators: []string{"initiator1", "initiator2"}, + }, + setup: func() { + // Override the k8s client + k8sutils.Kubeclient = &k8sutils.K8sClient{ + Clientset: fake.NewClientset([]runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone2"}, + }, + }, + }...), + } + getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { + return map[string]*array.PowerStoreArray{ + "Array1": { + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone1"}, + IP: "10.198.0.1", + Client: clientMock, + }, + "Array2": { + Endpoint: "https://10.198.0.2/api/rest", + GlobalID: "Array2", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone1", "topology.kubernetes.io/zone2": "zone2"}, + IP: "10.198.0.2", + Client: clientMock, + }, + } + } + }, + want: []string{}, + wantErr: true, + }, + { + name: "Failed: To get remote systems when both Array Label matches with Node Labels", + s: &MockService{ + Service: &Service{ + opts: Opts{KubeNodeName: "node1"}, + }, + }, + args: args{ + ctx: context.TODO(), + initiators: []string{"initiator1", "initiator2"}, + }, + setup: func() { + // Override the k8s client + k8sutils.Kubeclient = &k8sutils.K8sClient{ + Clientset: fake.NewClientset([]runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone1", "topology.kubernetes.io/zone2": "zone2"}, + }, + }, + }...), + } + getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { + return map[string]*array.PowerStoreArray{ + "Array1": { + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone1"}, + IP: "10.198.0.1", + Client: clientMock, + }, + "Array2": { + Endpoint: "https://10.198.0.2/api/rest", + GlobalID: "Array2", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone1"}, + IP: "10.198.0.2", + Client: clientMock, + }, + } + } + + getAllRemoteSystemsFunc = func(_ *array.PowerStoreArray, _ context.Context) ([]gopowerstore.RemoteSystem, error) { + return nil, fmt.Errorf("failed to get remote systems") + } + }, + want: []string{}, + wantErr: true, + }, + { + name: "Failed: To get remote systems when one Array Label matches with Node Labels", + s: &MockService{ + Service: &Service{ + opts: Opts{KubeNodeName: "node1"}, + }, + }, + args: args{ + ctx: context.TODO(), + initiators: []string{"initiator1", "initiator2"}, + }, + setup: func() { + getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { + return map[string]*array.PowerStoreArray{ + "Array1": { + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone2"}, + IP: "10.198.0.1", + Client: clientMock, + }, + "Array2": { + Endpoint: "https://10.198.0.2/api/rest", + GlobalID: "Array2", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone1"}, + IP: "10.198.0.2", + Client: clientMock, + }, + } + } + + getAllRemoteSystemsFunc = func(_ *array.PowerStoreArray, _ context.Context) ([]gopowerstore.RemoteSystem, error) { + return nil, fmt.Errorf("failed to get remote systems") + } + }, + want: []string{}, + wantErr: true, + }, + { + name: "Host Registration Failure - Array belongs to different zones", + s: &MockService{ + Service: &Service{ + opts: Opts{KubeNodeName: "node1"}, + }, + }, + args: args{ + ctx: context.TODO(), + initiators: []string{"initiator1", "initiator2"}, + }, + setup: func() { + // Override the k8s client + k8sutils.Kubeclient = &k8sutils.K8sClient{ + Clientset: fake.NewClientset([]runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone1", "topology.kubernetes.io/zone2": "zone2"}, + }, + }, + }...), + } + getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { + return map[string]*array.PowerStoreArray{ + "Array1": { + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone1"}, + IP: "10.198.0.1", + Client: clientMock, + }, + "Array2": { + Endpoint: "https://10.198.0.2/api/rest", + GlobalID: "Array2", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone2": "zone2"}, + IP: "10.198.0.2", + Client: clientMock, + }, + } + } + + getAllRemoteSystemsFunc = func(arr *array.PowerStoreArray, _ context.Context) ([]gopowerstore.RemoteSystem, error) { + if arr.GlobalID == "Array2" { + return []gopowerstore.RemoteSystem{ + { + ID: "arrayid1", + Name: "Pstore1", + Description: "", + SerialNumber: "Array1", + ManagementAddress: "10.198.0.1", + DataConnectionState: "OK", + Capabilities: []string{"Synchronous_Block_Replication"}, + }, + }, nil + } + return []gopowerstore.RemoteSystem{ + { + ID: "arrayid2", + Name: "Pstore2", + Description: "", + SerialNumber: "Array2", + ManagementAddress: "10.198.0.2", + DataConnectionState: "OK", + Capabilities: []string{"Synchronous_Block_Replication"}, + }, + }, nil + } + }, + want: []string{}, + wantErr: true, + }, + { + name: "Successful Host Registration with Co-Local and Co-remote", + s: &MockService{ + Service: &Service{ + opts: Opts{KubeNodeName: "node1"}, + }, + }, + args: args{ + ctx: context.TODO(), + initiators: []string{"initiator1", "initiator2"}, + }, + setup: func() { + // Override the k8s client + k8sutils.Kubeclient = &k8sutils.K8sClient{ + Clientset: fake.NewClientset([]runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Labels: map[string]string{"topology.kubernetes.io/zone2": "zone2"}, + }, + }, + }...), + } + getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { + return map[string]*array.PowerStoreArray{ + "Array1": { + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone2"}, + IP: "10.198.0.1", + Client: clientMock, + }, + "Array2": { + Endpoint: "https://10.198.0.2/api/rest", + GlobalID: "Array2", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone2": "zone2"}, + IP: "10.198.0.2", + Client: clientMock, + }, + } + } + + getAllRemoteSystemsFunc = func(arr *array.PowerStoreArray, _ context.Context) ([]gopowerstore.RemoteSystem, error) { + if arr.GlobalID == "Array2" { + return []gopowerstore.RemoteSystem{ + { + ID: "arrayid1", + Name: "Pstore1", + Description: "", + SerialNumber: "Array1", + ManagementAddress: "10.198.0.1", + DataConnectionState: "OK", + Capabilities: []string{"Synchronous_Block_Replication"}, + }, + }, nil + } + return []gopowerstore.RemoteSystem{ + { + ID: "arrayid2", + Name: "Pstore2", + Description: "", + SerialNumber: "Array2", + ManagementAddress: "10.198.0.2", + DataConnectionState: "OK", + Capabilities: []string{"Synchronous_Block_Replication"}, + }, + }, nil + } + }, + want: []string{"Array1", "Array2"}, + wantErr: false, + }, + { + name: "Host Registration Failure - Host Already registerd", + s: &MockService{ + Service: &Service{ + opts: Opts{KubeNodeName: "node1"}, + }, + }, + args: args{ + ctx: context.TODO(), + initiators: []string{"initiator1", "initiator2"}, + }, + setup: func() { + getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { + return map[string]*array.PowerStoreArray{ + "Array1": { + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone2"}, + IP: "10.198.0.1", + Client: clientMock, + }, + "Array2": { + Endpoint: "https://10.198.0.2/api/rest", + GlobalID: "Array2", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone1"}, + IP: "10.198.0.2", + Client: clientMock, + }, + } + } + + getAllRemoteSystemsFunc = func(_ *array.PowerStoreArray, _ context.Context) ([]gopowerstore.RemoteSystem, error) { + return nil, fmt.Errorf("failed to get remote systems") + } + + getIsHostAlreadyRegistered = func(_ *Service, _ context.Context, _ gopowerstore.Client, _ []string) bool { + return true + } + }, + want: []string{"Array1", "Array2"}, + wantErr: false, + }, + { + name: "Host Registration Failure - Host Already registerd with hostconnectivity", + s: &MockService{ + Service: &Service{ + opts: Opts{KubeNodeName: "node1"}, + }, + }, + args: args{ + ctx: context.TODO(), + initiators: []string{"initiator1", "initiator2"}, + }, + setup: func() { + getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { + return map[string]*array.PowerStoreArray{ + "Array1": { + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + HostConnectivity: &array.HostConnectivity{ + Local: k8score.NodeSelector{ + NodeSelectorTerms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"zone1"}, + }, + }, + }, + }, + }, + }, + IP: "10.198.0.1", + Client: clientMock, + }, + } + } + + getIsHostAlreadyRegistered = func(_ *Service, _ context.Context, _ gopowerstore.Client, _ []string) bool { + return true + } + }, + want: []string{"Array1"}, + wantErr: false, + }, + { + name: "Host Registration Failure - Create Host API fail", + s: &MockService{ + Service: &Service{ + opts: Opts{KubeNodeName: "node1"}, + }, + }, + args: args{ + ctx: context.TODO(), + initiators: []string{"initiator1", "initiator2"}, + }, + setup: func() { + getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { + return map[string]*array.PowerStoreArray{ + "Array1": { + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone2"}, + IP: "10.198.0.1", + Client: clientMock, + }, + "Array2": { + Endpoint: "https://10.198.0.2/api/rest", + GlobalID: "Array2", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone1"}, + IP: "10.198.0.2", + Client: clientMock, + }, + } + } + + getAllRemoteSystemsFunc = func(arr *array.PowerStoreArray, _ context.Context) ([]gopowerstore.RemoteSystem, error) { + if arr.GlobalID == "Array2" { + return []gopowerstore.RemoteSystem{ + { + ID: "arrayid1", + Name: "Pstore1", + Description: "", + SerialNumber: "Array1", + ManagementAddress: "10.198.0.1", + DataConnectionState: "OK", + Capabilities: []string{"Synchronous_Block_Replication"}, + }, + }, nil + } + return []gopowerstore.RemoteSystem{ + { + ID: "arrayid2", + Name: "Pstore2", + Description: "", + SerialNumber: "Array2", + ManagementAddress: "10.198.0.2", + DataConnectionState: "OK", + Capabilities: []string{"Synchronous_Block_Replication"}, + }, + }, nil + } + + getIsHostAlreadyRegistered = func(_ *Service, _ context.Context, _ gopowerstore.Client, _ []string) bool { + return false + } + + clientMock = new(gopowerstoremock.Client) + clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) + clientMock.On("CreateHost", mock.Anything, mock.Anything). + Return(gopowerstore.CreateResponse{}, fmt.Errorf("failed to create host")) + clientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(3.0), nil) + clientMock.On("GetCustomHTTPHeaders").Return(api.NewSafeHeader().GetHeader()) + }, + want: []string{}, + wantErr: true, + }, + { + name: "Host Registration Failure with Local only", + s: &MockService{ + Service: &Service{ + opts: Opts{KubeNodeName: "node1"}, + }, + }, + args: args{ + ctx: context.TODO(), + initiators: []string{"initiator1", "initiator2"}, + }, + setup: func() { + getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { + return map[string]*array.PowerStoreArray{ + "Array1": { + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + Client: clientMock, + }, + } + } + + getIsHostAlreadyRegistered = func(_ *Service, _ context.Context, _ gopowerstore.Client, _ []string) bool { + return false + } + + clientMock = new(gopowerstoremock.Client) + clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) + clientMock.On("CreateHost", mock.Anything, mock.Anything). + Return(gopowerstore.CreateResponse{}, fmt.Errorf("failed to create host")) + clientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(3.0), nil) + clientMock.On("GetCustomHTTPHeaders").Return(api.NewSafeHeader().GetHeader()) + }, + want: []string{}, + wantErr: true, + }, + { + name: "Host Registration Failure - getIsRemoteToOtherArray", + s: &MockService{ + Service: &Service{ + opts: Opts{KubeNodeName: "node1"}, + }, + }, + args: args{ + ctx: context.TODO(), + initiators: []string{"initiator1", "initiator2"}, + }, + setup: func() { + getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { + log.Info("InsideGetArray") + + return map[string]*array.PowerStoreArray{ + "Array1": { + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone2"}, + IP: "10.198.0.1", + Client: clientMock, + }, + "Array2": { + Endpoint: "https://10.198.0.2/api/rest", + GlobalID: "Array2", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone1"}, + IP: "10.198.0.2", + Client: clientMock, + }, + } + } + + getAllRemoteSystemsFunc = func(arr *array.PowerStoreArray, _ context.Context) ([]gopowerstore.RemoteSystem, error) { + if arr.GlobalID == "Array2" { + return []gopowerstore.RemoteSystem{ + { + ID: "arrayid1", + Name: "Pstore1", + Description: "", + SerialNumber: "Array1", + ManagementAddress: "10.198.0.1", + DataConnectionState: "OK", + Capabilities: []string{"Synchronous_Block_Replication"}, + }, + }, nil + } + + return []gopowerstore.RemoteSystem{ + { + ID: "arrayid2", + Name: "Pstore2", + Description: "", + SerialNumber: "Array2", + ManagementAddress: "10.198.0.2", + DataConnectionState: "OK", + Capabilities: []string{"Synchronous_Block_Replication"}, + }, + }, nil + } + + getIsHostAlreadyRegistered = func(_ *Service, _ context.Context, _ gopowerstore.Client, _ []string) bool { + return false + } + + clientMock = new(gopowerstoremock.Client) + clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) + clientMock.On("CreateHost", mock.Anything, mock.Anything). + Return(gopowerstore.CreateResponse{ID: validHostID}, nil) + clientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(3.0), nil) + clientMock.On("GetCustomHTTPHeaders").Return(api.NewSafeHeader().GetHeader()) + + getIsRemoteToOtherArray = func(_ *Service, _ context.Context, _, _ *array.PowerStoreArray) bool { + return false + } + }, + want: []string{"Array1", "Array2"}, + wantErr: false, + }, + { + name: "Host Registration Failure - Register Host fail", + s: &MockService{ + Service: &Service{ + opts: Opts{KubeNodeName: "node1"}, + }, + }, + args: args{ + ctx: context.TODO(), + initiators: []string{"initiator1", "initiator2"}, + }, + setup: func() { + getArrayfn = func(_ *Service) map[string]*array.PowerStoreArray { + return map[string]*array.PowerStoreArray{ + "Array1": { + Endpoint: "https://10.198.0.1/api/rest", + GlobalID: "Array1", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone2"}, + IP: "10.198.0.1", + Client: clientMock, + }, + "Array2": { + Endpoint: "https://10.198.0.2/api/rest", + GlobalID: "Array2", + Username: "admin", + Password: "Pass", + Insecure: true, + BlockProtocol: "auto", + MetroTopology: "Uniform", + Labels: map[string]string{"topology.kubernetes.io/zone1": "zone1"}, + IP: "10.198.0.2", + Client: clientMock, + }, + } + } + + getAllRemoteSystemsFunc = func(arr *array.PowerStoreArray, _ context.Context) ([]gopowerstore.RemoteSystem, error) { + if arr.GlobalID == "Array2" { + return []gopowerstore.RemoteSystem{ + { + ID: "arrayid1", + Name: "Pstore1", + Description: "", + SerialNumber: "Array1", + ManagementAddress: "10.198.0.1", + DataConnectionState: "OK", + Capabilities: []string{"Synchronous_Block_Replication"}, + }, + }, nil + } + return []gopowerstore.RemoteSystem{ + { + ID: "arrayid2", + Name: "Pstore2", + Description: "", + SerialNumber: "Array2", + ManagementAddress: "10.198.0.2", + DataConnectionState: "OK", + Capabilities: []string{"Synchronous_Block_Replication"}, + }, + }, nil + } + + getIsHostAlreadyRegistered = func(_ *Service, _ context.Context, _ gopowerstore.Client, _ []string) bool { + return false + } + + clientMock = new(gopowerstoremock.Client) + clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) + clientMock.On("CreateHost", mock.Anything, mock.Anything). + Return(gopowerstore.CreateResponse{}, fmt.Errorf("failed to create host")) + clientMock.On("GetSoftwareMajorMinorVersion", context.Background()).Return(float32(3.0), nil) + clientMock.On("GetCustomHTTPHeaders").Return(api.NewSafeHeader().GetHeader()) + + registerHostFunc = func(_ *Service, _ context.Context, _ gopowerstore.Client, _ string, _ []string, _ gopowerstore.HostConnectivityEnum) error { + return fmt.Errorf("failed to register Host") + } + }, + want: []string{}, + wantErr: true, + }, + // Add more test cases as needed + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + beforeEach() + defer afterEach() + tt.setup() + got, err := tt.s.createHost(tt.args.ctx, tt.args.initiators) + + if (err != nil) != tt.wantErr { + t.Errorf("Service.createHost() error = %v, wantErr %v", err, tt.wantErr) + return + } + + if len(tt.want) == 0 && got == "" { + // Special case: want is empty and got is empty + return + } + + found := false + for _, expected := range tt.want { + log.Infof("got %v expected %v", got, expected) + if got == expected { + found = true + break + } + } - }) + if !found { + t.Errorf("Service.createHost() = %v, want one of %v", got, tt.want) + } + }) + } } -func getNodeVolumeExpandValidRequest(volid string, isBlock bool) *csi.NodeExpandVolumeRequest { - var size int64 = controller.MaxVolumeSizeBytes / 100 - if !isBlock { - req := csi.NodeExpandVolumeRequest{ - VolumeId: volid, - VolumePath: validTargetPath, - CapacityRange: &csi.CapacityRange{ - RequiredBytes: size, - LimitBytes: controller.MaxVolumeSizeBytes, +func TestCheckIQNS(t *testing.T) { + tests := []struct { + name string + IQNs []string + host gopowerstore.Host + wantAdd []string + wantDelete []string + }{ + { + name: "IQNs to add and delete", + IQNs: []string{"iqn1", "iqn2", "iqn3"}, + host: gopowerstore.Host{ + Initiators: []gopowerstore.InitiatorInstance{ + {PortName: "iqn2"}, + {PortName: "iqn4"}, + }, + }, + wantAdd: []string{"iqn1", "iqn3"}, + wantDelete: []string{"iqn4"}, + }, + { + name: "No IQNs to add or delete", + IQNs: []string{"iqn1", "iqn2"}, + host: gopowerstore.Host{ + Initiators: []gopowerstore.InitiatorInstance{ + {PortName: "iqn1"}, + {PortName: "iqn2"}, + }, + }, + wantAdd: []string{}, + wantDelete: []string{}, + }, + { + name: "All IQNs to add", + IQNs: []string{"iqn1", "iqn2"}, + host: gopowerstore.Host{ + Initiators: []gopowerstore.InitiatorInstance{}, + }, + wantAdd: []string{"iqn1", "iqn2"}, + wantDelete: []string{}, + }, + { + name: "All IQNs to delete", + IQNs: []string{}, + host: gopowerstore.Host{ + Initiators: []gopowerstore.InitiatorInstance{ + {PortName: "iqn1"}, + {PortName: "iqn2"}, + }, }, + wantAdd: []string{}, + wantDelete: []string{"iqn1", "iqn2"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + log.Infof("Test: %s", tt.name) + gotAdd, gotDelete := checkIQNS(tt.IQNs, tt.host) + + if !elementsMatch(gotAdd, tt.wantAdd) { + t.Errorf("checkIQNS() = gotAdd %v, wantAdd %v", gotAdd, tt.wantAdd) + } + + if !elementsMatch(gotDelete, tt.wantDelete) { + t.Errorf("checkIQNS() = gotDelete %v, wantDelete %v", gotDelete, tt.wantDelete) + } + + if elementsMatch(gotAdd, tt.wantAdd) && elementsMatch(gotDelete, tt.wantDelete) { + log.Info("Success") + } + }) + } +} + +func elementsMatch(a, b []string) bool { + if len(a) != len(b) { + return false + } + m := make(map[string]int) + for _, v := range a { + m[v]++ + } + for _, v := range b { + if m[v] == 0 { + return false } - return &req + m[v]-- } - req := csi.NodeExpandVolumeRequest{ - VolumeId: volid, - VolumePath: validTargetPath + "/csi/volumeDevices/publish/", - CapacityRange: &csi.CapacityRange{ - RequiredBytes: size, - LimitBytes: controller.MaxVolumeSizeBytes, + return true +} + +func TestExtractPort(t *testing.T) { + tests := []struct { + name string + url string + expected string + expectError bool + }{ + { + name: "Valid URL with port", + url: "http://localhost:8080", + expected: "8080", + expectError: false, + }, + { + name: "Valid URL without port", + url: "http://localhost", + expected: "", + expectError: true, + }, + { + name: "Invalid URL format", + url: "://bad-url", + expected: "", + expectError: true, + }, + { + name: "HTTPS URL with port", + url: "https://example.com:443", + expected: "443", + expectError: false, + }, + { + name: "URL with path and port", + url: "http://example.com:9000/path", + expected: "9000", + expectError: false, + }, + { + name: "Empty string input", + url: "", + expected: "", + expectError: true, }, } - return &req + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + port, err := ExtractPort(tt.url) + + if tt.expectError { + if err == nil { + t.Errorf("Expected error, got none. Port: %s", port) + } + } else { + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + if port != tt.expected { + t.Errorf("Expected port: %s, got: %s", tt.expected, port) + } + } + }) + } +} + +func TestService_updateHost(t *testing.T) { + tests := []struct { + name string // description of this test case + // Named input parameters for target function. + initiators []string + client gopowerstore.Client + host gopowerstore.Host + arrayID string + connectivity *gopowerstore.HostConnectivityEnum + wantErr bool + }{ + { + name: "Update host", + initiators: []string{}, + client: nil, + host: gopowerstore.Host{}, + arrayID: "f16c:f7ec:cfa2:e1c5:9a3c:cb08:801f:36b8", + connectivity: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // TODO: construct the receiver type. + var s Service + gotErr := s.updateHost(context.Background(), tt.initiators, tt.client, tt.host, tt.arrayID, tt.connectivity) + if gotErr != nil { + if !tt.wantErr { + t.Errorf("updateHost() failed: %v", gotErr) + } + return + } + if tt.wantErr { + t.Fatal("updateHost() succeeded unexpectedly") + } + }) + } +} + +func TestMetroMatchNodeSelectorTerms(t *testing.T) { + tests := []struct { + name string + terms []k8score.NodeSelectorTerm + nodeLabels map[string]string + wantMatch bool + wantLabels map[string]string + }{ + { + name: "Match with NodeSelectorOpIn", + terms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"us-east-1a", "us-east-1b"}, + }, + }, + }, + }, + nodeLabels: map[string]string{"zone": "us-east-1a"}, + wantMatch: true, + wantLabels: map[string]string{"zone": "us-east-1a"}, + }, + { + name: "Mismatch with NodeSelectorOpIn", + terms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "zone", + Operator: k8score.NodeSelectorOpIn, + Values: []string{"us-west-1a"}, + }, + }, + }, + }, + nodeLabels: map[string]string{"zone": "us-east-1a"}, + wantMatch: false, + wantLabels: nil, + }, + { + name: "Match with NodeSelectorOpExists", + terms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "diskType", + Operator: k8score.NodeSelectorOpExists, + }, + }, + }, + }, + nodeLabels: map[string]string{"diskType": "ssd"}, + wantMatch: true, + wantLabels: map[string]string{"diskType": "ssd"}, + }, + { + name: "Mismatch with NodeSelectorOpDoesNotExist", + terms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "gpu", + Operator: k8score.NodeSelectorOpDoesNotExist, + }, + }, + }, + }, + nodeLabels: map[string]string{"gpu": "nvidia"}, + wantMatch: false, + wantLabels: nil, + }, + { + name: "Match with NodeSelectorOpNotIn", + terms: []k8score.NodeSelectorTerm{ + { + MatchExpressions: []k8score.NodeSelectorRequirement{ + { + Key: "env", + Operator: k8score.NodeSelectorOpNotIn, + Values: []string{"prod"}, + }, + }, + }, + }, + nodeLabels: map[string]string{"env": "dev"}, + wantMatch: true, + wantLabels: map[string]string{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotMatch, gotLabels := metroMatchNodeSelectorTerms(tt.terms, tt.nodeLabels) + assert.Equal(t, tt.wantMatch, gotMatch) + assert.Equal(t, tt.wantLabels, gotLabels) + }) + } +} + +func TestService_setupHost(t *testing.T) { + tests := []struct { + name string + // Named input parameters for target function. + initiators []string + client gopowerstore.Client + arrayIP string + arrayID string + wantErr bool + }{ + { + name: "Setup host", + initiators: []string{}, + client: nil, + arrayIP: "f16c:f7ec:cfa2:e1c5:9a3c:cb08:801f:36b8", + arrayID: "f16c:f7ec:cfa2:e1c5:9a3c:cb08:801f:36b8", + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var s Service + gotErr := s.setupHost(tt.initiators, tt.client, tt.arrayIP, tt.arrayID) + if gotErr != nil { + if !tt.wantErr { + t.Errorf("setupHost() failed: %v", gotErr) + } + return + } + if tt.wantErr { + t.Fatal("setupHost() succeeded unexpectedly") + } + }) + } +} + +func TestIsHostAlreadyRegistered(t *testing.T) { + tests := []struct { + name string + // Named input parameters for target function. + initiators []string + client gopowerstore.Client + before func(*gopowerstoremock.Client) + wantResult bool + }{ + { + name: "IsHostAlreadyRegistered - true", + initiators: []string{"my-port"}, + client: new(gopowerstoremock.Client), + before: func(client *gopowerstoremock.Client) { + client.On("GetHosts", mock.Anything).Return( + []gopowerstore.Host{{ + ID: "host-id", + Initiators: []gopowerstore.InitiatorInstance{{ + PortName: "my-port", + PortType: gopowerstore.InitiatorProtocolTypeEnumISCSI, + }}, + Name: "host-name", + }}, nil) + }, + wantResult: true, + }, + { + name: "IsHostAlreadyRegistered - not found", + initiators: []string{}, + client: new(gopowerstoremock.Client), + before: func(client *gopowerstoremock.Client) { + client.On("GetHosts", mock.Anything).Return( + []gopowerstore.Host{{ + ID: "host-id", + Initiators: []gopowerstore.InitiatorInstance{{ + PortName: "my-port", + PortType: gopowerstore.InitiatorProtocolTypeEnumISCSI, + }}, + Name: "host-name", + }}, nil) + }, + wantResult: false, + }, + { + name: "IsHostAlreadyRegistered - unable to get hosts", + initiators: []string{}, + client: new(gopowerstoremock.Client), + before: func(client *gopowerstoremock.Client) { + client.On("GetHosts", mock.Anything).Return( + []gopowerstore.Host{}, fmt.Errorf("unable to get hosts")) + }, + wantResult: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var s Service + if tt.before != nil { + tt.before(tt.client.(*gopowerstoremock.Client)) + } + gotResult := s.isHostAlreadyRegistered(context.Background(), tt.client, tt.initiators) + if gotResult != tt.wantResult { + t.Errorf("isHostAlreadyRegistered() = %v, want %v", gotResult, tt.wantResult) + } + }) + } +} + +func TestRemoveRemnantMounts(t *testing.T) { + t.Run("fails to get remnant target mounts", func(t *testing.T) { + mockFs := new(mocks.FsInterface) + mockFs.On("ReadFile", mock.Anything).Return(nil, fmt.Errorf("error")) + + _, err := removeRemnantMounts(context.Background(), "/var/lib/test", mockFs, csmlog.Fields{}) + if err == nil { + t.Errorf("expected an error, got nil") + } + }) + + t.Run("no remnant target mounts", func(t *testing.T) { + mockFs := new(mocks.FsInterface) + mockFs.On("ReadFile", mock.Anything).Return([]byte("data"), nil) + mockFs.On("ParseProcMounts", mock.Anything, mock.Anything).Return([]gofsutil.Info{ + { + Path: "/var/lib/other", + }, + }, nil) + _, err := removeRemnantMounts(context.Background(), "/var/lib/test", mockFs, csmlog.Fields{}) + if err == nil { + t.Errorf("expected an error, got nil") + } + }) +} + +func TestCountActiveSessionsInitiators(t *testing.T) { + tests := []struct { + name string + host gopowerstore.Host + wantCount int + }{ + { + name: "single initiator with one ActiveSessions", + host: gopowerstore.Host{ + Initiators: []gopowerstore.InitiatorInstance{ + {ActiveSessions: []gopowerstore.ActiveSessionInstance{{ApplianceID: "s1"}}}, + {ActiveSessions: nil}, + {ActiveSessions: []gopowerstore.ActiveSessionInstance{}}, + }, + }, + wantCount: 1, + }, + { + name: "single initiator with two ActiveSessions", + host: gopowerstore.Host{ + Initiators: []gopowerstore.InitiatorInstance{ + {ActiveSessions: []gopowerstore.ActiveSessionInstance{{ApplianceID: "s1"}}}, + {ActiveSessions: nil}, + {ActiveSessions: []gopowerstore.ActiveSessionInstance{{ApplianceID: "s2"}}}, + }, + }, + wantCount: 2, + }, + { + name: "single initiator with no ActiveSessions", + host: gopowerstore.Host{ + Initiators: []gopowerstore.InitiatorInstance{ + {ActiveSessions: nil}, + {ActiveSessions: nil}, + {ActiveSessions: []gopowerstore.ActiveSessionInstance{}}, + }, + }, + wantCount: 0, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + numOfInitiatorsWithActiveSession := countActiveSessionsInitiators(tc.host) + if numOfInitiatorsWithActiveSession != tc.wantCount { + t.Errorf("countActiveSessionsInitiators() = %d, want %d", numOfInitiatorsWithActiveSession, tc.wantCount) + } + }) + } } diff --git a/pkg/node/publisher.go b/pkg/node/publisher.go index 0fa6b9fb..966cbebe 100644 --- a/pkg/node/publisher.go +++ b/pkg/node/publisher.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2021-2023 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2021-2025 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,17 +21,18 @@ package node import ( "context" + "github.com/dell/csi-powerstore/v2/pkg/identifiers" + "github.com/dell/csi-powerstore/v2/pkg/identifiers/fs" + "github.com/dell/csmlog" "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/dell/csi-powerstore/v2/pkg/common/fs" - log "github.com/sirupsen/logrus" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) // VolumePublisher allows to node publish a volume type VolumePublisher interface { - Publish(ctx context.Context, logFields log.Fields, fs fs.Interface, - cap *csi.VolumeCapability, isRO bool, targetPath string, stagingPath string) (*csi.NodePublishVolumeResponse, error) + Publish(ctx context.Context, logFields csmlog.Fields, fs fs.Interface, + vc *csi.VolumeCapability, isRO bool, targetPath string, stagingPath string) (*csi.NodePublishVolumeResponse, error) } // SCSIPublisher implementation of NodeVolumePublisher for SCSI based (FC, iSCSI) volumes @@ -40,7 +41,7 @@ type SCSIPublisher struct { } // Publish publishes volume as either raw block or mount by mounting it to the target path -func (sp *SCSIPublisher) Publish(ctx context.Context, logFields log.Fields, fs fs.Interface, cap *csi.VolumeCapability, isRO bool, targetPath string, stagingPath string) (*csi.NodePublishVolumeResponse, error) { +func (sp *SCSIPublisher) Publish(ctx context.Context, logFields csmlog.Fields, fs fs.Interface, vc *csi.VolumeCapability, isRO bool, targetPath string, stagingPath string) (*csi.NodePublishVolumeResponse, error) { published, err := isAlreadyPublished(ctx, targetPath, getRWModeString(isRO), fs) if err != nil { return nil, err @@ -51,13 +52,14 @@ func (sp *SCSIPublisher) Publish(ctx context.Context, logFields log.Fields, fs f } if sp.isBlock { - return sp.publishBlock(ctx, logFields, fs, cap, isRO, targetPath, stagingPath) + return sp.publishBlock(ctx, logFields, fs, vc, isRO, targetPath, stagingPath) } - return sp.publishMount(ctx, logFields, fs, cap, isRO, targetPath, stagingPath) + return sp.publishMount(ctx, logFields, fs, vc, isRO, targetPath, stagingPath) } -func (sp *SCSIPublisher) publishBlock(ctx context.Context, logFields log.Fields, fs fs.Interface, cap *csi.VolumeCapability, isRO bool, targetPath string, stagingPath string) (*csi.NodePublishVolumeResponse, error) { - log.WithFields(logFields).Info("start publishing as block device") +func (sp *SCSIPublisher) publishBlock(ctx context.Context, logFields csmlog.Fields, fs fs.Interface, _ *csi.VolumeCapability, isRO bool, targetPath string, stagingPath string) (*csi.NodePublishVolumeResponse, error) { + log := log.WithFields(logFields).WithContext(ctx) + log.Info("start publishing as block device") if isRO { return nil, status.Error(codes.InvalidArgument, "read only not supported for Block Volume") @@ -67,32 +69,32 @@ func (sp *SCSIPublisher) publishBlock(ctx context.Context, logFields log.Fields, return nil, status.Errorf(codes.Internal, "can't create target file %s: %s", targetPath, err.Error()) } - log.WithFields(logFields).Info("target path successfully created") + log.Info("target path successfully created") if err := fs.GetUtil().BindMount(ctx, stagingPath, targetPath); err != nil { return nil, status.Errorf(codes.Internal, "error bind disk %s to target path: %s", stagingPath, err.Error()) } - log.WithFields(logFields).Info("volume successfully binded") + log.Info("volume successfully binded") return &csi.NodePublishVolumeResponse{}, nil } -func (sp *SCSIPublisher) publishMount(ctx context.Context, logFields log.Fields, fs fs.Interface, cap *csi.VolumeCapability, isRO bool, targetPath string, stagingPath string) (*csi.NodePublishVolumeResponse, error) { - if cap.GetAccessMode().GetMode() == csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER { - // MULTI_WRITER not supported for mount volumes - return nil, status.Error(codes.Unimplemented, "Mount volumes do not support AccessMode MULTI_NODE_MULTI_WRITER") +func (sp *SCSIPublisher) publishMount(ctx context.Context, logFields csmlog.Fields, fs fs.Interface, vc *csi.VolumeCapability, isRO bool, targetPath string, stagingPath string) (*csi.NodePublishVolumeResponse, error) { + log := log.WithFields(logFields).WithContext(ctx) + if vc.GetAccessMode().GetMode() == csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER { + log.Infof(" Mount volume with the AccessMode ReadWriteMany") } - if cap.GetAccessMode().GetMode() == csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY { + if vc.GetAccessMode().GetMode() == csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY { // Warning in case of MULTI_NODE_READER_ONLY for mount volumes - log.Warningf("Mount volume with the AccessMode ReadOnlyMany") + log.Warnf("Mount volume with the AccessMode ReadOnlyMany") } var opts []string - mountCap := cap.GetMount() + mountCap := vc.GetMount() mountFsType := mountCap.GetFsType() - mntFlags := mountCap.GetMountFlags() + mntFlags := identifiers.GetMountFlags(vc) if mountFsType == "xfs" { mntFlags = append(mntFlags, "nouuid") } @@ -100,12 +102,12 @@ func (sp *SCSIPublisher) publishMount(ctx context.Context, logFields log.Fields, if targetFS == "xfs" { opts = []string{"-m", "crc=0,finobt=0"} } - if err := fs.MkdirAll(targetPath, 0750); err != nil { + if err := fs.MkdirAll(targetPath, 0o750); err != nil { return nil, status.Errorf(codes.Internal, "can't create target dir with Mkdirall %s: %s", targetPath, err.Error()) } - log.WithFields(logFields).Info("target dir successfully created") + log.Info("target dir successfully created") curFS, err := fs.GetUtil().GetDiskFormat(ctx, stagingPath) if err != nil { @@ -120,7 +122,7 @@ func (sp *SCSIPublisher) publishMount(ctx context.Context, logFields log.Fields, } if curFS == "" { - log.WithFields(logFields).Infof("no filesystem found on staged disk %s", stagingPath) + log.Infof("no filesystem found on staged disk : %s", stagingPath) if isRO { return nil, status.Errorf(codes.FailedPrecondition, "RO mount required but no fs detected on staged volume %s", stagingPath) @@ -130,7 +132,7 @@ func (sp *SCSIPublisher) publishMount(ctx context.Context, logFields log.Fields, return nil, status.Errorf(codes.Internal, "can't format staged device %s: %s", stagingPath, err.Error()) } - log.WithFields(logFields).Infof("staged disk %s successfully formatted to %s", stagingPath, targetFS) + log.Infof("staged disk %s successfully formatted to %s", stagingPath, targetFS) } if isRO { mntFlags = append(mntFlags, "ro") @@ -140,18 +142,19 @@ func (sp *SCSIPublisher) publishMount(ctx context.Context, logFields log.Fields, return nil, status.Errorf(codes.Internal, "error performing mount for staging path %s: %s", stagingPath, err.Error()) } - log.WithFields(logFields).Info("volume successfully mounted") + log.Info("volume successfully mounted") return &csi.NodePublishVolumeResponse{}, nil } // NFSPublisher implementation of NodeVolumePublisher for NFS volumes -type NFSPublisher struct { -} +type NFSPublisher struct{} // Publish publishes nfs volume by mounting it to the target path -func (np *NFSPublisher) Publish(ctx context.Context, logFields log.Fields, fs fs.Interface, - cap *csi.VolumeCapability, isRO bool, targetPath string, stagingPath string) (*csi.NodePublishVolumeResponse, error) { +func (np *NFSPublisher) Publish(ctx context.Context, logFields csmlog.Fields, fs fs.Interface, + vc *csi.VolumeCapability, isRO bool, targetPath string, stagingPath string, +) (*csi.NodePublishVolumeResponse, error) { + log := log.WithFields(logFields).WithContext(ctx) published, err := isAlreadyPublished(ctx, targetPath, getRWModeString(isRO), fs) if err != nil { return nil, err @@ -161,14 +164,13 @@ func (np *NFSPublisher) Publish(ctx context.Context, logFields log.Fields, fs fs return &csi.NodePublishVolumeResponse{}, nil } - if err := fs.MkdirAll(targetPath, 0750); err != nil { + if err := fs.MkdirAll(targetPath, 0o750); err != nil { return nil, status.Errorf(codes.Internal, "can't create target folder %s: %s", stagingPath, err.Error()) } - log.WithFields(logFields).Info("target path successfully created") + log.Info("target path successfully created") - mountCap := cap.GetMount() - mntFlags := mountCap.GetMountFlags() + mntFlags := identifiers.GetMountFlags(vc) if isRO { mntFlags = append(mntFlags, "ro") @@ -179,6 +181,6 @@ func (np *NFSPublisher) Publish(ctx context.Context, logFields log.Fields, fs fs "error bind disk %s to target path: %s", stagingPath, err.Error()) } - log.WithFields(logFields).Info("volume successfully binded") + log.Info("volume successfully binded") return &csi.NodePublishVolumeResponse{}, nil } diff --git a/pkg/node/stager.go b/pkg/node/stager.go index 80200c7f..01d824fa 100644 --- a/pkg/node/stager.go +++ b/pkg/node/stager.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2021-2023 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2021-2025 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ package node import ( "context" + "errors" "fmt" "os" "path" @@ -28,13 +29,13 @@ import ( "strings" "time" - "github.com/container-storage-interface/spec/lib/go/csi" "github.com/dell/csi-powerstore/v2/pkg/array" - "github.com/dell/csi-powerstore/v2/pkg/common" - "github.com/dell/csi-powerstore/v2/pkg/common/fs" + "github.com/dell/csi-powerstore/v2/pkg/identifiers" + "github.com/dell/csi-powerstore/v2/pkg/identifiers/fs" + "github.com/dell/csmlog" "github.com/dell/gobrick" "github.com/dell/gopowerstore" - log "github.com/sirupsen/logrus" + "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -46,11 +47,11 @@ const ( // VolumeStager allows to node stage a volume type VolumeStager interface { - Stage(ctx context.Context, req *csi.NodeStageVolumeRequest, logFields log.Fields, fs fs.Interface, id string) (*csi.NodeStageVolumeResponse, error) + Stage(ctx context.Context, req *csi.NodeStageVolumeRequest, stagingPath string, nodeID string, logFields csmlog.Fields, fs fs.Interface, id string, isRemote bool, client gopowerstore.Client) (*csi.NodeStageVolumeResponse, error) } // ReachableEndPoint checks if the endpoint is reachable or not -var ReachableEndPoint = common.ReachableEndPoint +var ReachableEndPoint = identifiers.ReachableEndPoint // SCSIStager implementation of NodeVolumeStager for SCSI based (FC, iSCSI) volumes type SCSIStager struct { @@ -62,12 +63,48 @@ type SCSIStager struct { } // Stage stages volume by connecting it through either FC or iSCSI and creating bind mount to staging path -func (s *SCSIStager) Stage(ctx context.Context, req *csi.NodeStageVolumeRequest, - logFields log.Fields, fs fs.Interface, id string) (*csi.NodeStageVolumeResponse, error) { - // append additional path to be able to do bind mounts - stagingPath := getStagingPath(ctx, req.GetStagingTargetPath(), id) +func (s *SCSIStager) Stage(ctx context.Context, req *csi.NodeStageVolumeRequest, stagingPath string, nodeID string, + logFields csmlog.Fields, fs fs.Interface, id string, isRemote bool, client gopowerstore.Client, +) (*csi.NodeStageVolumeResponse, error) { + log := log.WithContext(ctx) + orginalContext := req.PublishContext + volume, err := client.GetVolume(ctx, id) + if err != nil { + return nil, err + } + targetMap := make(map[string]string) + err = s.AddTargetsInfoToMap(targetMap, volume.ApplianceID, client, isRemote) + if err != nil { + return nil, err + } - publishContext, err := readSCSIInfoFromPublishContext(req.PublishContext, s.useFC, s.useNVME) + if !isRemote { + wwn, ok := orginalContext[identifiers.TargetMapDeviceWWN] + lun, ok := orginalContext[identifiers.TargetMapLUNAddress] + if !ok { + wwn = strings.TrimPrefix(volume.Wwn, identifiers.WWNPrefix) + lun, err = getLunAddressFromArray(ctx, client, id, nodeID) + if err != nil { + return nil, err + } + } + targetMap[identifiers.TargetMapDeviceWWN] = wwn + targetMap[identifiers.TargetMapLUNAddress] = lun + } else { + wwn, ok := orginalContext[identifiers.TargetMapRemoteDeviceWWN] + lun, ok := orginalContext[identifiers.TargetMapRemoteLUNAddress] + if !ok { + wwn = strings.TrimPrefix(volume.Wwn, identifiers.WWNPrefix) + lun, err = getLunAddressFromArray(ctx, client, id, nodeID) + if err != nil { + return nil, err + } + } + targetMap[identifiers.TargetMapRemoteDeviceWWN] = wwn + targetMap[identifiers.TargetMapRemoteLUNAddress] = lun + } + + publishContext, err := readSCSIInfoFromPublishContext(targetMap, s.useFC, s.useNVME, isRemote) if err != nil { return nil, err } @@ -85,7 +122,7 @@ func (s *SCSIStager) Stage(ctx context.Context, req *csi.NodeStageVolumeRequest, logFields["WWN"] = publishContext.deviceWWN logFields["Lun"] = publishContext.volumeLUNAddress logFields["StagingPath"] = stagingPath - ctx = common.SetLogFields(ctx, logFields) + ctx = csmlog.SetLogFields(ctx, logFields) found, ready, err := isReadyToPublish(ctx, stagingPath, fs) if err != nil { @@ -95,8 +132,7 @@ func (s *SCSIStager) Stage(ctx context.Context, req *csi.NodeStageVolumeRequest, log.WithFields(logFields).Info("device already staged") return &csi.NodeStageVolumeResponse{}, nil } else if found { - log.WithFields(logFields).Warning("volume found in staging path but it is not ready for publish," + - "try to unmount it and retry staging again") + log.WithFields(logFields).Warn("volume found in staging path but it is not ready for publish, try to unmount it and retry staging again") _, err := unstageVolume(ctx, stagingPath, id, logFields, err, fs) if err != nil { return nil, status.Errorf(codes.Internal, "failed to unmount volume: %s", err.Error()) @@ -117,7 +153,8 @@ func (s *SCSIStager) Stage(ctx context.Context, req *csi.NodeStageVolumeRequest, } log.WithFields(logFields).Info("target path successfully created") - if err := fs.GetUtil().BindMount(ctx, devicePath, stagingPath); err != nil { + mntFlags := identifiers.GetMountFlags(req.GetVolumeCapability()) + if err := fs.GetUtil().BindMount(ctx, devicePath, stagingPath, mntFlags...); err != nil { return nil, status.Errorf(codes.Internal, "error bind disk %s to target path: %s", devicePath, err.Error()) } @@ -126,25 +163,47 @@ func (s *SCSIStager) Stage(ctx context.Context, req *csi.NodeStageVolumeRequest, return &csi.NodeStageVolumeResponse{}, nil } +func getLunAddressFromArray(ctx context.Context, client gopowerstore.Client, id string, nodeID string) (string, error) { + log := log.WithContext(ctx) + log.Infof("GetHostVolumeMappingByVolumeID for volId %s host %s", id, nodeID) + var node gopowerstore.Host + node, err := client.GetHostByName(ctx, nodeID) + if err != nil { + return "", status.Errorf(codes.Internal, + "failed to find host '%s' during staging: %s", nodeID, err.Error()) + } + mapping, err := client.GetHostVolumeMappingByVolumeID(ctx, id) + if err != nil { + return "", status.Errorf(codes.Internal, + "failed to get mapping for volume with ID '%s' during staging: %s", id, err.Error()) + } + for _, m := range mapping { + if m.HostID == node.ID { + lun := strconv.FormatInt(m.LogicalUnitNumber, 10) + return lun, nil + } + } + return "", status.Errorf(codes.Internal, + "failed to get LUN for volume with ID '%s' during staging", id) +} + // NFSStager implementation of NodeVolumeStager for NFS volumes type NFSStager struct { array *array.PowerStoreArray } // Stage stages volume by mounting volumes as nfs to the staging path -func (n *NFSStager) Stage(ctx context.Context, req *csi.NodeStageVolumeRequest, - logFields log.Fields, fs fs.Interface, id string) (*csi.NodeStageVolumeResponse, error) { - // append additional path to be able to do bind mounts - stagingPath := getStagingPath(ctx, req.GetStagingTargetPath(), id) - - hostIP := req.PublishContext[common.KeyHostIP] - exportID := req.PublishContext[common.KeyExportID] - nfsExport := req.PublishContext[common.KeyNfsExportPath] - allowRoot := req.PublishContext[common.KeyAllowRoot] - nasName := req.PublishContext[common.KeyNasName] +func (n *NFSStager) Stage(ctx context.Context, req *csi.NodeStageVolumeRequest, stagingPath string, _ string, + logFields csmlog.Fields, fs fs.Interface, id string, _ bool, _ gopowerstore.Client, +) (*csi.NodeStageVolumeResponse, error) { + hostIP := req.PublishContext[identifiers.KeyHostIP] + exportID := req.PublishContext[identifiers.KeyExportID] + nfsExport := req.PublishContext[identifiers.KeyNfsExportPath] + allowRoot := req.PublishContext[identifiers.KeyAllowRoot] + nasName := req.PublishContext[identifiers.KeyNasName] natIP := "" - if ip, ok := req.PublishContext[common.KeyNatIP]; ok { + if ip, ok := req.PublishContext[identifiers.KeyNatIP]; ok { natIP = ip } @@ -155,9 +214,9 @@ func (n *NFSStager) Stage(ctx context.Context, req *csi.NodeStageVolumeRequest, logFields["ExportID"] = exportID logFields["HostIP"] = hostIP logFields["NatIP"] = natIP - logFields["NFSv4ACLs"] = req.PublishContext[common.KeyNfsACL] + logFields["NFSv4ACLs"] = req.PublishContext[identifiers.KeyNfsACL] logFields["NasName"] = nasName - ctx = common.SetLogFields(ctx, logFields) + log := log.WithContext(ctx).WithFields(logFields) found, err := isReadyToPublishNFS(ctx, stagingPath, fs) if err != nil { @@ -165,37 +224,38 @@ func (n *NFSStager) Stage(ctx context.Context, req *csi.NodeStageVolumeRequest, } if found { - log.WithFields(logFields).Info("device already staged") + log.Info("device already staged") return &csi.NodeStageVolumeResponse{}, nil } - if err := fs.MkdirAll(stagingPath, 0750); err != nil { + if err := fs.MkdirAll(stagingPath, 0o750); err != nil { return nil, status.Errorf(codes.Internal, "can't create target folder %s: %s", stagingPath, err.Error()) } - log.WithFields(logFields).Info("stage path successfully created") + log.Info("stage path successfully created") - if err := fs.GetUtil().Mount(ctx, nfsExport, stagingPath, ""); err != nil { + mntFlags := identifiers.GetMountFlags(req.GetVolumeCapability()) + if err := fs.GetUtil().Mount(ctx, nfsExport, stagingPath, "", mntFlags...); err != nil { return nil, status.Errorf(codes.Internal, "error mount nfs share %s to target path: %s", nfsExport, err.Error()) } // Create folder with 1777 in nfs share so every user can use it - if err := fs.MkdirAll(filepath.Join(stagingPath, commonNfsVolumeFolder), 0750); err != nil { + if err := fs.MkdirAll(filepath.Join(stagingPath, commonNfsVolumeFolder), 0o750); err != nil { return nil, status.Errorf(codes.Internal, "can't create common folder %s: %s", filepath.Join(stagingPath, "volume"), err.Error()) } mode := os.ModePerm - acls := req.PublishContext[common.KeyNfsACL] + acls := req.PublishContext[identifiers.KeyNfsACL] aclsConfigured := false if acls != "" { if posixMode(acls) { - perm, err := strconv.ParseUint(acls, 8, 64) + perm, err := strconv.ParseUint(acls, 8, 32) if err == nil { - mode = os.FileMode(perm) + mode = os.FileMode(perm) // #nosec: G115 false positive } else { - log.WithFields(logFields).Warn("can't parse file mode, invalid mode specified. Default mode permissions will be set.") + log.Warn("can't parse file mode, invalid mode specified. Default mode permissions will be set.") } } else { aclsConfigured, err = validateAndSetACLs(ctx, &NFSv4ACLs{}, nasName, n.array.GetClient(), acls, filepath.Join(stagingPath, commonNfsVolumeFolder)) @@ -213,12 +273,14 @@ func (n *NFSStager) Stage(ctx context.Context, req *csi.NodeStageVolumeRequest, } if allowRoot == "false" { - log.WithFields(logFields).Info("removing allow root from nfs export") + log.Info("removing allow root from nfs export") var hostsToRemove []string var hostsToAdd []string - hostsToRemove = append(hostsToRemove, hostIP+"/255.255.255.255") - hostsToAdd = append(hostsToAdd, hostIP) + if hostIP != "" { + hostsToRemove = append(hostsToRemove, hostIP+"/255.255.255.255") + hostsToAdd = append(hostsToAdd, hostIP) + } if natIP != "" { hostsToRemove = append(hostsToRemove, natIP) @@ -237,7 +299,7 @@ func (n *NFSStager) Stage(ctx context.Context, req *csi.NodeStageVolumeRequest, } } - log.WithFields(logFields).Info("nfs share successfully mounted") + log.Info("nfs share successfully mounted") return &csi.NodeStageVolumeResponse{}, nil } @@ -250,47 +312,62 @@ type scsiPublishContextData struct { fcTargets []gobrick.FCTargetInfo } -func readSCSIInfoFromPublishContext(publishContext map[string]string, useFC bool, useNVMe bool) (scsiPublishContextData, error) { +func readSCSIInfoFromPublishContext(publishContext map[string]string, useFC bool, useNVMe bool, isRemote bool) (scsiPublishContextData, error) { // Get publishContext var data scsiPublishContextData - deviceWWN, ok := publishContext[common.PublishContextDeviceWWN] - if !ok { + deviceWwnKey := identifiers.TargetMapDeviceWWN + lunAddressKey := identifiers.TargetMapLUNAddress + if isRemote { + deviceWwnKey = identifiers.TargetMapRemoteDeviceWWN + lunAddressKey = identifiers.TargetMapRemoteLUNAddress + } + + deviceWWN, ok := publishContext[deviceWwnKey] + if !ok || deviceWWN == "" { return data, status.Error(codes.InvalidArgument, "deviceWWN must be in publish context") } - volumeLUNAddress, ok := publishContext[common.PublishContextLUNAddress] - if !ok { + volumeLUNAddress, ok := publishContext[lunAddressKey] + if !ok || volumeLUNAddress == "" { return data, status.Error(codes.InvalidArgument, "volumeLUNAddress must be in publish context") } - iscsiTargets := readISCSITargetsFromPublishContext(publishContext) + iscsiTargets := readISCSITargetsFromPublishContext(publishContext, isRemote) if len(iscsiTargets) == 0 && !useFC && !useNVMe { return data, status.Error(codes.InvalidArgument, "iscsiTargets data must be in publish context") } - nvmeTCPTargets := readNVMETCPTargetsFromPublishContext(publishContext) + nvmeTCPTargets := readNVMETCPTargetsFromPublishContext(publishContext, isRemote) if len(nvmeTCPTargets) == 0 && useNVMe && !useFC { return data, status.Error(codes.InvalidArgument, "NVMeTCP Targets data must be in publish context") } - nvmeFCTargets := readNVMEFCTargetsFromPublishContext(publishContext) + nvmeFCTargets := readNVMEFCTargetsFromPublishContext(publishContext, isRemote) if len(nvmeFCTargets) == 0 && useNVMe && useFC { return data, status.Error(codes.InvalidArgument, "NVMeFC Targets data must be in publish context") } - fcTargets := readFCTargetsFromPublishContext(publishContext) + fcTargets := readFCTargetsFromPublishContext(publishContext, isRemote) if len(fcTargets) == 0 && useFC && !useNVMe { return data, status.Error(codes.InvalidArgument, "fcTargets data must be in publish context") } - return scsiPublishContextData{deviceWWN: deviceWWN, volumeLUNAddress: volumeLUNAddress, - iscsiTargets: iscsiTargets, nvmetcpTargets: nvmeTCPTargets, nvmefcTargets: nvmeFCTargets, fcTargets: fcTargets}, nil + return scsiPublishContextData{ + deviceWWN: deviceWWN, volumeLUNAddress: volumeLUNAddress, + iscsiTargets: iscsiTargets, nvmetcpTargets: nvmeTCPTargets, nvmefcTargets: nvmeFCTargets, fcTargets: fcTargets, + }, nil } -func readISCSITargetsFromPublishContext(pc map[string]string) []gobrick.ISCSITargetInfo { +func readISCSITargetsFromPublishContext(pc map[string]string, isRemote bool) []gobrick.ISCSITargetInfo { var targets []gobrick.ISCSITargetInfo + iscsiTargetsKey := identifiers.TargetMapISCSITargetsPrefix + iscsiPortalsKey := identifiers.TargetMapISCSIPortalsPrefix + if isRemote { + iscsiTargetsKey = identifiers.TargetMapRemoteISCSITargetsPrefix + iscsiPortalsKey = identifiers.TargetMapRemoteISCSIPortalsPrefix + } for i := 0; ; i++ { target := gobrick.ISCSITargetInfo{} - t, tfound := pc[fmt.Sprintf("%s%d", common.PublishContextISCSITargetsPrefix, i)] + t, tfound := pc[fmt.Sprintf("%s%d", iscsiTargetsKey, i)] if tfound { target.Target = t } - p, pfound := pc[fmt.Sprintf("%s%d", common.PublishContextISCSIPortalsPrefix, i)] + p, pfound := pc[fmt.Sprintf("%s%d", iscsiPortalsKey, i)] if pfound { target.Portal = p } @@ -299,7 +376,7 @@ func readISCSITargetsFromPublishContext(pc map[string]string) []gobrick.ISCSITar } if ReachableEndPoint(p) { - // if the portals from the context (set in ControllerPublishVolume) is not reachable from the nodes + // if the portals from the context (set in NodeStageVolume) is not reachable from the nodes targets = append(targets, target) } } @@ -307,15 +384,21 @@ func readISCSITargetsFromPublishContext(pc map[string]string) []gobrick.ISCSITar return targets } -func readNVMETCPTargetsFromPublishContext(pc map[string]string) []gobrick.NVMeTargetInfo { +func readNVMETCPTargetsFromPublishContext(pc map[string]string, isRemote bool) []gobrick.NVMeTargetInfo { var targets []gobrick.NVMeTargetInfo + nvmeTCPTargetsKey := identifiers.TargetMapNVMETCPTargetsPrefix + nvmeTCPPortalsKey := identifiers.TargetMapNVMETCPPortalsPrefix + if isRemote { + nvmeTCPTargetsKey = identifiers.TargetMapRemoteNVMETCPTargetsPrefix + nvmeTCPPortalsKey = identifiers.TargetMapRemoteNVMETCPPortalsPrefix + } for i := 0; ; i++ { target := gobrick.NVMeTargetInfo{} - t, tfound := pc[fmt.Sprintf("%s%d", common.PublishContextNVMETCPTargetsPrefix, i)] + t, tfound := pc[fmt.Sprintf("%s%d", nvmeTCPTargetsKey, i)] if tfound { target.Target = t } - p, pfound := pc[fmt.Sprintf("%s%d", common.PublishContextNVMETCPPortalsPrefix, i)] + p, pfound := pc[fmt.Sprintf("%s%d", nvmeTCPPortalsKey, i)] if pfound { target.Portal = p } @@ -328,15 +411,21 @@ func readNVMETCPTargetsFromPublishContext(pc map[string]string) []gobrick.NVMeTa return targets } -func readNVMEFCTargetsFromPublishContext(pc map[string]string) []gobrick.NVMeTargetInfo { +func readNVMEFCTargetsFromPublishContext(pc map[string]string, isRemote bool) []gobrick.NVMeTargetInfo { var targets []gobrick.NVMeTargetInfo + nvmeFcTargetsKey := identifiers.TargetMapNVMEFCTargetsPrefix + nvmeFcPortalsKey := identifiers.TargetMapNVMEFCPortalsPrefix + if isRemote { + nvmeFcTargetsKey = identifiers.TargetMapRemoteNVMEFCTargetsPrefix + nvmeFcPortalsKey = identifiers.TargetMapRemoteNVMEFCPortalsPrefix + } for i := 0; ; i++ { target := gobrick.NVMeTargetInfo{} - t, tfound := pc[fmt.Sprintf("%s%d", common.PublishContextNVMEFCTargetsPrefix, i)] + t, tfound := pc[fmt.Sprintf("%s%d", nvmeFcTargetsKey, i)] if tfound { target.Target = t } - p, pfound := pc[fmt.Sprintf("%s%d", common.PublishContextNVMEFCPortalsPrefix, i)] + p, pfound := pc[fmt.Sprintf("%s%d", nvmeFcPortalsKey, i)] if pfound { target.Portal = p } @@ -349,10 +438,14 @@ func readNVMEFCTargetsFromPublishContext(pc map[string]string) []gobrick.NVMeTar return targets } -func readFCTargetsFromPublishContext(pc map[string]string) []gobrick.FCTargetInfo { +func readFCTargetsFromPublishContext(pc map[string]string, isRemote bool) []gobrick.FCTargetInfo { var targets []gobrick.FCTargetInfo + fcWwpnKey := identifiers.TargetMapFCWWPNPrefix + if isRemote { + fcWwpnKey = identifiers.TargetMapRemoteFCWWPNPrefix + } for i := 0; ; i++ { - wwpn, tfound := pc[fmt.Sprintf("%s%d", common.PublishContextFCWWPNPrefix, i)] + wwpn, tfound := pc[fmt.Sprintf("%s%d", fcWwpnKey, i)] if !tfound { break } @@ -363,11 +456,11 @@ func readFCTargetsFromPublishContext(pc map[string]string) []gobrick.FCTargetInf } func (s *SCSIStager) connectDevice(ctx context.Context, data scsiPublishContextData) (string, error) { - logFields := common.GetLogFields(ctx) + log := log.WithContext(ctx) var err error lun, err := strconv.Atoi(data.volumeLUNAddress) if err != nil { - log.WithFields(logFields).Errorf("failed to convert lun number to int: %s", err.Error()) + log.Errorf("failed to convert lun number to int: %s", err.Error()) return "", status.Errorf(codes.Internal, "failed to convert lun number to int: %s", err.Error()) } @@ -382,7 +475,7 @@ func (s *SCSIStager) connectDevice(ctx context.Context, data scsiPublishContextD } if err != nil { - log.WithFields(logFields).Errorf("Unable to find device after multiple discovery attempts: %s", err.Error()) + log.Errorf("Unable to find device after multiple discovery attempts: %s", err.Error()) return "", status.Errorf(codes.Internal, "unable to find device after multiple discovery attempts: %s", err.Error()) } @@ -391,8 +484,9 @@ func (s *SCSIStager) connectDevice(ctx context.Context, data scsiPublishContextD } func (s *SCSIStager) connectISCSIDevice(ctx context.Context, - lun int, data scsiPublishContextData) (gobrick.Device, error) { - logFields := common.GetLogFields(ctx) + lun int, data scsiPublishContextData, +) (gobrick.Device, error) { + logFields := csmlog.ExtractFieldsFromContext(ctx) var targets []gobrick.ISCSITargetInfo for _, t := range data.iscsiTargets { targets = append(targets, gobrick.ISCSITargetInfo{Target: t.Target, Portal: t.Portal}) @@ -401,7 +495,7 @@ func (s *SCSIStager) connectISCSIDevice(ctx context.Context, connectorCtx, cFunc := context.WithTimeout(context.Background(), time.Second*120) defer cFunc() - connectorCtx = common.SetLogFields(connectorCtx, logFields) + connectorCtx = csmlog.SetLogFields(connectorCtx, logFields) return s.iscsiConnector.ConnectVolume(connectorCtx, gobrick.ISCSIVolumeInfo{ Targets: targets, Lun: lun, @@ -409,8 +503,9 @@ func (s *SCSIStager) connectISCSIDevice(ctx context.Context, } func (s *SCSIStager) connectNVMEDevice(ctx context.Context, - wwn string, data scsiPublishContextData, useFC bool) (gobrick.Device, error) { - logFields := common.GetLogFields(ctx) + wwn string, data scsiPublishContextData, useFC bool, +) (gobrick.Device, error) { + logFields := csmlog.ExtractFieldsFromContext(ctx) var targets []gobrick.NVMeTargetInfo if useFC { @@ -426,7 +521,7 @@ func (s *SCSIStager) connectNVMEDevice(ctx context.Context, connectorCtx, cFunc := context.WithTimeout(context.Background(), time.Second*120) defer cFunc() - connectorCtx = common.SetLogFields(connectorCtx, logFields) + connectorCtx = csmlog.SetLogFields(connectorCtx, logFields) return s.nvmeConnector.ConnectVolume(connectorCtx, gobrick.NVMeVolumeInfo{ Targets: targets, WWN: wwn, @@ -434,8 +529,9 @@ func (s *SCSIStager) connectNVMEDevice(ctx context.Context, } func (s *SCSIStager) connectFCDevice(ctx context.Context, - lun int, data scsiPublishContextData) (gobrick.Device, error) { - logFields := common.GetLogFields(ctx) + lun int, data scsiPublishContextData, +) (gobrick.Device, error) { + logFields := csmlog.ExtractFieldsFromContext(ctx) var targets []gobrick.FCTargetInfo for _, t := range data.fcTargets { @@ -445,7 +541,7 @@ func (s *SCSIStager) connectFCDevice(ctx context.Context, connectorCtx, cFunc := context.WithTimeout(context.Background(), time.Second*120) defer cFunc() - connectorCtx = common.SetLogFields(connectorCtx, logFields) + connectorCtx = csmlog.SetLogFields(connectorCtx, logFields) return s.fcConnector.ConnectVolume(connectorCtx, gobrick.FCVolumeInfo{ Targets: targets, Lun: lun, @@ -453,18 +549,18 @@ func (s *SCSIStager) connectFCDevice(ctx context.Context, } func isReadyToPublish(ctx context.Context, stagingPath string, fs fs.Interface) (bool, bool, error) { - logFields := common.GetLogFields(ctx) + log := log.WithContext(ctx) stageInfo, found, err := getTargetMount(ctx, stagingPath, fs) if err != nil { return found, false, err } if !found { - log.WithFields(logFields).Warning("staged device not found") + log.Warn("staged device not found") return found, false, nil } if strings.HasSuffix(stageInfo.Source, "deleted") { - log.WithFields(logFields).Warning("staged device linked with deleted path") + log.Warn("staged device linked with deleted path") return found, false, nil } @@ -476,20 +572,81 @@ func isReadyToPublish(ctx context.Context, stagingPath string, fs fs.Interface) } func isReadyToPublishNFS(ctx context.Context, stagingPath string, fs fs.Interface) (bool, error) { - logFields := common.GetLogFields(ctx) + log := log.WithContext(ctx) stageInfo, found, err := getTargetMount(ctx, stagingPath, fs) if err != nil { return found, err } if !found { - log.WithFields(logFields).Warning("staged device not found") + log.Warn("staged device not found") return found, nil } if strings.HasSuffix(stageInfo.Source, "deleted") { - log.WithFields(logFields).Warning("staged device linked with deleted path") + log.Warn("staged device linked with deleted path") return found, nil } return found, nil } + +func (s *SCSIStager) AddTargetsInfoToMap( + targetMap map[string]string, volumeApplianceID string, client gopowerstore.Client, isRemote bool, +) error { + iscsiPortalsKey := identifiers.TargetMapISCSIPortalsPrefix + iscsiTargetsKey := identifiers.TargetMapISCSITargetsPrefix + fcWwpnKey := identifiers.TargetMapFCWWPNPrefix + nvmeFcPortalsKey := identifiers.TargetMapNVMEFCPortalsPrefix + nvmeFcTargetsKey := identifiers.TargetMapNVMEFCTargetsPrefix + nvmeTCPPortalsKey := identifiers.TargetMapNVMETCPPortalsPrefix + nvmeTCPTargetsKey := identifiers.TargetMapNVMETCPTargetsPrefix + if isRemote { + iscsiPortalsKey = identifiers.TargetMapRemoteISCSIPortalsPrefix + iscsiTargetsKey = identifiers.TargetMapRemoteISCSITargetsPrefix + fcWwpnKey = identifiers.TargetMapRemoteFCWWPNPrefix + nvmeFcPortalsKey = identifiers.TargetMapRemoteNVMEFCPortalsPrefix + nvmeFcTargetsKey = identifiers.TargetMapRemoteNVMEFCTargetsPrefix + nvmeTCPPortalsKey = identifiers.TargetMapRemoteNVMETCPPortalsPrefix + nvmeTCPTargetsKey = identifiers.TargetMapRemoteNVMETCPTargetsPrefix + } + + iscsiTargetsInfo, err := identifiers.GetISCSITargetsInfoFromStorage(client, volumeApplianceID) + if err != nil { + log.Errorf("error unable to get iSCSI targets from array %s", err.Error()) + } + for i, t := range iscsiTargetsInfo { + targetMap[fmt.Sprintf("%s%d", iscsiPortalsKey, i)] = t.Portal + targetMap[fmt.Sprintf("%s%d", iscsiTargetsKey, i)] = t.Target + } + fcTargetsInfo, err := identifiers.GetFCTargetsInfoFromStorage(client, volumeApplianceID) + if err != nil { + log.Errorf("error unable to get FC targets from array %s", err.Error()) + } + for i, t := range fcTargetsInfo { + targetMap[fmt.Sprintf("%s%d", fcWwpnKey, i)] = t.WWPN + } + + nvmefcTargetInfo, err := identifiers.GetNVMEFCTargetInfoFromStorage(client, volumeApplianceID) + if err != nil { + log.Errorf("error unable to get NVMeFC targets from array %s", err.Error()) + } + for i, t := range nvmefcTargetInfo { + targetMap[fmt.Sprintf("%s%d", nvmeFcPortalsKey, i)] = t.Portal + targetMap[fmt.Sprintf("%s%d", nvmeFcTargetsKey, i)] = t.Target + } + + nvmetcpTargetInfo, err := identifiers.GetNVMETCPTargetsInfoFromStorage(client, volumeApplianceID) + if err != nil { + log.Errorf("error unable to get NVMeTCP targets from array %s", err.Error()) + } + for i, t := range nvmetcpTargetInfo { + targetMap[fmt.Sprintf("%s%d", nvmeTCPPortalsKey, i)] = t.Portal + targetMap[fmt.Sprintf("%s%d", nvmeTCPTargetsKey, i)] = t.Target + } + + // If the system is not capable of any protocol, then we will through the error + if len(iscsiTargetsInfo) == 0 && len(fcTargetsInfo) == 0 && len(nvmefcTargetInfo) == 0 && len(nvmetcpTargetInfo) == 0 { + return errors.New("unable to get targets for any protocol") + } + return nil +} diff --git a/pkg/node/stager_test.go b/pkg/node/stager_test.go index f35123b8..56ee349e 100644 --- a/pkg/node/stager_test.go +++ b/pkg/node/stager_test.go @@ -1,6 +1,6 @@ /* * - * Copyright © 2021-2023 Dell Inc. or its subsidiaries. All Rights Reserved. + * Copyright © 2021-2024 Dell Inc. or its subsidiaries. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,41 +20,80 @@ package node import ( "context" + "errors" + "fmt" "path/filepath" + "reflect" "testing" - "github.com/container-storage-interface/spec/lib/go/csi" "github.com/dell/csi-powerstore/v2/mocks" - "github.com/dell/csi-powerstore/v2/pkg/common" + "github.com/dell/csi-powerstore/v2/pkg/identifiers" + "github.com/dell/csmlog" + "github.com/dell/gopowerstore" + gopowerstoremock "github.com/dell/gopowerstore/mocks" + "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/dell/gobrick" "github.com/dell/gofsutil" "github.com/golang/mock/gomock" - log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" ) +var validBaseVolID = "39bb1b5f-5624-490d-9ece-18f7b28a904e" + func getValidPublishContext() map[string]string { return map[string]string{ - common.PublishContextLUNAddress: validLUNID, - common.PublishContextDeviceWWN: validDeviceWWN, - common.PublishContextISCSIPortalsPrefix + "0": validISCSIPortals[0], - common.PublishContextISCSIPortalsPrefix + "1": validISCSIPortals[1], - common.PublishContextISCSITargetsPrefix + "0": validISCSITargets[0], - common.PublishContextISCSITargetsPrefix + "1": validISCSITargets[1], - common.PublishContextNVMEFCPortalsPrefix + "0": validNVMEFCPortals[0], - common.PublishContextNVMEFCPortalsPrefix + "1": validNVMEFCPortals[1], - common.PublishContextNVMEFCTargetsPrefix + "0": validNVMEFCTargets[0], - common.PublishContextNVMEFCTargetsPrefix + "1": validNVMEFCTargets[1], - common.PublishContextNVMETCPPortalsPrefix + "0": validNVMETCPPortals[0], - common.PublishContextNVMETCPPortalsPrefix + "1": validNVMETCPPortals[1], - common.PublishContextNVMETCPTargetsPrefix + "0": validNVMETCPTargets[0], - common.PublishContextNVMETCPTargetsPrefix + "1": validNVMETCPTargets[1], - common.PublishContextFCWWPNPrefix + "0": validFCTargetsWWPN[0], - common.PublishContextFCWWPNPrefix + "1": validFCTargetsWWPN[1], + identifiers.TargetMapLUNAddress: validLUNID, + identifiers.TargetMapDeviceWWN: validDeviceWWN, + identifiers.TargetMapISCSIPortalsPrefix + "0": validISCSIPortals[0], + identifiers.TargetMapISCSIPortalsPrefix + "1": validISCSIPortals[1], + identifiers.TargetMapISCSITargetsPrefix + "0": validISCSITargets[0], + identifiers.TargetMapISCSITargetsPrefix + "1": validISCSITargets[1], + identifiers.TargetMapNVMEFCPortalsPrefix + "0": validNVMEFCPortals[0], + identifiers.TargetMapNVMEFCPortalsPrefix + "1": validNVMEFCPortals[1], + identifiers.TargetMapNVMEFCTargetsPrefix + "0": validNVMEFCTargets[0], + identifiers.TargetMapNVMEFCTargetsPrefix + "1": validNVMEFCTargets[1], + identifiers.TargetMapNVMETCPPortalsPrefix + "0": validNVMETCPPortals[0], + identifiers.TargetMapNVMETCPPortalsPrefix + "1": validNVMETCPPortals[1], + identifiers.TargetMapNVMETCPTargetsPrefix + "0": validNVMETCPTargets[0], + identifiers.TargetMapNVMETCPTargetsPrefix + "1": validNVMETCPTargets[1], + identifiers.TargetMapFCWWPNPrefix + "0": validFCTargetsWWPN[0], + identifiers.TargetMapFCWWPNPrefix + "1": validFCTargetsWWPN[1], } } +func getValidUniformMetroPublishContext() map[string]string { + publishContext := getValidPublishContext() + publishContext[identifiers.TargetMapRemoteLUNAddress] = validLUNID + publishContext[identifiers.TargetMapRemoteDeviceWWN] = validDeviceWWN + publishContext[identifiers.TargetMapRemoteISCSIPortalsPrefix+"0"] = validRemoteISCSIPortals[0] + publishContext[identifiers.TargetMapRemoteISCSIPortalsPrefix+"1"] = validRemoteISCSIPortals[1] + publishContext[identifiers.TargetMapRemoteISCSITargetsPrefix+"0"] = validRemoteISCSITargets[0] + publishContext[identifiers.TargetMapRemoteISCSITargetsPrefix+"1"] = validRemoteISCSITargets[1] + publishContext[identifiers.TargetMapRemoteFCWWPNPrefix+"0"] = validRemoteFCTargetsWWPN[0] + publishContext[identifiers.TargetMapRemoteFCWWPNPrefix+"1"] = validRemoteFCTargetsWWPN[1] + + return publishContext +} + +// getValidRemoteMetroPublishContext contains an empty local publish context +// and a populated remote publish context. +func getValidRemoteMetroPublishContext() map[string]string { + publishContext := make(map[string]string) + publishContext[identifiers.TargetMapRemoteLUNAddress] = validLUNID + publishContext[identifiers.TargetMapRemoteDeviceWWN] = validDeviceWWN + publishContext[identifiers.TargetMapRemoteISCSIPortalsPrefix+"0"] = validRemoteISCSIPortals[0] + publishContext[identifiers.TargetMapRemoteISCSIPortalsPrefix+"1"] = validRemoteISCSIPortals[1] + publishContext[identifiers.TargetMapRemoteISCSITargetsPrefix+"0"] = validRemoteISCSITargets[0] + publishContext[identifiers.TargetMapRemoteISCSITargetsPrefix+"1"] = validRemoteISCSITargets[1] + publishContext[identifiers.TargetMapRemoteFCWWPNPrefix+"0"] = validRemoteFCTargetsWWPN[0] + publishContext[identifiers.TargetMapRemoteFCWWPNPrefix+"1"] = validRemoteFCTargetsWWPN[1] + + return publishContext +} + func getCapabilityWithVoltypeAccessFstype(voltype, access, fstype string) *csi.VolumeCapability { // Construct the volume capability capability := new(csi.VolumeCapability) @@ -97,11 +136,110 @@ func scsiStageVolumeOK(util *mocks.UtilInterface, fs *mocks.FsInterface) { fs.On("GetUtil").Return(util) } +func scsiStageVolumeFail(util *mocks.UtilInterface, fs *mocks.FsInterface) { + util.On("BindMount", mock.Anything, "/dev", filepath.Join(nodeStagePrivateDir, validBaseVolumeID)).Return(nil) + fs.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, errors.New("mock staging failure")).Once() +} + +func scsiStageRemoteMetroVolumeOK(util *mocks.UtilInterface, fs *mocks.FsInterface) { + util.On("BindMount", mock.Anything, "/dev", filepath.Join(nodeStagePrivateDir, validBaseVolumeID)).Return(nil) + fs.On("ReadFile", "/proc/self/mountinfo").Return([]byte{}, nil).Times(2) + fs.On("ParseProcMounts", context.Background(), mock.Anything).Return([]gofsutil.Info{}, nil) + fs.On("MkFileIdempotent", filepath.Join(nodeStagePrivateDir, validBaseVolumeID)).Return(true, nil) + fs.On("GetUtil").Return(util) +} + +// setDefaultClientMocks sets default mock values for gopowerstore client, no matter what protocol is used, the mocks needed are the same +func setDefaultClientMocks() { + clientMock.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9"}, nil) + + clientMock.On("GetVolume", mock.Anything, validRemoteVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9"}, nil) + + clientMock.On("GetCluster", mock.Anything). + Return(gopowerstore.Cluster{Name: validClusterName}, nil) + + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn"}, + }, + }, nil) + clientMock.On("GetStorageNVMETCPTargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn"}, + }, + }, nil) + clientMock.On("GetFCPorts", mock.Anything). + Return([]gopowerstore.FcPort{ + { + IsLinkUp: true, + Wwn: "58:cc:f0:93:48:a0:03:a3", + WwnNVMe: "58ccf091492b0c22", + WwnNode: "58ccf090c9200c22", + }, + }, nil) +} + +func setCustomClientMocks(iscsiTargets string, wwn string, nvmeTCPTargets string, nvmeNqn string) { + // for any given item, return an error if the item is not set + if iscsiTargets == "" { + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). + Return(nil, errors.New("error")) + } else { + clientMock.On("GetStorageISCSITargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: iscsiTargets}, + }, + }, nil) + } + + if wwn == "" { + clientMock.On("GetFCPorts", mock.Anything). + Return(nil, errors.New("error")) + } else { + clientMock.On("GetFCPorts", mock.Anything). + Return([]gopowerstore.FcPort{ + { + IsLinkUp: true, + Wwn: wwn, + WwnNVMe: wwn, + WwnNode: wwn, + }, + }, nil) + } + + if nvmeTCPTargets == "" { + clientMock.On("GetStorageNVMETCPTargetAddresses", mock.Anything). + Return(nil, errors.New("error")) + } else { + clientMock.On("GetStorageNVMETCPTargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: nvmeTCPTargets}, + }, + }, nil) + } + + // always set the cluster value + clientMock.On("GetCluster", mock.Anything). + Return(gopowerstore.Cluster{Name: validClusterName, NVMeNQN: nvmeNqn}, nil) +} + func TestSCSIStager_Stage(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() t.Run("iscsi -- success test", func(t *testing.T) { + setVariables() + setDefaultClientMocks() iscsiConnectorMock := new(mocks.ISCSIConnector) fcConnectorMock := new(mocks.FcConnector) nvmeConnectorMock := new(mocks.NVMEConnector) @@ -114,37 +252,25 @@ func TestSCSIStager_Stage(t *testing.T) { fcConnector: fcConnectorMock, } - iscsiConnectorMock.On("ConnectVolume", mock.Anything, gobrick.ISCSIVolumeInfo{ - Targets: []gobrick.ISCSITargetInfo{ - { - Portal: validISCSIPortals[0], - Target: validISCSITargets[0], - }, - { - Portal: validISCSIPortals[1], - Target: validISCSITargets[1], - }, - }, - Lun: validLUNIDINT, - }).Return(gobrick.Device{}, nil) + iscsiConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything).Return(gobrick.Device{}, nil) utilMock := new(mocks.UtilInterface) fsMock := new(mocks.FsInterface) scsiStageVolumeOK(utilMock, fsMock) - _, err := stager.Stage(context.Background(), &csi.NodeStageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: nodeStagePrivateDir, VolumeCapability: getCapabilityWithVoltypeAccessFstype( "block", "single-writer", "none"), - }, log.Fields{}, fsMock, validBaseVolumeID) - + }, filepath.Join(nodeStagePrivateDir, validBaseVolumeID), "node-1", csmlog.Fields{}, fsMock, validBaseVolumeID, false, clientMock) assert.Nil(t, err) }) t.Run("nvmefc -- success test", func(t *testing.T) { + setVariables() + setDefaultClientMocks() iscsiConnectorMock := new(mocks.ISCSIConnector) fcConnectorMock := new(mocks.FcConnector) nvmeConnectorMock := new(mocks.NVMEConnector) @@ -157,19 +283,7 @@ func TestSCSIStager_Stage(t *testing.T) { fcConnector: fcConnectorMock, } - nvmeConnectorMock.On("ConnectVolume", mock.Anything, gobrick.NVMeVolumeInfo{ - Targets: []gobrick.NVMeTargetInfo{ - { - Portal: validNVMEFCPortals[0], - Target: validNVMEFCTargets[0], - }, - { - Portal: validNVMEFCPortals[1], - Target: validNVMEFCTargets[1], - }, - }, - WWN: validDeviceWWN, - }, true).Return(gobrick.Device{}, nil) + nvmeConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything, true).Return(gobrick.Device{}, nil) utilMock := new(mocks.UtilInterface) fsMock := new(mocks.FsInterface) @@ -177,17 +291,141 @@ func TestSCSIStager_Stage(t *testing.T) { scsiStageVolumeOK(utilMock, fsMock) _, err := stager.Stage(context.Background(), &csi.NodeStageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: nodeStagePrivateDir, VolumeCapability: getCapabilityWithVoltypeAccessFstype( "block", "single-writer", "none"), - }, log.Fields{}, fsMock, validBaseVolumeID) + }, filepath.Join(nodeStagePrivateDir, validBaseVolumeID), "node-1", csmlog.Fields{}, fsMock, validBaseVolumeID, false, clientMock) assert.Nil(t, err) }) t.Run("nvmetcp -- success test", func(t *testing.T) { + setVariables() + setDefaultClientMocks() + iscsiConnectorMock := new(mocks.ISCSIConnector) + fcConnectorMock := new(mocks.FcConnector) + nvmeConnectorMock := new(mocks.NVMEConnector) + + stager := &SCSIStager{ + useFC: false, + useNVME: true, + iscsiConnector: iscsiConnectorMock, + nvmeConnector: nvmeConnectorMock, + fcConnector: fcConnectorMock, + } + + nvmeConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything, false).Return(gobrick.Device{}, nil) + + utilMock := new(mocks.UtilInterface) + fsMock := new(mocks.FsInterface) + + scsiStageVolumeOK(utilMock, fsMock) + _, err := stager.Stage(context.Background(), &csi.NodeStageVolumeRequest{ + VolumeId: validBlockVolumeHandle, + PublishContext: getValidPublishContext(), + StagingTargetPath: nodeStagePrivateDir, + VolumeCapability: getCapabilityWithVoltypeAccessFstype( + "block", "single-writer", "none"), + }, filepath.Join(nodeStagePrivateDir, validBaseVolumeID), "node-1", csmlog.Fields{}, fsMock, validBaseVolumeID, false, clientMock) + + assert.Nil(t, err) + }) + + // originally a test for publisher, the logic and corresponding test for checking targets is now in the stager, so this test has been moved here + t.Run("no protocols can be used", func(t *testing.T) { + setVariables() + client := new(gopowerstoremock.Client) + iscsiConnectorMock := new(mocks.ISCSIConnector) + fcConnectorMock := new(mocks.FcConnector) + nvmeConnectorMock := new(mocks.NVMEConnector) + + stager := &SCSIStager{ + useFC: false, + useNVME: false, + iscsiConnector: iscsiConnectorMock, + nvmeConnector: nvmeConnectorMock, + fcConnector: fcConnectorMock, + } + + e := errors.New("unable to get targets for any protocol") + client.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9"}, nil) + client.On("GetCluster", mock.Anything). + Return(gopowerstore.Cluster{Name: validClusterName}, nil) + iscsiConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything).Return(gobrick.Device{}, nil) + client.On("GetStorageISCSITargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{}, e) + client.On("GetStorageNVMETCPTargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{}, e) + client.On("GetFCPorts", mock.Anything). + Return([]gopowerstore.FcPort{}, nil) + + utilMock := new(mocks.UtilInterface) + fsMock := new(mocks.FsInterface) + + scsiStageVolumeOK(utilMock, fsMock) + _, err := stager.Stage(context.Background(), &csi.NodeStageVolumeRequest{ + VolumeId: validBlockVolumeHandle, + PublishContext: getValidPublishContext(), + StagingTargetPath: nodeStagePrivateDir, + VolumeCapability: getCapabilityWithVoltypeAccessFstype( + "block", "single-writer", "none"), + }, filepath.Join(nodeStagePrivateDir, validBaseVolumeID), "node-1", csmlog.Fields{}, fsMock, validBaseVolumeID, false, client) + assert.NotNil(t, err) + assert.Contains(t, err.Error(), "unable to get targets for any protocol") + }) + t.Run("nvmeFC is specified but cannot be used", func(t *testing.T) { + setVariables() + client := new(gopowerstoremock.Client) + iscsiConnectorMock := new(mocks.ISCSIConnector) + fcConnectorMock := new(mocks.FcConnector) + nvmeConnectorMock := new(mocks.NVMEConnector) + + stager := &SCSIStager{ + useFC: true, + useNVME: true, + iscsiConnector: iscsiConnectorMock, + nvmeConnector: nvmeConnectorMock, + fcConnector: fcConnectorMock, + } + + e := errors.New("unable to get targets for any protocol") + client.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9"}, nil) + client.On("GetCluster", mock.Anything). + Return(gopowerstore.Cluster{Name: validClusterName}, nil) + nvmeConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything).Return(gobrick.Device{}, nil) + client.On("GetStorageISCSITargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{}, e) + client.On("GetStorageNVMETCPTargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn"}, + }, + }, nil) + client.On("GetFCPorts", mock.Anything). + Return([]gopowerstore.FcPort{}, nil) + + utilMock := new(mocks.UtilInterface) + fsMock := new(mocks.FsInterface) + + scsiStageVolumeOK(utilMock, fsMock) + _, err := stager.Stage(context.Background(), &csi.NodeStageVolumeRequest{ + VolumeId: validBlockVolumeHandle, + PublishContext: getValidPublishContext(), + StagingTargetPath: nodeStagePrivateDir, + VolumeCapability: getCapabilityWithVoltypeAccessFstype( + "block", "single-writer", "none"), + }, filepath.Join(nodeStagePrivateDir, validBaseVolumeID), "node-1", csmlog.Fields{}, fsMock, validBaseVolumeID, false, client) + assert.NotNil(t, err) + assert.Contains(t, err.Error(), "NVMeFC Targets data must be in publish context") + }) + t.Run("nvmeTCP is specified but cannot be used", func(t *testing.T) { + setVariables() + client := new(gopowerstoremock.Client) iscsiConnectorMock := new(mocks.ISCSIConnector) fcConnectorMock := new(mocks.FcConnector) nvmeConnectorMock := new(mocks.NVMEConnector) @@ -200,32 +438,329 @@ func TestSCSIStager_Stage(t *testing.T) { fcConnector: fcConnectorMock, } - nvmeConnectorMock.On("ConnectVolume", mock.Anything, gobrick.NVMeVolumeInfo{ - Targets: []gobrick.NVMeTargetInfo{ + e := errors.New("unable to get targets for any protocol") + client.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9"}, nil) + client.On("GetCluster", mock.Anything). + Return(gopowerstore.Cluster{Name: validClusterName}, nil) + nvmeConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything).Return(gobrick.Device{}, nil) + client.On("GetStorageISCSITargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{}, e) + client.On("GetStorageNVMETCPTargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{}, e) + client.On("GetFCPorts", mock.Anything). + Return([]gopowerstore.FcPort{ { - Portal: validNVMETCPPortals[0], - Target: validNVMETCPTargets[0], + IsLinkUp: true, + Wwn: "58:cc:f0:93:48:a0:03:a3", + WwnNVMe: "58ccf091492b0c22", + WwnNode: "58ccf090c9200c22", }, + }, nil) + utilMock := new(mocks.UtilInterface) + fsMock := new(mocks.FsInterface) + + scsiStageVolumeOK(utilMock, fsMock) + _, err := stager.Stage(context.Background(), &csi.NodeStageVolumeRequest{ + VolumeId: validBlockVolumeHandle, + PublishContext: getValidPublishContext(), + StagingTargetPath: nodeStagePrivateDir, + VolumeCapability: getCapabilityWithVoltypeAccessFstype( + "block", "single-writer", "none"), + }, filepath.Join(nodeStagePrivateDir, validBaseVolumeID), "node-1", csmlog.Fields{}, fsMock, validBaseVolumeID, false, client) + assert.NotNil(t, err) + assert.Contains(t, err.Error(), "NVMeTCP Targets data must be in publish context") + }) + t.Run("iscsi is specified but cannot be used", func(t *testing.T) { + setVariables() + client := new(gopowerstoremock.Client) + iscsiConnectorMock := new(mocks.ISCSIConnector) + fcConnectorMock := new(mocks.FcConnector) + nvmeConnectorMock := new(mocks.NVMEConnector) + + stager := &SCSIStager{ + useFC: false, + useNVME: false, + iscsiConnector: iscsiConnectorMock, + nvmeConnector: nvmeConnectorMock, + fcConnector: fcConnectorMock, + } + + e := errors.New("unable to get targets for any protocol") + client.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9"}, nil) + client.On("GetCluster", mock.Anything). + Return(gopowerstore.Cluster{Name: validClusterName}, nil) + iscsiConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything).Return(gobrick.Device{}, nil) + client.On("GetStorageISCSITargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{}, e) + client.On("GetStorageNVMETCPTargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{}, e) + client.On("GetFCPorts", mock.Anything). + Return([]gopowerstore.FcPort{ { - Portal: validNVMETCPPortals[1], - Target: validNVMETCPTargets[1], + IsLinkUp: true, + Wwn: "58:cc:f0:93:48:a0:03:a3", + WwnNVMe: "58ccf091492b0c22", + WwnNode: "58ccf090c9200c22", }, - }, - WWN: validDeviceWWN, - }, false).Return(gobrick.Device{}, nil) + }, nil) utilMock := new(mocks.UtilInterface) fsMock := new(mocks.FsInterface) scsiStageVolumeOK(utilMock, fsMock) _, err := stager.Stage(context.Background(), &csi.NodeStageVolumeRequest{ - VolumeId: validBlockVolumeID, + VolumeId: validBlockVolumeHandle, PublishContext: getValidPublishContext(), StagingTargetPath: nodeStagePrivateDir, VolumeCapability: getCapabilityWithVoltypeAccessFstype( "block", "single-writer", "none"), - }, log.Fields{}, fsMock, validBaseVolumeID) + }, filepath.Join(nodeStagePrivateDir, validBaseVolumeID), "node-1", csmlog.Fields{}, fsMock, validBaseVolumeID, false, client) + assert.NotNil(t, err) + assert.Contains(t, err.Error(), "iscsiTargets data must be in publish context") + }) + t.Run("fc is specified but cannot be used", func(t *testing.T) { + setVariables() + client := new(gopowerstoremock.Client) + iscsiConnectorMock := new(mocks.ISCSIConnector) + fcConnectorMock := new(mocks.FcConnector) + nvmeConnectorMock := new(mocks.NVMEConnector) - assert.Nil(t, err) + stager := &SCSIStager{ + useFC: true, + useNVME: false, + iscsiConnector: iscsiConnectorMock, + nvmeConnector: nvmeConnectorMock, + fcConnector: fcConnectorMock, + } + + e := errors.New("unable to get targets for any protocol") + client.On("GetVolume", mock.Anything, validBaseVolID). + Return(gopowerstore.Volume{ID: validBaseVolID, Wwn: "naa.68ccf098003ceb5e4577a20be6d11bf9"}, nil) + client.On("GetCluster", mock.Anything). + Return(gopowerstore.Cluster{Name: validClusterName}, nil) + nvmeConnectorMock.On("ConnectVolume", mock.Anything, mock.Anything).Return(gobrick.Device{}, nil) + client.On("GetStorageISCSITargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{}, e) + client.On("GetStorageNVMETCPTargetAddresses", mock.Anything). + Return([]gopowerstore.IPPoolAddress{ + { + Address: "192.168.1.1", + IPPort: gopowerstore.IPPortInstance{TargetIqn: "iqn"}, + }, + }, nil) + client.On("GetFCPorts", mock.Anything). + Return([]gopowerstore.FcPort{}, nil) + + utilMock := new(mocks.UtilInterface) + fsMock := new(mocks.FsInterface) + + scsiStageVolumeOK(utilMock, fsMock) + _, err := stager.Stage(context.Background(), &csi.NodeStageVolumeRequest{ + VolumeId: validBlockVolumeHandle, + PublishContext: getValidPublishContext(), + StagingTargetPath: nodeStagePrivateDir, + VolumeCapability: getCapabilityWithVoltypeAccessFstype( + "block", "single-writer", "none"), + }, filepath.Join(nodeStagePrivateDir, validBaseVolumeID), "node-1", csmlog.Fields{}, fsMock, validBaseVolumeID, false, client) + assert.NotNil(t, err) + assert.Contains(t, err.Error(), "fcTargets data must be in publish context") }) } + +func TestSCSIStager_getLunAddressFromArray(t *testing.T) { + tests := []struct { + name string + getHostByNameFunc func(ctx context.Context, name string) (gopowerstore.Host, error) + getHostVolumeMappingFunc func(ctx context.Context, volumeID string) ([]gopowerstore.HostVolumeMapping, error) + volumeID string + nodeID string + want string + wantErr bool + }{ + { + name: "valid array and volume", + getHostByNameFunc: func(_ context.Context, _ string) (gopowerstore.Host, error) { + return gopowerstore.Host{ + ID: validHostID, + Initiators: []gopowerstore.InitiatorInstance{}, + Name: "host-name", + }, nil + }, + getHostVolumeMappingFunc: func(_ context.Context, _ string) ([]gopowerstore.HostVolumeMapping, error) { + return []gopowerstore.HostVolumeMapping{ + {HostID: validHostID, LogicalUnitNumber: validLUNIDINT}, + }, nil + }, + volumeID: "valid-volume-id", + nodeID: validHostID, + want: validLUNID, + wantErr: false, + }, + { + name: "invalid volume ID", + getHostByNameFunc: func(_ context.Context, _ string) (gopowerstore.Host, error) { + return gopowerstore.Host{ + ID: validHostID, + Initiators: []gopowerstore.InitiatorInstance{}, + Name: "host-name", + }, nil + }, + getHostVolumeMappingFunc: func(_ context.Context, _ string) ([]gopowerstore.HostVolumeMapping, error) { + return nil, errors.New("invalid volume ID") + }, + volumeID: "invalid-volume-id", + nodeID: "valid-node-id", + want: "", + wantErr: true, + }, + { + name: "host not found", + getHostByNameFunc: func(_ context.Context, _ string) (gopowerstore.Host, error) { + return gopowerstore.Host{}, errors.New("invalid host") + }, + getHostVolumeMappingFunc: func(_ context.Context, _ string) ([]gopowerstore.HostVolumeMapping, error) { + return []gopowerstore.HostVolumeMapping{ + {HostID: validHostID, LogicalUnitNumber: validLUNIDINT}, + }, nil + }, + volumeID: "valid-volume-id", + nodeID: "valid-node-id", + want: "", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + clientMock.ExpectedCalls = nil // reset previous expectations + + // Mock GetHostByName using closures + clientMock.On("GetHostByName", mock.Anything, mock.AnythingOfType("string")). + Return(func(ctx context.Context, name string) gopowerstore.Host { + host, _ := tt.getHostByNameFunc(ctx, name) + return host + }, func(ctx context.Context, name string) error { + _, err := tt.getHostByNameFunc(ctx, name) + return err + }) + + // Mock GetHostVolumeMappingByVolumeID using closures + clientMock.On("GetHostVolumeMappingByVolumeID", mock.Anything, mock.AnythingOfType("string")). + Return(func(ctx context.Context, volumeID string) []gopowerstore.HostVolumeMapping { + mapping, _ := tt.getHostVolumeMappingFunc(ctx, volumeID) + return mapping + }, func(ctx context.Context, volumeID string) error { + _, err := tt.getHostVolumeMappingFunc(ctx, volumeID) + return err + }) + + // Call the method under test + got, err := getLunAddressFromArray(context.Background(), clientMock, tt.volumeID, tt.nodeID) + + // Validate error expectation + if (err != nil) != tt.wantErr { + t.Errorf("getLunAddressFromArray() error = %v, wantErr %v", err, tt.wantErr) + return + } + + // Validate result + if got != tt.want { + t.Errorf("getLunAddressFromArray() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestSCSIStager_AddTargetsInfoToMap(t *testing.T) { + tests := []struct { + name string + isRemote bool + volumeApplianceID string + iscsiTargetsInfo string + fcWwn string + nvmeNqn string + nvmeTCPTargetsInfo string + expectedTargetMap map[string]string + expectErr bool + }{ + { + name: "local targets - all", + isRemote: false, + volumeApplianceID: "", // should be empty in most cases + iscsiTargetsInfo: "test", + fcWwn: "testWwn", + nvmeNqn: "testNqn", + nvmeTCPTargetsInfo: "test2", // only determines if NVME TCP will be added to mock/error out, value will be nvmeNqn + expectedTargetMap: map[string]string{ + identifiers.TargetMapISCSIPortalsPrefix + "0": "192.168.1.1:3260", + identifiers.TargetMapISCSITargetsPrefix + "0": "test", + identifiers.TargetMapFCWWPNPrefix + "0": "testWwn", + identifiers.TargetMapNVMEFCPortalsPrefix + "0": "nn-0xtestWwn:pn-0xtestWwn", + identifiers.TargetMapNVMEFCTargetsPrefix + "0": "testNqn", + identifiers.TargetMapNVMETCPPortalsPrefix + "0": "192.168.1.1:4420", + identifiers.TargetMapNVMETCPTargetsPrefix + "0": "testNqn", + }, + expectErr: false, + }, + { + name: "fail to get NVME and FC targets but ISCSI succeeds", + isRemote: false, + volumeApplianceID: "", // should be empty in most cases + iscsiTargetsInfo: "test", + fcWwn: "", + nvmeNqn: "", + expectedTargetMap: map[string]string{ + identifiers.TargetMapISCSIPortalsPrefix + "0": "192.168.1.1:3260", + identifiers.TargetMapISCSITargetsPrefix + "0": "test", + }, + expectErr: false, + }, + { + name: "fail to get all targets", + isRemote: false, + volumeApplianceID: "", // should be empty in most cases + iscsiTargetsInfo: "", + fcWwn: "", + nvmeNqn: "", + expectedTargetMap: map[string]string{}, + expectErr: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // set the variables before each run + setVariables() + + // put test-specificvalues into the client mock + setCustomClientMocks(test.iscsiTargetsInfo, test.fcWwn, test.nvmeTCPTargetsInfo, test.nvmeNqn) + + // fill any values that are not set by custom + // note: mock.On will not override previous values with mock.anything + setDefaultClientMocks() + + targetMap := make(map[string]string) + stager := &SCSIStager{} + err := stager.AddTargetsInfoToMap(targetMap, test.volumeApplianceID, clientMock, test.isRemote) + fmt.Println("Actual map: ") + fmt.Println(targetMap) + + // if there was an error and we expected none, fail + if err != nil && test.expectErr == false { + t.Errorf("AddTargetsInfoToMap returned unexpected error: %v", err) + } + + // if there was no error and we expected one, fail + if test.expectErr { + assert.NotNil(t, err) + } + + // if the returned map doesn't match what was expected, fail + if !reflect.DeepEqual(targetMap, test.expectedTargetMap) { + t.Errorf("AddTargetsInfoToMap returned unexpected target map. Expected: %+v, Actual: %+v", test.expectedTargetMap, targetMap) + } + }) + } +} diff --git a/pkg/tracer/tracer.go b/pkg/tracer/tracer.go index bd7fdf6d..12d2ed71 100644 --- a/pkg/tracer/tracer.go +++ b/pkg/tracer/tracer.go @@ -37,7 +37,6 @@ type Configurator interface { func NewTracer(configurator Configurator) (opentracing.Tracer, io.Closer, error) { // load config from environment variables cfg, err := configurator.FromEnv() - if err != nil { return nil, nil, err } diff --git a/samples/secret/secret.yaml b/samples/secret/secret.yaml index e8c1c6f9..7bd6aa29 100644 --- a/samples/secret/secret.yaml +++ b/samples/secret/secret.yaml @@ -1,92 +1,240 @@ -# -# -# Copyright © 2021-2023 Dell Inc. or its subsidiaries. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# - -# You can apply current config to Kubernetes cluster by running following command: -# -# kubectl create secret generic powerstore-config -n csi-powerstore --from-file=config=secret.yaml -# -arrays: - # endpoint: full URL path to the PowerStore API - # Allowed Values: https://*.*.*.*/api/rest or https://abc.com/api/rest - # Default Value: None - - endpoint: "https://10.0.0.1/api/rest" - - # globalID: unique id of the PowerStore array - # Allowed Values: string - # Default Value: None - globalID: "unique" - - # username: username for connecting to API - # Allowed Values: string - # Default Value: None - username: "user" - - # password: password for connecting to API - # Allowed Values: string - # Default Value: None - password: "password" - - # skipCertificateValidation: indicates if client side validation of (management)server's certificate can be skipped - # Allowed Values: - # true: client side validation of (management)server's certificate will be skipped - # false: client side validation of (management)server's certificate will not be skipped - # Default Value: None - skipCertificateValidation: true - - # isDefault: treat current array as a default - # Allowed Values: - # true: would be used by storage classes without arrayID parameter - # false: would not be used by default - # Default Value: false - isDefault: true - - # blockProtocol: what SCSI transport protocol use on node side (FC, ISCSI, NVMeTCP, NVMeFC, None, or auto) - # Allowed Values: - # FC: FC protocol will be used - # ISCSI: iSCSI protocol will be used - # NVMeTCP: NVMe/TCP protocol will be used - # NVMeFC: NVMe/FC protocol will be used - # None: No block protocol can be used - # auto: NVMeFC, NVMe/TCP, FC or iSCSI protocol will be used - # Default Value: None - blockProtocol: "auto" - - # nasName: what NAS should be used for NFS volumes - # Allowed Values: string - (name of NAS server) - # Default Value: None - nasName: "nas-server" - - # nfsAcls: enables setting permissions on NFS mount directory - # This value will be used if a storage class does not have the NFS ACL (nfsAcls) parameter specified - # Permissions can be specified in two formats: - # 1) Unix mode (NFSv3) - # 2) NFSv4 ACLs (NFSv4) - # NFSv4 ACLs are supported on NFSv4 share only. - # Allowed values: - # 1) Unix mode: valid octal mode number - # Examples: "0777", "777", "0755" - # 2) NFSv4 acls: valid NFSv4 acls, seperated by comma - # Examples: "A::OWNER@:RWX,A::GROUP@:RWX", "A::OWNER@:rxtncy" - # Optional: true - # Default value: "0777" - # nfsAcls: "0777" - - - endpoint: "https://11.0.0.1/api/rest" - globalID: "unique" - username: "user" - password: "password" - skipCertificateValidation: true - blockProtocol: "FC" +# +# +# Copyright © 2021-2023 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +# You can apply current config to Kubernetes cluster by running following command: +# +# kubectl create secret generic powerstore-config -n csi-powerstore --from-file=config=secret.yaml +# +arrays: + # endpoint: full URL path to the PowerStore API + # Allowed Values: https://*.*.*.*/api/rest or https://abc.com/api/rest + # Default Value: None + - endpoint: "https://10.0.0.1/api/rest" + + # globalID: unique id of the PowerStore array + # Allowed Values: string + # Default Value: None + globalID: "unique" + + # username: username for connecting to API + # Allowed Values: string + # Default Value: None + username: "user" + + # password: password for connecting to API + # Allowed Values: string + # Default Value: None + password: "password" + + # skipCertificateValidation: indicates if client side validation of (management)server's certificate can be skipped + # Allowed Values: + # true: client side validation of (management)server's certificate will be skipped + # false: client side validation of (management)server's certificate will not be skipped + # Default Value: None + skipCertificateValidation: true + + # isDefault: treat current array as a default + # Allowed Values: + # true: would be used by storage classes without arrayID parameter + # false: would not be used by default + # Default Value: false + isDefault: true + + # blockProtocol: what SCSI transport protocol use on node side (FC, ISCSI, NVMeTCP, NVMeFC, None, or auto) + # Allowed Values: + # FC: FC protocol will be used + # ISCSI: iSCSI protocol will be used + # NVMeTCP: NVMe/TCP protocol will be used + # NVMeFC: NVMe/FC protocol will be used + # None: No block protocol can be used + # auto: NVMeFC, NVMe/TCP, FC or iSCSI protocol will be used + # Default Value: None + blockProtocol: "auto" + + # nasName: what NAS should be used for NFS volumes + # Allowed Values: string - (name of NAS server) + # Default Value: None + nasName: "nas-server" + + # nfsAcls: enables setting permissions on NFS mount directory + # This value will be used if a storage class does not have the NFS ACL (nfsAcls) parameter specified + # Permissions can be specified in two formats: + # 1) Unix mode (NFSv3) + # 2) NFSv4 ACLs (NFSv4) + # NFSv4 ACLs are supported on NFSv4 share only. + # Allowed values: + # 1) Unix mode: valid octal mode number + # Examples: "0777", "777", "0755" + # 2) NFSv4 acls: valid NFSv4 acls, seperated by comma + # Examples: "A::OWNER@:RWX,A::GROUP@:RWX", "A::OWNER@:rxtncy" + # Optional: true + # Default value: "0777" + # nfsAcls: "0777" + + # metroTopology is used in conjunction with "labels" to provide host based + # registration for powerstore metro, where host nodes containing the provided + # label will be registered as co-located local hosts, and nodes with the label + # specified in the configuration for the metro replication target will be + # registered as co-located remote hosts. All other nodes are registered as + # co-located both hosts. + # + # *** Deprecated *** + # Use hostConnectivity instead. + # metroTopology is deprecated and remains only to support backward compatibility. + # + # Allowed Values: Uniform + # Default Value: Uniform + # + # Example: + # metroTopology: Uniform + + # label is used in conjunction with metroTopology to determine if the host + # will be used to match the node label to decide the type of host registration, + # Only one label should be specified. + # + # *** Deprecated *** + # Use hostConnectivity instead. + # Use of "labels" with "metroTopology" is deprecated and remains to support + # backward compatibility. + # + # Optional: true + # Allowed Values: a single key-value pair + # Default Value: None + # + # Example: + # labels: + # topology.kubernetes.io/zone: "zone-a" + + # hostConnectivity is used in conjunction with metro replication to configure + # multipath optimization for a node's host entry in this PowerStore system. + # + # UNIFORM METRO + # Uniform metro is configured under the `metro` field. + # For each desired optimization (`metro.colocatedLocal`, `metro.colocatedRemote`, + # `metro.colocatedBoth`), provide a set of label expressions to describe a node + # whose host should be registered with this PowerStore, with the given multipath + # optimization. + # + # If local, non-metro hosts are required alongside uniform metro hosts, under + # `hostConnectivity.local`, provide a set of label expressions to describe a node + # whose host should be registered with this PowerStore, without multipath optimization. + # + # NON-UNIFORM METRO + # Non-uniform host nodes should be registered as `local` and are configured + # under the `local` field. Non-uniform hosts in PowerStore do not require multipath + # optimization because the host nodes are only communicating with one side of the metro + # volume. + # To register a node as a non-uniform, metro host, under `hostConnectivity.local`, + # provide a set of label expressions to describe a node whose host should be + # registered with this PowerStore, without multipath optimization. + # + # matchExpressions follow the format outlined by Kubernetes when describing Node Affinity. + # + # Optional: true + # Default: none + # + # Example: Uniform metro with additional local, non-metro hosts + # hostConnectivity: + # local: + # nodeSelectorTerms: + # - matchExpressions: + # # This expression matches the remaining nodes that are not already described by + # # the metro topology described below. + # - key: "topology.kubernetes.io/zone" + # operator: "NotIn" + # values: + # - "zone-a" + # - "zone-b" + # - "zone-ab" + # metro: + # colocatedLocal: + # # A list of node selection terms. The terms are ORed. + # nodeSelectorTerms: + # # "matchExpressions" is a list of node selector requirements. + # # The requirements are ANDed. + # - matchExpressions: + # # "Key" is the label key that the selector applies to. + # - key: "topology.kubernetes.io/zone" + # # "Operator" represents a key's relationship to a set of values. + # # Allowed values: "In", "NotIn", "Exists", "DoesNotExist", "Gt", "Lt" + # operator: "In" + # # "Values" is an array of string values. + # # If the operator is In or NotIn, the values array must be non-empty. + # # If the operator is Exists or DoesNotExist, the values array must be empty. + # # If the operator is Gt or Lt, the values array must have a single element, + # # which will be interpreted as an integer. + # values: + # - "zone-a" + # colocatedRemote: + # nodeSelectorTerms: + # - matchExpressions: + # - key: "topology.kubernetes.io/zone" + # operator: "In" + # values: + # - "zone-b" + # colocatedBoth: + # nodeSelectorTerms: + # - matchExpressions: + # - key: "topology.kubernetes.io/zone" + # operator: "In" + # values: + # - "zone-ab" + # # OR: to successfully register a node, it must match AT LEAST ONE of the + # # "matchExpressions" under the "nodeSelectorTerms" + # - matchExpressions: + # - key: "topology.kubernetes.io/region" + # operator: "In" + # values: + # - "region-1" + # # AND: to successfully register a node, it must match ALL conditions + # # described by entries under "matchExpressions" + # - key: "topology.kubernetes.io/zone" + # operator: "NotIn" + # values: + # - "zone-a" + # - "zone-b" + + +# To add more PowerStore arrays, uncomment the following lines and provide the required values +# - endpoint: "https://11.0.0.1/api/rest" +# globalID: "unique" +# username: "user" +# password: "password" +# skipCertificateValidation: true +# blockProtocol: "FC" +# hostConnectivity: +# metro: +# colocatedLocal: +# nodeSelectorTerms: +# - matchExpressions: +# - key: "topology.kubernetes.io/zone" +# operator: "In" +# values: +# - "zone-b" +# colocatedRemote: +# nodeSelectorTerms: +# - matchExpressions: +# - key: "topology.kubernetes.io/zone" +# operator: "In" +# values: +# - "zone-a" +# colocatedBoth: +# nodeSelectorTerms: +# - matchExpressions: +# - key: "topology.kubernetes.io/zone" +# operator: "In" +# values: +# - "zone-ab" diff --git a/samples/storageclass/powerstore-metro.yaml b/samples/storageclass/powerstore-metro.yaml new file mode 100644 index 00000000..5fd293f4 --- /dev/null +++ b/samples/storageclass/powerstore-metro.yaml @@ -0,0 +1,57 @@ +# Copyright © 2024 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: "powerstore-metro" +provisioner: "csi-powerstore.dellemc.com" +reclaimPolicy: Delete +volumeBindingMode: Immediate +allowVolumeExpansion: true +parameters: + # Indicates whether replication is enabled + # Allowed values: + # true: replication is enabled + # false: replication is disabled + # Default value: false + replication.storage.dell.com/isReplicationEnabled: "true" + + # Indicates the replication mode + # Allowed values: + # "ASYNC" - Asynchronous mode + # "SYNC" - Synchronous mode + # "METRO" - Metro mode + # Default value: "ASYNC" + replication.storage.dell.com/mode: "METRO" + + # Indicates the remote PowerStore system to be used to configure Metro replication + # Allowed values: string + # Default value: None + replication.storage.dell.com/remoteSystem: "RT-0000" + + # Indicates the array ID to be used for provisioning the volume + # Allowed values: arrayID corresponding to array's globalID specified in secret.yaml + # Default value: None + arrayID: "Unique" + + # Indicates the file system type for mounted volumes + # Allowed values: + # ext3: ext3 filesystem type + # ext4: ext4 filesystem type + # xfs: XFS filesystem type + # nfs: NFS filesystem type + # Optional: true + # Default value: None if defaultFsType is not mentioned in values.yaml + # Else defaultFsType value mentioned in values.yaml will be used + csi.storage.k8s.io/fstype: "ext4" diff --git a/samples/storageclass/powerstore-nfs-replication.yaml b/samples/storageclass/powerstore-nfs-replication.yaml new file mode 100644 index 00000000..a568bdf3 --- /dev/null +++ b/samples/storageclass/powerstore-nfs-replication.yaml @@ -0,0 +1,100 @@ +# +# +# Copyright © 2021-2024 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: "powerstore-nfs-replication" +provisioner: "csi-powerstore.dellemc.com" +reclaimPolicy: Delete +volumeBindingMode: Immediate +parameters: + # replicationPrefix paramater in values.yaml must be used as prefix for all replication parameters in storage class + # for e.g., all replication parameters have prefix: replication.storage.dell.com here + + # replication.storage.dell.com/isReplicationEnabled: + # Allowed values: + # true: replication is enabled + # false: replication is disabled + # Optional: true + # Default value: false + replication.storage.dell.com/isReplicationEnabled: "true" + + # replication.storage.dell.com/mode: replication mode + # Allowed values: + # "ASYNC" - Asynchronous mode + # "SYNC" - Synchronous mode + # "METRO" - Metro mode + # Optional: true + # Default value: "ASYNC" + replication.storage.dell.com/mode: "ASYNC" + + # replication.storage.dell.com/remoteStorageClassName: + # Allowed values: string + # Optional: true + # Default value: None + replication.storage.dell.com/remoteStorageClassName: "powerstore-replication" + + # replication.storage.dell.com/remoteClusterID: point to correct remote cluster id + # Allowed values: string + # Optional: true + # Default value: None + replication.storage.dell.com/remoteClusterID: "tgt-cluster-id" + + # replication.storage.dell.com/remoteSystem: point to correct remote PowerStore system + # Allowed values: string + # Optional: true + # Default value: None + replication.storage.dell.com/remoteSystem: "RT-0000" + + # replication.storage.dell.com/rpo: change to any other RPOs supported by PowerStore + # Allowed values: "Five_Minutes", "Fifteen_Minutes", "Thirty_Minutes", "One_Hour", "Six_Hours", "Twelve_Hours", "One_Day","Zero" + # Optional: true + # Default value: None + # For SYNC replication, this value must be set to Zero + replication.storage.dell.com/rpo: Five_Minutes + + # replication.storage.dell.com/ignoreNamespaces: set to 'true' if you want to ignore namespaces and use one volume group + # Allowed values: + # true: ignore namespaces and use one volume group + # false: create separate volume group per namespace + # Optional: true + # Default value: None + replication.storage.dell.com/ignoreNamespaces: "false" + + # replication.storage.dell.com/volumeGroupPrefix: volume group prefix + # Allowed values: string + # Optional: true + # Default value: None + replication.storage.dell.com/volumeGroupPrefix: "csi" + + # arrayID: id of array to be used for volumes + # Allowed values: arrayID corresponding to array's globalID specified in secret.yaml + # Optional: false + # Default value: None + arrayID: "Unique" + + # FsType: file system type for mounted volumes + # Allowed values: + # ext3: ext3 filesystem type + # ext4: ext4 filesystem type + # xfs: XFS filesystem type + # nfs: NFS filesystem + # Optional: true + # Default value: None if defaultFsType is not mentioned in values.yaml + # Else defaultFsType value mentioned in values.yaml + # will be used as default value + csi.storage.k8s.io/fstype: "nfs" diff --git a/samples/storageclass/powerstore-nfs.yaml b/samples/storageclass/powerstore-nfs.yaml index b06d6ddf..d6626f88 100644 --- a/samples/storageclass/powerstore-nfs.yaml +++ b/samples/storageclass/powerstore-nfs.yaml @@ -38,10 +38,11 @@ parameters: # will be used as default value csi.storage.k8s.io/fstype: "nfs" # nasName: NAS server's name. If not specified, value from secret.yaml will be used + # User can specify one or multiple NAS servers, separated by commas. # Allowed values: string # Optional: true # Default value: None - nasName: "nas-server" + nasName: "nas-server1,nas-server2,nas-server3" # allowRoot: enables or disables root squashing (valid only for NFS) # Allowed values: diff --git a/samples/storageclass/powerstore-replication.yaml b/samples/storageclass/powerstore-replication.yaml index ce350c3a..5e2411d3 100644 --- a/samples/storageclass/powerstore-replication.yaml +++ b/samples/storageclass/powerstore-replication.yaml @@ -1,6 +1,6 @@ # # -# Copyright © 2021-2022 Dell Inc. or its subsidiaries. All Rights Reserved. +# Copyright © 2021-2024 Dell Inc. or its subsidiaries. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,28 +19,37 @@ kind: StorageClass metadata: name: "powerstore-replication" provisioner: "csi-powerstore.dellemc.com" -reclaimPolicy: Retain +reclaimPolicy: Delete volumeBindingMode: Immediate parameters: # replicationPrefix paramater in values.yaml must be used as prefix for all replication parameters in storage class # for e.g., all replication parameters have prefix: replication.storage.dell.com here - # replication.storage.dell.com/isReplicationEnabled: + # replication.storage.dell.com/isReplicationEnabled: # Allowed values: - # true: enable replication sidecar - # false: disable replication sidecar + # true: replication is enabled + # false: replication is disabled # Optional: true # Default value: false replication.storage.dell.com/isReplicationEnabled: "true" - # replication.storage.dell.com/remoteStorageClassName: + # replication.storage.dell.com/mode: replication mode + # Allowed values: + # "ASYNC" - Asynchronous mode + # "SYNC" - Synchronous mode + # "METRO" - Metro mode + # Optional: true + # Default value: "ASYNC" + replication.storage.dell.com/mode: "ASYNC" + + # replication.storage.dell.com/remoteStorageClassName: # Allowed values: string # Optional: true # Default value: None replication.storage.dell.com/remoteStorageClassName: "powerstore-replication" # replication.storage.dell.com/remoteClusterID: point to correct remote cluster id - # Allowed values: string + # Allowed values: string # Optional: true # Default value: None replication.storage.dell.com/remoteClusterID: "tgt-cluster-id" @@ -52,9 +61,10 @@ parameters: replication.storage.dell.com/remoteSystem: "RT-0000" # replication.storage.dell.com/rpo: change to any other RPOs supported by PowerStore - # Allowed values: "Five_Minutes", "Fifteen_Minutes", "Thirty_Minutes", "One_Hour", "Six_Hours", "Twelve_Hours", "One_Day" + # Allowed values: "Five_Minutes", "Fifteen_Minutes", "Thirty_Minutes", "One_Hour", "Six_Hours", "Twelve_Hours", "One_Day","Zero" # Optional: true # Default value: None + # For SYNC replication, this value must be set to Zero replication.storage.dell.com/rpo: Five_Minutes # replication.storage.dell.com/ignoreNamespaces: set to 'true' if you want to ignore namespaces and use one volume group @@ -64,7 +74,7 @@ parameters: # Optional: true # Default value: None replication.storage.dell.com/ignoreNamespaces: "false" - + # replication.storage.dell.com/volumeGroupPrefix: volume group prefix # Allowed values: string # Optional: true @@ -76,6 +86,7 @@ parameters: # Optional: false # Default value: None arrayID: "Unique" + # FsType: file system type for mounted volumes # Allowed values: # ext3: ext3 filesystem type diff --git a/samples/storageclass/powerstore-topology.yaml b/samples/storageclass/powerstore-topology.yaml index cee54b3e..cf586513 100644 --- a/samples/storageclass/powerstore-topology.yaml +++ b/samples/storageclass/powerstore-topology.yaml @@ -1,75 +1,75 @@ -# -# -# Copyright © 2021-2023 Dell Inc. or its subsidiaries. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# - -# topology can be used with any other SC as well -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: "powerstore-topology" -provisioner: "csi-powerstore.dellemc.com" -parameters: - # arrayID: id of array to be used for volumes - # Allowed values: arrayID corresponding to array's globalID specified in secret.yaml - # Optional: false - # Default value: None - arrayID: "Unique" - # FsType: file system type for mounted volumes - # Allowed values: - # ext3: ext3 filesystem type - # ext4: ext4 filesystem type - # xfs: XFS filesystem type - # nfs: NFS filesystem - # Optional: true - # Default value: None if defaultFsType is not mentioned in values.yaml - # Else defaultFsType value mentioned in values.yaml - # will be used as default value - csi.storage.k8s.io/fstype: "ext4" - -# reclaimPolicy: PVs that are dynamically created by a StorageClass will have the reclaim policy specified here -# Allowed values: -# Reclaim: retain the PV after PVC deletion -# Delete: delete the PV after PVC deletion -# Optional: true -# Default value: Delete -reclaimPolicy: Delete - -# allowVolumeExpansion: allows the users to resize the volume by editing the corresponding PVC object -# Allowed values: -# true: allow users to resize the PVC -# false: does not allow users to resize the PVC -# Optional: true -# Default value: false -allowVolumeExpansion: true - -# volumeBindingMode controls when volume binding and dynamic provisioning should occur. -# Allowed values: -# Immediate: indicates that volume binding and dynamic provisioning occurs once the -# PersistentVolumeClaim is created -# WaitForFirstConsumer: will delay the binding and provisioning of a PersistentVolume (must use this with topology) -# until a Pod using the PersistentVolumeClaim is created -# Optional: true -# Default value: Immediate -volumeBindingMode: WaitForFirstConsumer - -# allowedTopologies: helps scheduling pods on worker nodes which match all of below expressions. -# replace "-iscsi" with "-fc", "-nvmetcp", "-nvmefc" or "-nfs" at the end to use FC, NVMeTCP, NVMeFC, or NFS enabled hosts -# replace "12.34.56.78" with PowerStore endpoint IP or abc.com -# Optional: true -allowedTopologies: - - matchLabelExpressions: - - key: "csi-powerstore.dellemc.com/12.34.56.78-iscsi" - values: - - "true" +# +# +# Copyright © 2021-2023 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +# topology can be used with any other SC as well +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: "powerstore-topology" +provisioner: "csi-powerstore.dellemc.com" +parameters: + # arrayID: id of array to be used for volumes + # Allowed values: arrayID corresponding to array's globalID specified in secret.yaml + # Optional: false + # Default value: None + arrayID: "Unique" + # FsType: file system type for mounted volumes + # Allowed values: + # ext3: ext3 filesystem type + # ext4: ext4 filesystem type + # xfs: XFS filesystem type + # nfs: NFS filesystem + # Optional: true + # Default value: None if defaultFsType is not mentioned in values.yaml + # Else defaultFsType value mentioned in values.yaml + # will be used as default value + csi.storage.k8s.io/fstype: "ext4" + +# reclaimPolicy: PVs that are dynamically created by a StorageClass will have the reclaim policy specified here +# Allowed values: +# Reclaim: retain the PV after PVC deletion +# Delete: delete the PV after PVC deletion +# Optional: true +# Default value: Delete +reclaimPolicy: Delete + +# allowVolumeExpansion: allows the users to resize the volume by editing the corresponding PVC object +# Allowed values: +# true: allow users to resize the PVC +# false: does not allow users to resize the PVC +# Optional: true +# Default value: false +allowVolumeExpansion: true + +# volumeBindingMode controls when volume binding and dynamic provisioning should occur. +# Allowed values: +# Immediate: indicates that volume binding and dynamic provisioning occurs once the +# PersistentVolumeClaim is created +# WaitForFirstConsumer: will delay the binding and provisioning of a PersistentVolume (must use this with topology) +# until a Pod using the PersistentVolumeClaim is created +# Optional: true +# Default value: Immediate +volumeBindingMode: WaitForFirstConsumer + +# allowedTopologies: helps scheduling pods on worker nodes which match all of below expressions. +# replace "-iscsi" with "-fc", "-nvmetcp", "-nvmefc" or "-nfs" at the end to use FC, NVMeTCP, NVMeFC, or NFS enabled hosts +# replace "12.34.56.78" with PowerStore endpoint IP or abc.com +# Optional: true +allowedTopologies: + - matchLabelExpressions: + - key: "csi-powerstore.dellemc.com/12.34.56.78-iscsi" + values: + - "true" diff --git a/samples/volumesnapshotclass/snapclass.yaml b/samples/volumesnapshotclass/snapclass.yaml index 16bf147c..478d7f47 100644 --- a/samples/volumesnapshotclass/snapclass.yaml +++ b/samples/volumesnapshotclass/snapclass.yaml @@ -18,5 +18,5 @@ apiVersion: snapshot.storage.k8s.io/v1 kind: VolumeSnapshotClass metadata: name: powerstore-snapshot -driver: "csi-powerstore.dellemc.com" #driver name from values.yaml +driver: "csi-powerstore.dellemc.com" # driver name from values.yaml deletionPolicy: Delete diff --git a/tests/e2e/e2e-values.yaml b/tests/e2e/e2e-values.yaml deleted file mode 100644 index b9a6f144..00000000 --- a/tests/e2e/e2e-values.yaml +++ /dev/null @@ -1,34 +0,0 @@ -# -# -# Copyright © 2022 Dell Inc. or its subsidiaries. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# - -kubeconfigEnvVar : "KUBECONFIG" -busyBoxImageOnGcr : "gcr.io/google_containers/busybox:1.27" - -driverNamespace : "csi-powerstore" -e2eCSIDriverName : "csi-powerstore.dellemc.com" -arrayID: "unique" - -diskSize : "3Gi" -execCommand : "while true ; do sleep 2 ; done" - -# config for externalAccess e2e test suite -externalAccess: - endPoint: "https://10.0.0.1/api/rest" # array's endpoint - userName: "user" - password: "password" - externalAccessIP: "10.0.0.1/25" # IP configured in values.yaml file while installing the driver - NASServer: "nas-server" - testStatefulset: true # if wanted to test ExternalAccess with Deployment as well as with Statefulset resource \ No newline at end of file diff --git a/tests/e2e/externalAccess.go b/tests/e2e/externalAccess.go deleted file mode 100644 index 77096ba7..00000000 --- a/tests/e2e/externalAccess.go +++ /dev/null @@ -1,304 +0,0 @@ -/* - * - * Copyright © 2022-2023 Dell Inc. or its subsidiaries. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package e2etest - -import ( - "context" - "crypto/rand" - "fmt" - "math/big" - "strconv" - "time" - - "github.com/dell/csi-powerstore/v2/pkg/common" - "github.com/dell/gopowerstore" - "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" - - v1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - - storagev1 "k8s.io/api/storage/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/kubernetes/test/e2e/framework" - "k8s.io/kubernetes/test/e2e/framework/deployment" - fpv "k8s.io/kubernetes/test/e2e/framework/pv" - fss "k8s.io/kubernetes/test/e2e/framework/statefulset" -) - -/* - - Create NFS SC. - Create PVC using above SC. - Deployment having 10 replicas using above PVC. - Scale down to 1 pod. - Check if externalAccess is present in the NFS Export after scaling down to 1 pod - Scale down to 0 pods. - Cleanup the NS, i.e. SC, PVC, Pod. - - -*/ - -var ( - testParameters map[interface{}]interface{} - deploymentObject *v1.Deployment - mountPath = "/data" - extCredential ExternaAccess -) - -// ExternaAccess for storing ExternalAccess credentials -type ExternaAccess struct { - EndPoint string - UserName string - Password string - ExternalAccessIP string - NASServer string - TestStatefulset bool -} - -var _ = ginkgo.Describe("External Access Test", func() { - - // Building a namespace api object, basename external-access - var ( - namespace string - client clientset.Interface - ) - - framework.TestContext.VerifyServiceAccount = false - f := framework.NewDefaultFramework("external-access") - // prevent annoying psp warning - - //f.SkipPrivilegedPSPBinding = true - defer ginkgo.GinkgoRecover() - framework.Logf("run e2e test default timeouts %#v ", f.Timeouts) - ginkgo.BeforeEach(func() { - namespace = getNamespaceToRunTests(f) - client = f.ClientSet - bootstrap() - }) - - ginkgo.AfterEach(func() { - _, cancel := context.WithCancel(context.Background()) - defer cancel() - DeleteDeployment(client, deploymentObject, namespace) - if extCredential.TestStatefulset { - ginkgo.By(fmt.Sprintf("Deleting all statefulsets in namespace: %v", namespace)) - fss.DeleteAllStatefulSets(client, namespace) - } - }) - - // in case you want to log and exit framework.Fail("stop test") - - // Test for external Access feature check - ginkgo.It("[csi-externalAccess] Verify Host Access List for exteral access", func() { - curtime := time.Now().Unix() - nBig, err := rand.Int(rand.Reader, big.NewInt(27)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - randomValue := nBig.Int64() - val := strconv.FormatInt(int64(randomValue), 10) - curtimestring := strconv.FormatInt(curtime, 10) - scName := "exteral-access-sc-" + curtimestring + val - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - ginkgo.By("Creating NFS Storage Class & PVC") - - // get array credential from config file - extCredential = getExternalAccessCredential(testParameters["externalAccess"]) - - testParameters["allowRoot"] = "true" - testParameters["csi.storage.k8s.io/fstype"] = "nfs" - testParameters["nasName"] = extCredential.NASServer - - ds := fmt.Sprintf("%v", testParameters["diskSize"]) - storageclasspvc, pvclaim, err := createPVCAndStorageClass(client, - namespace, nil, testParameters, ds, nil, storagev1.VolumeBindingImmediate, true, "ReadWriteMany", scName) - - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - defer func() { - err = client.StorageV1().StorageClasses().Delete(ctx, storageclasspvc.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Expect PVC's claim status to be in Bound state") - err = fpv.WaitForPersistentVolumeClaimPhase(corev1.ClaimBound, client, - pvclaim.Namespace, pvclaim.Name, framework.Poll, 2*time.Minute) - - gomega.Expect(err).NotTo(gomega.HaveOccurred(), - fmt.Sprintf("Failed to find the volume in bound state with err: %v", err)) - - podLabels := map[string]string{ - "app": "dell-wip", - } - // Deployment is a resource to deploy a stateless application, if using a PVC, all replicas will be using the same Volume - ginkgo.By("Creating Deployment having 10 replicas") - deploymentObject, err = deployment.CreateDeployment(client, 10, podLabels, nil, namespace, []*corev1.PersistentVolumeClaim{pvclaim}, fmt.Sprintf("%v", testParameters["execCommand"])) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), - fmt.Sprintf("Failed to create deployment resource with err: %v", err)) - - ginkgo.By("Deployment got created") - // now get the pvc, more intrested in volumeId - v := getPvFromClaim(client, namespace, pvclaim.Name) - framework.Logf("Volume Name that got attached to all pods: %s", v.GetName()) - - ginkgo.By("Scaling down to 1 replica") - ScaleDownDeployment(client, deploymentObject, namespace, 1) - - // fetch the NFS export object from array to avoid conflict in the response - // Get host access list from array for above volume - clientOptions := gopowerstore.NewClientOptions() - clientOptions.SetInsecure(true) - clientForArray, err := gopowerstore.NewClientWithArgs( - extCredential.EndPoint, extCredential.UserName, extCredential.Password, clientOptions) - - gomega.Expect(err).NotTo(gomega.HaveOccurred(), - fmt.Sprintf("Failed to connect with PowerStore Array, err: %v", err)) - checkExternalAccessPresence(ctx, clientForArray, extCredential.ExternalAccessIP, v.GetName(), true) - - // now in NFS Export only externalIP will be present and other node's IP will be deleted - ScaleDownDeployment(client, deploymentObject, namespace, 0) - checkExternalAccessPresence(ctx, clientForArray, extCredential.ExternalAccessIP, v.GetName(), true) - - err = fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), - fmt.Sprintf("Unable to delete PVC with err: %v", err)) - - if extCredential.TestStatefulset { - // statefulset logic - scName2 := scName + "stateful-set" - sc, err := createStorageClass(client, testParameters, nil, "", - storagev1.VolumeBindingImmediate, - true, scName2) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - defer func() { - err := client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - statefulset := GetStatefulSetFromManifest(namespace) - ginkgo.By("Creating statefulset") - - statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1].Spec.AccessModes[0] = - corev1.ReadWriteMany - - statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Annotations["volume.beta.kubernetes.io/storage-class"] = scName2 - - CreateStatefulSet(namespace, statefulset, client) - - defer func() { - ginkgo.By(fmt.Sprintf("Deleting all statefulsets in namespace: %v", namespace)) - fss.DeleteAllStatefulSets(client, namespace) - }() - replicas := *(statefulset.Spec.Replicas) - // Waiting for pods status to be Ready - fss.WaitForStatusReadyReplicas(client, statefulset, replicas) - - gomega.Expect(fss.CheckMount(client, statefulset, mountPath)).NotTo(gomega.HaveOccurred()) - - ssPodsBeforeScaleDown := fss.GetPodList(client, statefulset) - gomega.Expect(ssPodsBeforeScaleDown.Items).NotTo(gomega.BeEmpty(), - fmt.Sprintf("Unable to get list of Pods from the Statefulset: %v", statefulset.Name)) - gomega.Expect(len(ssPodsBeforeScaleDown.Items) == int(replicas)).To(gomega.BeTrue(), - "Number of Pods in the statefulset should match with number of replicas") - - // Get the list of Volumes attached to Pods before scale down - for _, sspod := range ssPodsBeforeScaleDown.Items { - _, err := client.CoreV1().Pods(namespace).Get(ctx, sspod.Name, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - for _, volumespec := range sspod.Spec.Volumes { - if volumespec.PersistentVolumeClaim != nil { - pv := getPvFromClaim(client, statefulset.Namespace, volumespec.PersistentVolumeClaim.ClaimName) - gomega.Expect(pv).NotTo(gomega.BeNil()) - } - } - } - - // 1 - replicas = 1 - ginkgo.By(fmt.Sprintf("Scaling down statefulsets to number of Replica: %v", replicas)) - _, scaledownErr := fss.Scale(client, statefulset, replicas) - gomega.Expect(scaledownErr).NotTo(gomega.HaveOccurred()) - fss.WaitForStatusReplicas(client, statefulset, replicas) - ssPodsAfterScaleDown := fss.GetPodList(client, statefulset) - gomega.Expect(ssPodsAfterScaleDown.Items).NotTo(gomega.BeEmpty(), - fmt.Sprintf("Unable to get list of Pods from the Statefulset: %v", statefulset.Name)) - gomega.Expect(len(ssPodsAfterScaleDown.Items) == int(replicas)).To(gomega.BeTrue(), - "Number of Pods in the statefulset should match with number of replicas") - - var pv *corev1.PersistentVolume - // Get the list of Volumes attached to Pods after scale down - // for us one iteration is also okay to have since PV Id/Name will be same. - for _, sspod := range ssPodsAfterScaleDown.Items { - _, err := client.CoreV1().Pods(namespace).Get(ctx, sspod.Name, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - for _, volumespec := range sspod.Spec.Volumes { - if volumespec.PersistentVolumeClaim != nil { - pv = getPvFromClaim(client, statefulset.Namespace, volumespec.PersistentVolumeClaim.ClaimName) - gomega.Expect(pv).NotTo(gomega.BeNil()) - } - } - } - - checkExternalAccessPresence(ctx, clientForArray, extCredential.ExternalAccessIP, pv.GetName(), true) - - // deleting all pods - replicas = 0 - ginkgo.By(fmt.Sprintf("Scaling down statefulsets to number of Replica: %v", replicas)) - _, scaledownErr = fss.Scale(client, statefulset, replicas) - gomega.Expect(scaledownErr).NotTo(gomega.HaveOccurred()) - fss.WaitForStatusReplicas(client, statefulset, replicas) - ssPodsAfterScaleDown = fss.GetPodList(client, statefulset) - gomega.Expect(len(ssPodsAfterScaleDown.Items) == int(replicas)).To(gomega.BeTrue(), - "Number of Pods in the statefulset should match with number of replicas") - - err = fpv.DeletePersistentVolumeClaim(client, pv.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), - fmt.Sprintf("Unable to delete PVC with err: %v", err)) - } - }) -}) - -func getExternalAccessCredential(credential interface{}) (ext ExternaAccess) { - myMap := credential.(map[interface{}]interface{}) - ext.EndPoint = fmt.Sprintf("%v", myMap["endPoint"]) - ext.UserName = fmt.Sprintf("%v", myMap["userName"]) - ext.Password = fmt.Sprintf("%v", myMap["password"]) - ext.NASServer = fmt.Sprintf("%v", myMap["NASServer"]) - ext.ExternalAccessIP = fmt.Sprintf("%v", myMap["externalAccessIP"]) - ext.TestStatefulset, _ = strconv.ParseBool(fmt.Sprintf("%v", myMap["testStatefulset"])) - return ext -} - -func checkExternalAccessPresence(ctx context.Context, clientForArray gopowerstore.Client, externalAccessIP string, vol string, shouldBePresent bool) { - nfsExport, err := clientForArray.GetNFSExportByName(ctx, vol) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), - fmt.Sprintf("Failed to GET NFS export details from Array, err: %v", err)) - present := common.ExternalAccessAlreadyAdded(nfsExport, externalAccessIP) - if shouldBePresent && !present { - gomega.Expect(present).NotTo(gomega.BeFalse(), - "External access should be present on host access list on array") - } - if !shouldBePresent && present { - gomega.Expect(present).NotTo(gomega.BeTrue(), - "External access should not be present on host access list on array") - } -} diff --git a/tests/e2e/go.mod b/tests/e2e/go.mod deleted file mode 100644 index 09210a36..00000000 --- a/tests/e2e/go.mod +++ /dev/null @@ -1,191 +0,0 @@ -module github.com/dell/csi-powerstore/v2/tests/e2e - -go 1.20 - -require ( - github.com/onsi/gomega v1.27.4 - k8s.io/kubernetes v1.26.0-beta.0 - //sigs.k8s.io/vsphere-csi-driver/v2 v2.5.1 - sigs.k8s.io/yaml v1.3.0 // indirect -) - -require ( - github.com/dell/csi-powerstore/v2 v2.6.0 - github.com/dell/gopowerstore v1.11.0 - github.com/onsi/ginkgo/v2 v2.9.2 - gopkg.in/yaml.v2 v2.4.0 - k8s.io/api v0.26.3 - k8s.io/apimachinery v0.26.3 - k8s.io/client-go v0.26.3 -) - -require ( - cloud.google.com/go v0.97.0 // indirect - github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab // indirect - github.com/Microsoft/go-winio v0.4.17 // indirect - github.com/Microsoft/hcsshim v0.8.22 // indirect - github.com/NYTimes/gziphandler v1.1.1 // indirect - github.com/apparentlymart/go-cidr v1.1.0 // indirect - github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e // indirect - github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect - github.com/aws/aws-sdk-go v1.44.116 // indirect - github.com/beorn7/perks v1.0.1 // indirect - github.com/blang/semver/v4 v4.0.0 // indirect - github.com/cenkalti/backoff/v4 v4.1.3 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect - github.com/checkpoint-restore/go-criu/v5 v5.3.0 // indirect - github.com/cilium/ebpf v0.7.0 // indirect - github.com/container-storage-interface/spec v1.7.0 // indirect - github.com/containerd/cgroups v1.0.1 // indirect - github.com/containerd/console v1.0.3 // indirect - github.com/containerd/ttrpc v1.1.0 // indirect - github.com/coreos/go-systemd/v22 v22.3.2 // indirect - github.com/cyphar/filepath-securejoin v0.2.3 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/dell/gobrick v1.7.0 // indirect - github.com/dell/gocsi v1.7.0 // indirect - github.com/dell/gofsutil v1.12.0 // indirect - github.com/dell/goiscsi v1.7.0 // indirect - github.com/dell/gonvme v1.4.0 // indirect - github.com/docker/distribution v2.8.2+incompatible // indirect - github.com/docker/go-units v0.5.0 // indirect - github.com/emicklei/go-restful/v3 v3.10.0 // indirect - github.com/euank/go-kmsg-parser v2.0.0+incompatible // indirect - github.com/evanphx/json-patch v5.6.0+incompatible // indirect - github.com/felixge/httpsnoop v1.0.3 // indirect - github.com/fsnotify/fsnotify v1.6.0 // indirect - github.com/go-logr/logr v1.2.3 // indirect - github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-openapi/errors v0.20.2 // indirect - github.com/go-openapi/jsonpointer v0.19.5 // indirect - github.com/go-openapi/jsonreference v0.20.0 // indirect - github.com/go-openapi/strfmt v0.21.3 // indirect - github.com/go-openapi/swag v0.19.14 // indirect - github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect - github.com/godbus/dbus/v5 v5.0.6 // indirect - github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/mock v1.6.0 // indirect - github.com/golang/protobuf v1.5.3 // indirect - github.com/google/cadvisor v0.46.0 // indirect - github.com/google/gnostic v0.5.7-v3refs // indirect - github.com/google/go-cmp v0.5.9 // indirect - github.com/google/gofuzz v1.2.0 // indirect - github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect - github.com/google/uuid v1.3.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect - github.com/imdario/mergo v0.3.12 // indirect - github.com/inconshreveable/mousetrap v1.0.1 // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect - github.com/josharian/intern v1.0.0 // indirect - github.com/json-iterator/go v1.1.12 // indirect - github.com/karrick/godirwalk v1.17.0 // indirect - github.com/mailru/easyjson v0.7.6 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect - github.com/mindprince/gonvml v0.0.0-20190828220739-9ebdce4bb989 // indirect - github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/moby/spdystream v0.2.0 // indirect - github.com/moby/sys/mountinfo v0.6.2 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/mrunalp/fileutils v0.5.0 // indirect - github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect - github.com/oklog/ulid v1.3.1 // indirect - github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/runc v1.1.5 // indirect - github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 // indirect - github.com/opencontainers/selinux v1.10.0 // indirect - github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_golang v1.14.0 // indirect - github.com/prometheus/client_model v0.3.0 // indirect - github.com/prometheus/common v0.37.0 // indirect - github.com/prometheus/procfs v0.8.0 // indirect - github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646 // indirect - github.com/sirupsen/logrus v1.9.0 // indirect - github.com/spf13/cobra v1.6.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect - github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect - github.com/vishvananda/netlink v1.1.0 // indirect - github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae // indirect - go.mongodb.org/mongo-driver v1.11.2 // indirect - go.opencensus.io v0.23.0 // indirect - go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.35.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.0 // indirect - go.opentelemetry.io/otel v1.10.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0 // indirect - go.opentelemetry.io/otel/metric v0.31.0 // indirect - go.opentelemetry.io/otel/sdk v1.10.0 // indirect - go.opentelemetry.io/otel/trace v1.10.0 // indirect - go.opentelemetry.io/proto/otlp v0.19.0 // indirect - golang.org/x/crypto v0.1.0 // indirect - golang.org/x/net v0.8.0 // indirect - golang.org/x/oauth2 v0.0.0-20220524215830-622c5d57e401 // indirect - golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.6.0 // indirect - golang.org/x/term v0.6.0 // indirect - golang.org/x/text v0.8.0 // indirect - golang.org/x/time v0.0.0-20220411224347-583f2d630306 // indirect - golang.org/x/tools v0.7.0 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20220601144221-27df5f98adab // indirect - google.golang.org/grpc v1.49.0 // indirect - google.golang.org/protobuf v1.28.1 // indirect - gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.0.0 // indirect - k8s.io/apiserver v0.26.1 // indirect - k8s.io/cloud-provider v0.0.0 // indirect - k8s.io/component-base v0.26.1 // indirect - k8s.io/component-helpers v0.26.1 // indirect - k8s.io/cri-api v0.0.0 // indirect - k8s.io/csi-translation-lib v0.0.0 // indirect - k8s.io/klog/v2 v2.80.1 // indirect - k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect - k8s.io/kube-scheduler v0.0.0 // indirect - k8s.io/kubectl v0.0.0 // indirect - k8s.io/kubelet v0.26.3 // indirect - k8s.io/mount-utils v0.0.0 // indirect - k8s.io/pod-security-admission v0.0.0 // indirect - k8s.io/utils v0.0.0-20230202215443-34013725500c // indirect - sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.36 // indirect - sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect -) - -replace ( - github.com/dell/csi-powerstore/v2 => ../../ - k8s.io/api => k8s.io/api v0.26.1 - k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.26.1 - k8s.io/apimachinery => k8s.io/apimachinery v0.26.1 - k8s.io/apiserver => k8s.io/apiserver v0.26.1 - k8s.io/cli-runtime => k8s.io/cli-runtime v0.26.1 - k8s.io/client-go => k8s.io/client-go v0.26.1 - k8s.io/cloud-provider => k8s.io/cloud-provider v0.26.1 - k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.26.1 - k8s.io/code-generator => k8s.io/code-generator v0.26.1 - k8s.io/component-base => k8s.io/component-base v0.26.1 - k8s.io/component-helpers => k8s.io/component-helpers v0.26.1 - k8s.io/controller-manager => k8s.io/controller-manager v0.26.1 - k8s.io/cri-api => k8s.io/cri-api v0.26.1 - k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.26.1 - k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.26.1 - k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.26.1 - k8s.io/kube-proxy => k8s.io/kube-proxy v0.26.1 - k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.26.1 - k8s.io/kubectl => k8s.io/kubectl v0.26.1 - k8s.io/kubelet => k8s.io/kubelet v0.26.1 - k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.26.1 - k8s.io/metrics => k8s.io/metrics v0.26.1 - k8s.io/mount-utils => k8s.io/mount-utils v0.26.1 - k8s.io/node-api => k8s.io/node-api v0.26.1 - k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.26.1 - k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.26.1 - k8s.io/sample-cli-plugin => k8s.io/sample-cli-plugin v0.26.1 - k8s.io/sample-controller => k8s.io/sample-controller v0.26.1 - sigs.k8s.io/controller-runtime => sigs.k8s.io/controller-runtime v0.11.1 -) diff --git a/tests/e2e/go.sum b/tests/e2e/go.sum deleted file mode 100644 index 46baf119..00000000 --- a/tests/e2e/go.sum +++ /dev/null @@ -1,1131 +0,0 @@ -bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= -cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= -cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= -cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= -cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= -cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= -cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= -cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= -cloud.google.com/go v0.97.0 h1:3DXvAyifywvq64LfkKaMOmkWPS1CikIQdMe2lY9vxU8= -cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/azure-sdk-for-go v55.0.0+incompatible h1:L4/vUGbg1Xkw5L20LZD+hJI5I+ibWSytqQ68lTCfLwY= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= -github.com/Azure/go-autorest/autorest v0.11.27 h1:F3R3q42aWytozkV8ihzcgMO4OA4cuqr3bNlsEuF6//A= -github.com/Azure/go-autorest/autorest/adal v0.9.20 h1:gJ3E98kMpFB1MFqQCvA1yFab8vthOeD4VlFRQULxahg= -github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= -github.com/Azure/go-autorest/autorest/mocks v0.4.2 h1:PGN4EDXnuQbojHbU0UWoNvmu9AGVwYHG9/fkDYhtAfw= -github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk= -github.com/Azure/go-autorest/autorest/validation v0.1.0 h1:ISSNzGUh+ZSzizJWOWzs8bwpXIePbGLW4z/AmUFGH5A= -github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= -github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/GoogleCloudPlatform/k8s-cloud-provider v1.18.1-0.20220218231025-f11817397a1b h1:Heo1J/ttaQFgGJSVnCZquy3e5eH5j1nqxBuomztB3P0= -github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab h1:UKkYhof1njT1/xq4SEg5z+VpTgjmNeHwPGRQl7takDI= -github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab/go.mod h1:3VYc5hodBMJ5+l/7J4xAyMeuM2PNuepvHlGs8yilUCA= -github.com/Microsoft/go-winio v0.4.15/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= -github.com/Microsoft/go-winio v0.4.17 h1:iT12IBVClFevaf8PuVyi3UmZOVh4OqnaLxDTW2O6j3w= -github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/Microsoft/hcsshim v0.8.22 h1:CulZ3GW8sNJExknToo+RWD+U+6ZM5kkNfuxywSDPd08= -github.com/Microsoft/hcsshim v0.8.22/go.mod h1:91uVCVzvX2QD16sMCenoxxXo6L1wJnLMX2PSufFMtF0= -github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= -github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/apparentlymart/go-cidr v1.1.0 h1:2mAhrMoF+nhXqxTzSZMUzDHkLjmIHC+Zzn4tdgBZjnU= -github.com/apparentlymart/go-cidr v1.1.0/go.mod h1:EBcsNrHc3zQeuaeCeCtQruQm+n9/YjEn/vI25Lg7Gwc= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e h1:QEF07wC0T1rKkctt1RINW/+RMTVmiwxETico2l3gxJA= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= -github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= -github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= -github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= -github.com/aws/aws-sdk-go v1.35.24/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k= -github.com/aws/aws-sdk-go v1.44.116 h1:NpLIhcvLWXJZAEwvPj3TDHeqp7DleK6ZUVYyW01WNHY= -github.com/aws/aws-sdk-go v1.44.116/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= -github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= -github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= -github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= -github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/checkpoint-restore/go-criu/v5 v5.3.0 h1:wpFFOoomK3389ue2lAb0Boag6XPht5QYpipxmSNL4d8= -github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= -github.com/cilium/ebpf v0.7.0 h1:1k/q3ATgxSXRdrmPfH8d7YK0GfqVsEKZAX9dQZvs56k= -github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/container-storage-interface/spec v1.7.0 h1:gW8eyFQUZWWrMWa8p1seJ28gwDoN5CVJ4uAbQ+Hdycw= -github.com/container-storage-interface/spec v1.7.0/go.mod h1:JYuzLqr9VVNoDJl44xp/8fmCOvWPDKzuGTwCoklhuqk= -github.com/containerd/cgroups v1.0.1 h1:iJnMvco9XGvKUvNQkv88bE4uJXxRQH18efbKo9w5vHQ= -github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU= -github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw= -github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ= -github.com/containerd/console v1.0.3 h1:lIr7SlA5PxZyMV30bDW0MGbiOPXwc63yRuCP0ARubLw= -github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= -github.com/containerd/containerd v1.4.9/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM= -github.com/containerd/fifo v1.0.0/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= -github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= -github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= -github.com/containerd/ttrpc v1.1.0 h1:GbtyLRxb0gOLR0TYQWt3O6B0NvT8tMdorEHqIQo/lWI= -github.com/containerd/ttrpc v1.1.0/go.mod h1:XX4ZTnoOId4HklF4edwc4DcqskFZuvXB1Evzy5KFQpQ= -github.com/containerd/typeurl v1.0.2 h1:Chlt8zIieDbzQFzXzAeBEF92KhExuE4p9p92/QmY7aY= -github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= -github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/cyphar/filepath-securejoin v0.2.3 h1:YX6ebbZCZP7VkM3scTTokDgBL2TY741X51MTk3ycuNI= -github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dell/dell-csi-extensions/common v1.1.1 h1:PXdqCm6lL1g02KuR9SrsrZeoTk6jZoyPp56s3ve+8DE= -github.com/dell/dell-csi-extensions/podmon v1.1.2 h1:U0p2IS3PA/GPXYwtmY3bM+xfhj7hPY6nPkRAyFh0slw= -github.com/dell/dell-csi-extensions/replication v1.3.1-0.20230316192210-d1b46db0cbe0 h1:8puwDAHQI+SI9KoKigJARxmwRE1tk40zRFekt6WQQ/o= -github.com/dell/dell-csi-extensions/volumeGroupSnapshot v1.2.2 h1:g3HZyuXgCiHpCkkVuCNKXFRETjvOkO6/16vrAH5ls90= -github.com/dell/gobrick v1.7.0 h1:XM72GhtyIKpBuj/yjTr3N04uWLFNMyDjXjDGjgCmtno= -github.com/dell/gobrick v1.7.0/go.mod h1:fUM/uqSvswD3368dEs+iz2wtp/pwlylN4oVQMcJm3jA= -github.com/dell/gocsi v1.7.0 h1:fMQO2zwAXCaIsUoPCcnnuPMwfQMoaI1/0aqkQVndlxU= -github.com/dell/gocsi v1.7.0/go.mod h1:X/8Ll8qqKAKCenmd1gPJMUvUmgY8cK0LiS8Pck12UaU= -github.com/dell/gofsutil v1.12.0 h1:oo2YHfGFKHvHS1urtqjOIKpaHwcdyqacwKHLXzUg33M= -github.com/dell/gofsutil v1.12.0/go.mod h1:mGMN5grVDtHv2imNw5+gFr2RmCqeyYgBFBldUbHtV78= -github.com/dell/goiscsi v1.7.0 h1:LL/6v7uaN48Lc8d3G8t/tLZI+EuJGBRQ7NZLgJrg3RQ= -github.com/dell/goiscsi v1.7.0/go.mod h1:l3PNZbHbYKDg50e5kdJWrf5+lOxcM5+8PjMr9HibvLs= -github.com/dell/gonvme v1.4.0 h1:SK94ETt0pYZbaKkRJOcq81TbrzC38ufBX+w4uKwJnks= -github.com/dell/gonvme v1.4.0/go.mod h1:fIu54BDTyIu8JOTXo6Q0BqMF1tOjpO+wKXVXjLReR2o= -github.com/dell/gopowerstore v1.11.0 h1:h4Opj5zDho3RnhqS/kgyeFOS+Y8+mE1OutnTDybgEhQ= -github.com/dell/gopowerstore v1.11.0/go.mod h1:d5OP8ZaRcZ65GTkUuZHLGa0IFgRdA3vF4i6JccfHBYE= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= -github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v20.10.18+incompatible h1:SN84VYXTBNGn92T/QwIRPlum9zfemfitN7pbsp26WSc= -github.com/docker/docker v20.10.18+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= -github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= -github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= -github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= -github.com/emicklei/go-restful/v3 v3.10.0 h1:X4gma4HM7hFm6WMeAsTfqA0GOfdNoCzBIkHGoRLGXuM= -github.com/emicklei/go-restful/v3 v3.10.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= -github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/euank/go-kmsg-parser v2.0.0+incompatible h1:cHD53+PLQuuQyLZeriD1V/esuG4MuU0Pjs5y6iknohY= -github.com/euank/go-kmsg-parser v2.0.0+incompatible/go.mod h1:MhmAMZ8V4CYH4ybgdRwPr2TU5ThnS43puaKEMpja1uw= -github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= -github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= -github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY= -github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= -github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= -github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-openapi/errors v0.20.2 h1:dxy7PGTqEh94zj2E3h1cUmQQWiM1+aeCROfAr02EmK8= -github.com/go-openapi/errors v0.20.2/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= -github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= -github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA= -github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= -github.com/go-openapi/strfmt v0.21.3 h1:xwhj5X6CjXEZZHMWy1zKJxvW9AfHC9pkyUjLvHtKG7o= -github.com/go-openapi/strfmt v0.21.3/go.mod h1:k+RzNO0Da+k3FrrynSNN8F7n/peCmQQqbbXjtDfvmGg= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= -github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= -github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/godbus/dbus/v5 v5.0.6 h1:mkgN1ofwASrYnJ5W6U/BxG15eXXXjirgZc7CLqkcaro= -github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= -github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6xDCrzib4= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v4 v4.2.0 h1:besgBTC8w8HjP6NzQdxwKH9Z5oQMZ24ThTrHp3cZ8eU= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= -github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= -github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= -github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/cadvisor v0.46.0 h1:ryTIniqhN8/wR8UA1RuYSXHvsAtdpk/01XwTZtYHekY= -github.com/google/cadvisor v0.46.0/go.mod h1:YnCDnR8amaS0HoMEjheOI0TMPzFKCBLc30mciLEjwGI= -github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= -github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= -github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= -github.com/googleapis/gax-go/v2 v2.1.1 h1:dp3bWCh+PPO1zjRRiCSczJav13sBvG4UhNyVTa1KqdU= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QGdLI4y34qQGjGWO0= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= -github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= -github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/jarcoal/httpmock v1.2.0 h1:gSvTxxFR/MEMfsGrvRbdfpRUMBStovlSRLw0Ep1bwwc= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/karrick/godirwalk v1.17.0 h1:b4kY7nqDdioR/6qnbHQyDvmA17u5G1cZ6J+CZXwSWoI= -github.com/karrick/godirwalk v1.17.0/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= -github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.2 h1:hAHbPm5IJGijwng3PWk09JkG9WeqChjprR5s9bBZ+OM= -github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/mindprince/gonvml v0.0.0-20190828220739-9ebdce4bb989 h1:PS1dLCGtD8bb9RPKJrc8bS7qHL6JnW1CZvwzH9dPoUs= -github.com/mindprince/gonvml v0.0.0-20190828220739-9ebdce4bb989/go.mod h1:2eu9pRWp8mo84xCg6KswZ+USQHjwgRhNp06sozOdsTY= -github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible h1:aKW/4cBs+yK6gpqU3K/oIwk9Q/XICqd3zOX/UFuvqmk= -github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= -github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= -github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= -github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78= -github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= -github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/mrunalp/fileutils v0.5.0 h1:NKzVxiH7eSk+OQ4M+ZYW1K6h27RUV3MI6NUTsHhU6Z4= -github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= -github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= -github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/ginkgo/v2 v2.9.2 h1:BA2GMJOtfGAfagzYtrAlufIP0lq6QERkFmHLMLPwFSU= -github.com/onsi/ginkgo/v2 v2.9.2/go.mod h1:WHcJJG2dIlcCqVfBAwUCrJxSPFb6v4azBwgxeMeDuts= -github.com/onsi/gomega v1.27.4 h1:Z2AnStgsdSayCMDiCU42qIz+HLqEPcgiOCXjAU/w+8E= -github.com/onsi/gomega v1.27.4/go.mod h1:riYq/GJKh8hhoM01HN6Vmuy93AarCXCBGpvFDK3q3fQ= -github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= -github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= -github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/runc v1.1.4/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg= -github.com/opencontainers/runc v1.1.5 h1:L44KXEpKmfWDcS02aeGm8QNTFXTo2D+8MYGDIJ/GDEs= -github.com/opencontainers/runc v1.1.5/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg= -github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 h1:3snG66yBm59tKhhSPQrQ/0bCrv1LQbKt40LnUPiUxdc= -github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/selinux v1.10.0 h1:rAiKF8hTcgLI3w0DHm6i0ylVVcOrlgR1kK99DRLDhyU= -github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= -github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= -github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= -github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= -github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= -github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= -github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rubiojr/go-vhd v0.0.0-20200706105327-02e210299021 h1:if3/24+h9Sq6eDx8UUz1SO9cT9tizyIsATfB7b4D3tc= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646 h1:RpforrEYXWkmGwJHIGnLZ3tTWStkjVVstwzNGqxX2Ds= -github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= -github.com/spf13/cobra v1.6.0 h1:42a0n6jwCot1pUmomAp4T7DeMD+20LFv4Q54pxLf2LI= -github.com/spf13/cobra v1.6.0/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI= -github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= -github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= -github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o= -github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0= -github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= -github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= -github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae h1:4hwBBUfQCFe3Cym0ZtKyq7L16eZUtYKs+BaHDN6mAns= -github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= -github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= -github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= -github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.mongodb.org/mongo-driver v1.10.0/go.mod h1:wsihk0Kdgv8Kqu1Anit4sfK+22vSFbUrAVEYRhCXrA8= -go.mongodb.org/mongo-driver v1.11.2 h1:+1v2rDQUWNcGW7/7E0Jvdz51V38XXxJfhzbV17aNHCw= -go.mongodb.org/mongo-driver v1.11.2/go.mod h1:s7p5vEtfbeR1gYi6pnj3c3/urpbLv2T5Sfd6Rp2HBB8= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.35.0 h1:KQjX0qQ8H21oBUAvFp4ZLKJMMLIluONvSPDAFIGmX58= -go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.35.0/go.mod h1:DQYkU9srMFqLUTVA/7/WlRHdnYDB7wyMMlle2ktMjfI= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0 h1:xFSRQBbXF6VvYRf2lqMJXxoB72XI1K/azav8TekHHSw= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0/go.mod h1:h8TWwRAhQpOd0aM5nYsRD8+flnkj+526GEIVlarH7eY= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.0 h1:Ajldaqhxqw/gNzQA45IKFWLdG7jZuXX/wBW1d5qvbUI= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.0/go.mod h1:9NiG9I2aHTKkcxqCILhjtyNA1QEiCjdBACv4IvrFQ+c= -go.opentelemetry.io/contrib/propagators/b3 v1.10.0 h1:6AD2VV8edRdEYNaD8cNckpzgdMLU2kbV9OYyxt2kvCg= -go.opentelemetry.io/otel v1.10.0 h1:Y7DTJMR6zs1xkS/upamJYk0SxxN4C9AqRd77jmZnyY4= -go.opentelemetry.io/otel v1.10.0/go.mod h1:NbvWjCthWHKBEUMpf0/v8ZRZlni86PpGFEMA9pnQSnQ= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0 h1:TaB+1rQhddO1sF71MpZOZAuSPW1klK2M8XxfrBMfK7Y= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0/go.mod h1:78XhIg8Ht9vR4tbLNUhXsiOnE2HOuSeKAiAcoVQEpOY= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0 h1:pDDYmo0QadUPal5fwXoY1pmMpFcdyhXOmL5drCrI3vU= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0/go.mod h1:Krqnjl22jUJ0HgMzw5eveuCvFDXY4nSYb4F8t5gdrag= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0 h1:KtiUEhQmj/Pa874bVYKGNVdq8NPKiacPbaRRtgXi+t4= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0/go.mod h1:OfUCyyIiDvNXHWpcWgbF+MWvqPZiNa3YDEnivcnYsV0= -go.opentelemetry.io/otel/metric v0.31.0 h1:6SiklT+gfWAwWUR0meEMxQBtihpiEs4c+vL9spDTqUs= -go.opentelemetry.io/otel/metric v0.31.0/go.mod h1:ohmwj9KTSIeBnDBm/ZwH2PSZxZzoOaG2xZeekTRzL5A= -go.opentelemetry.io/otel/sdk v1.10.0 h1:jZ6K7sVn04kk/3DNUdJ4mqRlGDiXAVuIG+MMENpTNdY= -go.opentelemetry.io/otel/sdk v1.10.0/go.mod h1:vO06iKzD5baltJz1zarxMCNHFpUlUiOy4s65ECtn6kE= -go.opentelemetry.io/otel/trace v1.10.0 h1:npQMbR8o7mum8uF95yFbOEJffhs1sbCOfDh8zAJiH5E= -go.opentelemetry.io/otel/trace v1.10.0/go.mod h1:Sij3YYczqAdz+EhmGhE6TpTxUO5/F/AzrK+kxfGqySM= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw= -go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= -go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= -golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220524215830-622c5d57e401 h1:zwrSfklXn0gxyLRX/aR+q6cgHbV/ItVyzbPlbA+dkAw= -golang.org/x/oauth2 v0.0.0-20220524215830-622c5d57e401/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200120151820-655fe14d7479/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= -golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220411224347-583f2d630306 h1:+gHMid33q6pen7kv9xvT+JRinntgeXO2AeZVd0AWD3w= -golang.org/x/time v0.0.0-20220411224347-583f2d630306/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= -golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= -google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= -google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= -google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= -google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= -google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= -google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= -google.golang.org/api v0.60.0 h1:eq/zs5WPH4J9undYM9IP1O7dSr7Yh8Y0GtSCpzGzIUk= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200117163144-32f20d992d24/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= -google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= -google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220601144221-27df5f98adab h1:YYs5818GyaApJxN5iyBnJxr7FUDrKpcXX+GaPrv0Cms= -google.golang.org/genproto v0.0.0-20220601144221-27df5f98adab/go.mod h1:yKyY4AMRwFiC8yMMNaMi+RkCnjZJt9LoWuvhXjMs+To= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.49.0 h1:WTLtQzmQori5FUH25Pq4WT22oCsv8USpQ+F6rqtsmxw= -google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/gcfg.v1 v1.2.0 h1:0HIbH907iBTAntm+88IJV2qmJALDAh8sPekI9Vc1fm0= -gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= -gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/warnings.v0 v0.1.1 h1:XM28wIgFzaBmeZ5dNHIpWLQpt/9DGKxk+rCg/22nnYE= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= -gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.26.1 h1:f+SWYiPd/GsiWwVRz+NbFyCgvv75Pk9NK6dlkZgpCRQ= -k8s.io/api v0.26.1/go.mod h1:xd/GBNgR0f707+ATNyPmQ1oyKSgndzXij81FzWGsejg= -k8s.io/apiextensions-apiserver v0.26.1 h1:cB8h1SRk6e/+i3NOrQgSFij1B2S0Y0wDoNl66bn8RMI= -k8s.io/apiextensions-apiserver v0.26.1/go.mod h1:AptjOSXDGuE0JICx/Em15PaoO7buLwTs0dGleIHixSM= -k8s.io/apimachinery v0.26.1 h1:8EZ/eGJL+hY/MYCNwhmDzVqq2lPl3N3Bo8rvweJwXUQ= -k8s.io/apimachinery v0.26.1/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74= -k8s.io/apiserver v0.26.1 h1:6vmnAqCDO194SVCPU3MU8NcDgSqsUA62tBUSWrFXhsc= -k8s.io/apiserver v0.26.1/go.mod h1:wr75z634Cv+sifswE9HlAo5FQ7UoUauIICRlOE+5dCg= -k8s.io/client-go v0.26.1 h1:87CXzYJnAMGaa/IDDfRdhTzxk/wzGZ+/HUQpqgVSZXU= -k8s.io/client-go v0.26.1/go.mod h1:IWNSglg+rQ3OcvDkhY6+QLeasV4OYHDjdqeWkDQZwGE= -k8s.io/cloud-provider v0.26.1 h1:qEZmsGWGptOtVSpeMdTsapHX2BEqIk7rc5MA4caBqE0= -k8s.io/cloud-provider v0.26.1/go.mod h1:6PheIxRySYuRBBxtTUADya8S2rbr18xKi+fhGbLkduc= -k8s.io/component-base v0.26.1 h1:4ahudpeQXHZL5kko+iDHqLj/FSGAEUnSVO0EBbgDd+4= -k8s.io/component-base v0.26.1/go.mod h1:VHrLR0b58oC035w6YQiBSbtsf0ThuSwXP+p5dD/kAWU= -k8s.io/component-helpers v0.26.1 h1:Y5h1OYUJTGyHZlSAsc7mcfNsWF08S/MlrQyF/vn93mU= -k8s.io/component-helpers v0.26.1/go.mod h1:jxNTnHb1axLe93MyVuvKj9T/+f4nxBVrj/xf01/UNFk= -k8s.io/cri-api v0.26.1 h1:HTlvEzrhrjuXvjrrGWC2UMfM3vpxxtFJSs20QffHtMA= -k8s.io/cri-api v0.26.1/go.mod h1:I5TGOn/ziMzqIcUvsYZzVE8xDAB1JBkvcwvR0yDreuw= -k8s.io/csi-translation-lib v0.26.1 h1:GQT88qX4e903HlFne1ovGFilvsd7kJUVi6SWOkOg2SQ= -k8s.io/csi-translation-lib v0.26.1/go.mod h1:tbcXKaVAS3G9iIAi+8Ujp+LPLetZ+vZ2AsZj+c1yXd8= -k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= -k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= -k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E= -k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= -k8s.io/kube-scheduler v0.26.1 h1:OsNOWNPYUeMIkm+LpX5V3Z7EhhZ8wYnOPIoL4dz2g4U= -k8s.io/kube-scheduler v0.26.1/go.mod h1:9SZcwHMANGrfXCJUOw/rPi8Iwd0X4HXx4czCHcR4HiU= -k8s.io/kubectl v0.26.1 h1:K8A0Jjlwg8GqrxOXxAbjY5xtmXYeYjLU96cHp2WMQ7s= -k8s.io/kubectl v0.26.1/go.mod h1:miYFVzldVbdIiXMrHZYmL/EDWwJKM+F0sSsdxsATFPo= -k8s.io/kubelet v0.26.1 h1:wQyCQYmLW6GN3v7gVTxnc3jAE4zMYDlzdF3FZV4rKas= -k8s.io/kubelet v0.26.1/go.mod h1:gFVZ1Ab4XdjtnYdVRATwGwku7FhTxo6LVEZwYoQaDT8= -k8s.io/kubernetes v1.26.0-beta.0 h1:yj4QZWKkBVkTXmObtYgPNKkvIpKD4dBsrfofOUP/XJY= -k8s.io/kubernetes v1.26.0-beta.0/go.mod h1:wrHfSVvj2ypp/C+qiyUni64MVGIwyxF6Ypy1vmq4ZjE= -k8s.io/legacy-cloud-providers v0.26.1 h1:1uWqF3CkFVzwOQyFObJRjTMkIe8bMBthxrJLoYx00l8= -k8s.io/mount-utils v0.26.1 h1:deN1IBPyi5UFEAgQYXBEDUejzQUNzRC1ML7BUMWljzA= -k8s.io/mount-utils v0.26.1/go.mod h1:au99w4FWU5ZWelLb3Yx6kJc8RZ387IyWVM9tN65Yhxo= -k8s.io/pod-security-admission v0.26.1 h1:EDIxsYFeKMzNvN/JB0PgQcuwBP6fIkIG2O8ZWJhzOp4= -k8s.io/pod-security-admission v0.26.1/go.mod h1:hCbYTG5UtLlivmukkMPjAWf23PUBUHzEvR60xNVWN4c= -k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20230202215443-34013725500c h1:YVqDar2X7YiQa/DVAXFMDIfGF8uGrHQemlrwRU5NlVI= -k8s.io/utils v0.0.0-20230202215443-34013725500c/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.36 h1:PUuX1qIFv309AT8hF/CdPKDmsG/hn/L8zRX7VvISM3A= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.36/go.mod h1:WxjusMwXlKzfAs4p9km6XJRndVt2FROgMVCE4cdohFo= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= -sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= -sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/tests/e2e/suite_test.go b/tests/e2e/suite_test.go deleted file mode 100644 index 0723ede2..00000000 --- a/tests/e2e/suite_test.go +++ /dev/null @@ -1,76 +0,0 @@ -/* - * - * Copyright © 2022-2023 Dell Inc. or its subsidiaries. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package e2etest - -import ( - "flag" - "fmt" - "os" - "path/filepath" - "testing" - - . "github.com/onsi/ginkgo/v2" - "github.com/onsi/ginkgo/v2/reporters" - . "github.com/onsi/gomega" - "k8s.io/kubernetes/test/e2e/framework" - config "k8s.io/kubernetes/test/e2e/framework/config" -) - -func init() { - var yamlError error - - testParameters, yamlError = readYaml("e2e-values.yaml") - if yamlError != nil { - framework.Failf("Unable to read yaml e2e-values.yaml: %s", yamlError.Error()) - } - - // k8s.io/kubernetes/tests/e2e/framework requires env KUBECONFIG to be set - // it does not fall back to defaults - if os.Getenv(fmt.Sprintf("%v", testParameters["kubeconfigEnvVar"])) == "" { - kubeconfig := filepath.Join(os.Getenv("HOME"), ".kube", "config") - os.Setenv(fmt.Sprintf("%v", testParameters["kubeconfigEnvVar"]), kubeconfig) - } - - framework.TestContext.Provider = "local" - - t := framework.TestContextType{} - - framework.AfterReadingAllFlags(&t) -} - -func TestE2E(t *testing.T) { - handleFlags() - RegisterFailHandler(Fail) - - // pass/fail/skip results summarized to this file - junitReporter := reporters.NewJUnitReporter("junit.xml") - - // dont dump huge logs of node / pods on error - framework.TestContext.DumpLogsOnFailure = false - - // framework.TestContext.DeleteNamespace = false - - // runs all ginkgo tests in go files - RunSpecsWithDefaultAndCustomReporters(t, "CSI Driver End-to-End Tests", []Reporter{junitReporter}) -} - -func handleFlags() { - config.CopyFlags(config.Flags, flag.CommandLine) - framework.RegisterCommonFlags(flag.CommandLine) - framework.RegisterClusterFlags(flag.CommandLine) - flag.Parse() -} diff --git a/tests/e2e/testing-manifests/statefulset/statefulset.yaml b/tests/e2e/testing-manifests/statefulset/statefulset.yaml deleted file mode 100644 index 0cbc60ea..00000000 --- a/tests/e2e/testing-manifests/statefulset/statefulset.yaml +++ /dev/null @@ -1,47 +0,0 @@ -# -# -# Copyright © 2022 Dell Inc. or its subsidiaries. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# - -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: external-access -spec: - replicas: 10 - selector: - matchLabels: - app: external-access - template: - metadata: - labels: - app: external-access - spec: - containers: - - name: busybox - image: gcr.io/google_containers/busybox:1.27 - command: ["/bin/sh", "-c", "sleep 3600"] - volumeMounts: - - name: www - mountPath: /data - volumeClaimTemplates: - - metadata: - name: www - annotations: - volume.beta.kubernetes.io/storage-class: external-access-sc - spec: - accessModes: [ "ReadWriteMany" ] - resources: - requests: - storage: 3Gi diff --git a/tests/e2e/utils.go b/tests/e2e/utils.go deleted file mode 100644 index e3b36cae..00000000 --- a/tests/e2e/utils.go +++ /dev/null @@ -1,340 +0,0 @@ -/* - * - * Copyright © 2022-2023 Dell Inc. or its subsidiaries. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package e2etest - -import ( - "context" - "fmt" - "os" - "path/filepath" - "time" - - "github.com/onsi/gomega" - "gopkg.in/yaml.v2" - apps "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" - storagev1 "k8s.io/api/storage/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/kubernetes/test/e2e/framework" - "k8s.io/kubernetes/test/e2e/framework/deployment" - "k8s.io/kubernetes/test/e2e/framework/manifest" - fpv "k8s.io/kubernetes/test/e2e/framework/pv" - fss "k8s.io/kubernetes/test/e2e/framework/statefulset" - - "k8s.io/kubernetes/test/e2e/framework/testfiles" - - "github.com/onsi/ginkgo/v2" -) - -// getNamespaceToRunTests returns the namespace in which the tests are expected -// to run. For test setups, returns random namespace name -func getNamespaceToRunTests(f *framework.Framework) string { - return f.Namespace.Name -} - -// not usedgit git -// bootstrap function takes care of initializing necessary tests context for e2e tests -func bootstrap(withoutDc ...bool) { - var err error - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - // ctx - _, cancel := context.WithCancel(context.Background()) - defer cancel() - - if framework.TestContext.RepoRoot != "" { - testfiles.AddFileSource(testfiles.RootFileSource{Root: framework.TestContext.RepoRoot}) - } - framework.TestContext.Provider = "local" -} - -// getPvFromClaim returns PersistentVolume for requested claim. -func getPvFromClaim(client clientset.Interface, namespace string, claimName string) *v1.PersistentVolume { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - pvclaim, err := client.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, claimName, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pv, err := client.CoreV1().PersistentVolumes().Get(ctx, pvclaim.Spec.VolumeName, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - return pv -} - -// DeleteDeployment : delete on the basis of namespace and deployment name -func DeleteDeployment(client clientset.Interface, deploymentObject *apps.Deployment, namespace string) { - deletePolicy := metav1.DeletePropagationForeground - deploymentsClient := client.AppsV1().Deployments(namespace) - err := deploymentsClient.Delete(context.TODO(), deploymentObject.Name, metav1.DeleteOptions{ - PropagationPolicy: &deletePolicy, - }) - framework.ExpectNoError(err) -} - -// ScaleDownDeployment : Decreasing the replica count -func ScaleDownDeployment(client clientset.Interface, deploymentObject *apps.Deployment, namespace string, replicaCount int32) { - s, err := client.AppsV1(). - Deployments(namespace). - GetScale(context.TODO(), deploymentObject.Name, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - sc := *s - sc.Spec.Replicas = replicaCount - - _, err = client.AppsV1(). - Deployments(namespace). - UpdateScale(context.TODO(), - deploymentObject.Name, &sc, metav1.UpdateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - // wait for few seconds to delete pods - for i := 0; i < 2; i++ { - time.Sleep(10 * time.Second) - s, err := client.AppsV1(). - Deployments(namespace). - GetScale(context.TODO(), deploymentObject.Name, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - if s.Spec.Replicas == replicaCount { - break - } - } -} - -// CreateStatefulSet creates a StatefulSet from the manifest at manifestPath in the given namespace. -func CreateStatefulSet(ns string, ss *apps.StatefulSet, c clientset.Interface) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - framework.Logf(fmt.Sprintf("Creating statefulset %v/%v with %d replicas and selector %+v", - ss.Namespace, ss.Name, *(ss.Spec.Replicas), ss.Spec.Selector)) - - _, err := c.AppsV1().StatefulSets(ns).Create(ctx, ss, metav1.CreateOptions{}) - framework.ExpectNoError(err) - fss.WaitForRunningAndReady(c, *ss.Spec.Replicas, ss) -} - -// CreateDeployment creates a deployment in the provided namespace -func CreateDeployment() (ns string, ss *apps.Deployment, c clientset.Interface) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - framework.Logf(fmt.Sprintf("Creating Deployment %v/%v with %d replicas and selector %+v", - ss.Namespace, ss.Name, *(ss.Spec.Replicas), ss.Spec.Selector)) - - _, err := c.AppsV1().Deployments(ns).Create(ctx, ss, metav1.CreateOptions{}) - framework.ExpectNoError(err) - deployment.WaitForDeploymentComplete(c, ss) - return -} - -// GetStatefulSetFromManifest creates a StatefulSet from the statefulset.yaml -// file present in the manifest path. -func GetStatefulSetFromManifest(ns string) *apps.StatefulSet { - pwd, _ := os.Getwd() - manifestPath := pwd + "/testing-manifests/statefulset/" - - ssManifestFilePath := filepath.Join(manifestPath, "statefulset.yaml") - framework.Logf("Parsing statefulset from %v", ssManifestFilePath) - ss, err := manifest.StatefulSetFromManifest(ssManifestFilePath, ns) - framework.ExpectNoError(err) - return ss -} - -// createPVCAndStorageClass helps creates a storage class with specified name, -// storageclass parameters and PVC using storage class. -func createPVCAndStorageClass(client clientset.Interface, pvcnamespace string, - pvclaimlabels map[string]string, testParameters map[interface{}]interface{}, ds string, - allowedTopologies []v1.TopologySelectorLabelRequirement, bindingMode storagev1.VolumeBindingMode, - allowVolumeExpansion bool, accessMode v1.PersistentVolumeAccessMode, - names ...string) (*storagev1.StorageClass, *v1.PersistentVolumeClaim, error) { - scName := "" - if len(names) > 0 { - scName = names[0] - } - - storageclass, err := createStorageClass(client, testParameters, - allowedTopologies, "", bindingMode, allowVolumeExpansion, scName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - pvclaim, err := createPVC(client, pvcnamespace, pvclaimlabels, ds, storageclass, accessMode) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - return storageclass, pvclaim, err -} - -// createStorageClass helps creates a storage class with specified name, -// storageclass parameters. -func createStorageClass(client clientset.Interface, testParameters map[interface{}]interface{}, - allowedTopologies []v1.TopologySelectorLabelRequirement, - scReclaimPolicy v1.PersistentVolumeReclaimPolicy, bindingMode storagev1.VolumeBindingMode, - allowVolumeExpansion bool, scName string) (*storagev1.StorageClass, error) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - var storageclass *storagev1.StorageClass - var err error - isStorageClassPresent := false - // since array's credentials are present in testParameters map, setting it as nil to not print sensitive data - testParameters["externalAccess"] = nil - ginkgo.By(fmt.Sprintf("Creating StorageClass %s with scParameters: %+v and allowedTopologies: %+v "+ - "and ReclaimPolicy: %+v and allowVolumeExpansion: %t", - scName, testParameters, allowedTopologies, scReclaimPolicy, allowVolumeExpansion)) - - storageclass, err = client.StorageV1().StorageClasses().Get(ctx, scName, metav1.GetOptions{}) - if !apierrors.IsNotFound(err) { - gomega.Expect(err).To(gomega.HaveOccurred()) - } - - if storageclass != nil && err == nil { - isStorageClassPresent = true - } - - if !isStorageClassPresent { - storageclass, err = client.StorageV1().StorageClasses().Create(ctx, getStorageClassSpec(scName, - testParameters, allowedTopologies, scReclaimPolicy, bindingMode, allowVolumeExpansion), metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), fmt.Sprintf("Failed to create storage class with err: %v", err)) - } - - return storageclass, err -} - -// getStorageClassSpec returns Storage Class Spec with supplied storage -// class parameters. -func getStorageClassSpec(scName string, testParameters map[interface{}]interface{}, - allowedTopologies []v1.TopologySelectorLabelRequirement, scReclaimPolicy v1.PersistentVolumeReclaimPolicy, - bindingMode storagev1.VolumeBindingMode, allowVolumeExpansion bool) *storagev1.StorageClass { - - /* vals := make([]string, 0) - vals = append(vals, testParameters["e2eCSIDriverName"]) - - topo := v1.TopologySelectorLabelRequirement{ - Key: testParameters["e2eCSIDriverName"] + "/" + testParameters["scParamStorageSystemValue"], - Values: vals, - } - */ - // allowedTopologies = append(allowedTopologies, topo) - - if bindingMode == "" { - bindingMode = storagev1.VolumeBindingWaitForFirstConsumer - } - - var sc = &storagev1.StorageClass{ - TypeMeta: metav1.TypeMeta{ - Kind: "StorageClass", - }, - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "sc-", - }, - Provisioner: fmt.Sprintf("%v", testParameters["e2eCSIDriverName"]), - VolumeBindingMode: &bindingMode, - AllowVolumeExpansion: &allowVolumeExpansion, - } - // If scName is specified, use that name, else auto-generate storage class - // name. - - if scName != "" { - sc.ObjectMeta = metav1.ObjectMeta{ - Name: scName, - } - } - - if testParameters != nil { - testParametersMap := make(map[string]string) - for k, v := range testParameters { - xType := fmt.Sprintf("%T", v) - // since we are using a common file(e2e-values) for configuration so the easiest way here is to pick the values that is of string not object - if xType == "string" { - testParametersMap[fmt.Sprintf("%v", k)] = fmt.Sprintf("%v", v) - } - } - sc.Parameters = testParametersMap - } - if allowedTopologies != nil { - sc.AllowedTopologies = []v1.TopologySelectorTerm{ - { - MatchLabelExpressions: allowedTopologies, - }, - } - } - if scReclaimPolicy != "" { - sc.ReclaimPolicy = &scReclaimPolicy - } - - return sc -} - -// createPVC helps creates pvc with given namespace and labels using given -// storage class. -func createPVC(client clientset.Interface, pvcnamespace string, pvclaimlabels map[string]string, ds string, - storageclass *storagev1.StorageClass, accessMode v1.PersistentVolumeAccessMode) (*v1.PersistentVolumeClaim, error) { - - pvcspec := getPersistentVolumeClaimSpecWithStorageClass(pvcnamespace, ds, storageclass, pvclaimlabels, accessMode) - - ginkgo.By(fmt.Sprintf("Creating PVC using the Storage Class %s with disk size %s and labels: %+v accessMode: %+v", - storageclass.Name, ds, pvclaimlabels, accessMode)) - - pvclaim, err := fpv.CreatePVC(client, pvcnamespace, pvcspec) - - gomega.Expect(err).NotTo(gomega.HaveOccurred(), fmt.Sprintf("Failed to create pvc with err: %v", err)) - framework.Logf("PVC created: %v in namespace: %v", pvclaim.Name, pvcnamespace) - return pvclaim, err -} - -// getPersistentVolumeClaimSpecWithStorageClass return the PersistentVolumeClaim -// spec with specified storage class. -func getPersistentVolumeClaimSpecWithStorageClass(namespace string, ds string, storageclass *storagev1.StorageClass, - pvclaimlabels map[string]string, accessMode v1.PersistentVolumeAccessMode) *v1.PersistentVolumeClaim { - disksize := fmt.Sprintf("%v", testParameters["diskSize"]) - if ds != "" { - disksize = ds - } - if accessMode == "" { - // If accessMode is not specified, set the default accessMode. - accessMode = v1.ReadWriteOnce - } - claim := &v1.PersistentVolumeClaim{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "pvc-", - Namespace: namespace, - }, - Spec: v1.PersistentVolumeClaimSpec{ - AccessModes: []v1.PersistentVolumeAccessMode{ - accessMode, - }, - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceName(v1.ResourceStorage): resource.MustParse(disksize), - }, - }, - StorageClassName: &(storageclass.Name), - }, - } - - if pvclaimlabels != nil { - claim.Labels = pvclaimlabels - } - return claim -} - -func readYaml(values string) (map[interface{}]interface{}, error) { - yfile, err := os.ReadFile(filepath.Clean(values)) - if err != nil { - return nil, err - } - data := make(map[interface{}]interface{}) - err = yaml.Unmarshal(yfile, &data) - if err != nil { - return nil, err - } - return data, nil -} diff --git a/tests/sanity/README.md b/tests/sanity/README.md index 52369db2..74cdd7ac 100644 --- a/tests/sanity/README.md +++ b/tests/sanity/README.md @@ -1,24 +1,69 @@ -## Sanity Tests For CSI PowerStore +# Sanity Tests For CSI PowerStore -Testing done by standard test suite from [Sanity Test Command Line Program](https://github.com/kubernetes-csi/csi-test/tree/master/cmd/csi-sanity) +Testing done by standard test suite from [Sanity Test Command Line Program](https://github.com/kubernetes-csi/csi-test/tree/master/cmd/csi-sanity) -### Building Image +## Prerequisites -To run these tests you need to build an image by yourself and upload it to any available repository. +To run these tests you need to: -### Running +1. Build and install the csi-sanity binary: -#### Prerequisites -Copy the `values.yaml` from `sanity-csi-powerstore` folder to folder with `install-sanity.sh` script and rename it to myvalues. -In `myvalues.yaml` point to your PowerStore array. -Install to kubernetes cluster by running install-sanity.sh. -> It will install bare version of driver without any sidecar containers +```sh +git clone https://github.com/kubernetes-csi/csi-test.git +cd csi-test +make build-sanity +cp cmd/csi-sanity/csi-sanity /usr/local/bin/ +``` + +2. Build the driver binary: + +```sh +cd csi-powerstore/ +make build +``` + +3. Fill in the following files in tests/sanity/; anything with a "REPLACE" prefix needs to be replaced with a real value: + +- config.yaml, this file will be used by the binary built in step 2 (from now on, referred to as "the binary" for short) to connect to array +- setup-driver-controller-sanity.sh, this file is used to start the driver's controller service from the binary +- setup-driver-node-sanity.sh, this file is used to start the driver's node service from the binary +- params.yaml, this file is used by the sanity test to pass in parameters that would be defined in the storageclass +- [Optional] driver-config-params.yaml, this file controls how the binary's logger is configured + +4. Install Disaster Recovery (DR) CRD from operatorconfig/moduleconfig/common/disaster-recovery/dr-crds.yaml in csm-operator repository. +```sh +kubectl apply -f dr-crds.yaml +``` -To run the tests run the `install-sanity.sh` script with full path to your csi-sanity image as first argument +## Running -Example: +1. Run the shell script to setup the driver's node service + +```sh +./setup-driver-node-sanity.sh +... +{"level":"info","msg":"node service registered","time":"2025-06-04T21:11:42.493415761+01:00"} +{"endpoint":"unix:///root/csi-powerstore/tests/sanity/node.sock","level":"info","msg":"serving","time":"2025-06-04T21:11:42.493449589+01:00"} +``` + +2. In a new terminal window, run the shell script to setup the driver's controller service + +```sh +./setup-driver-controller-sanity.sh +... +{"level":"info","msg":"node service registered","time":"2025-06-04T21:11:42.493415761+01:00"} +{"endpoint":"unix:///root/csi-powerstore/tests/sanity/node.sock","level":"info","msg":"serving","time":"2025-06-04T21:11:42.493449589+01:00"} ``` -./install-sanity.sh csi-sanity:latest + +3. In (another) new terminal window, run the shell script to run the sanity test + +```sh +./run-csi-sanity.sh ``` -Wait until testing is finished +Tests should pass in 10-12 minutes + +```sh +Ran 68 of 92 Specs in 706.781 seconds +SUCCESS! -- 68 Passed | 0 Failed | 1 Pending | 23 Skipped +``` diff --git a/tests/sanity/helm/config.yaml b/tests/sanity/config.yaml similarity index 64% rename from tests/sanity/helm/config.yaml rename to tests/sanity/config.yaml index 00c12172..821fb7ac 100644 --- a/tests/sanity/helm/config.yaml +++ b/tests/sanity/config.yaml @@ -1,6 +1,4 @@ -# -# -# Copyright © 2021-2022 Dell Inc. or its subsidiaries. All Rights Reserved. +# Copyright © 2020-2025 Dell Inc. or its subsidiaries. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,14 +10,13 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# -# sed "s/CONFIG_YAML/`cat config.yaml|base64 -w0`/g" secret.yaml | kubectl apply -f - arrays: - - endpoint: "https://1.2.3.4/api/rest" - username: "username" - password: "password" - insecure: true - default: true - block-protocol: "auto" - + - endpoint: "https://REPLACE_ENDPOINT/api/rest" + username: "REPLACE_USER" + globalID: "REPLACE_ARRAY_ID" + password: "REPLACE_PASS" + skipCertificateValidation: true + isDefault: true + blockProtocol: "none" + nasName: "REPLACE_NAS" diff --git a/tests/sanity/helm/secret.yaml b/tests/sanity/driver-config-params.yaml similarity index 69% rename from tests/sanity/helm/secret.yaml rename to tests/sanity/driver-config-params.yaml index aa4f0cb7..529f164e 100644 --- a/tests/sanity/helm/secret.yaml +++ b/tests/sanity/driver-config-params.yaml @@ -1,6 +1,4 @@ -# -# -# Copyright © 2021-2022 Dell Inc. or its subsidiaries. All Rights Reserved. +# Copyright © 2020-2025 Dell Inc. or its subsidiaries. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,14 +10,10 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# -apiVersion: v1 -kind: Secret -metadata: - name: powerstore-config - # Set driver namespace - namespace: sanity -type: Opaque -data: - config: CONFIG_YAML +CSI_LOG_LEVEL: "info" +CSI_LOG_FORMAT: "JSON" +PODMON_CONTROLLER_LOG_LEVEL: "debug" +PODMON_CONTROLLER_LOG_FORMAT: "JSON" +PODMON_NODE_LOG_LEVEL: "debug" +PODMON_NODE_LOG_FORMAT: "JSON" diff --git a/tests/sanity/helm/sanity-csi-powerstore/Chart.yaml b/tests/sanity/helm/sanity-csi-powerstore/Chart.yaml deleted file mode 100644 index 01aabacb..00000000 --- a/tests/sanity/helm/sanity-csi-powerstore/Chart.yaml +++ /dev/null @@ -1,32 +0,0 @@ -# -# -# Copyright © 2020-2022 Dell Inc. or its subsidiaries. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# - -name: sanity-csi-powerstore -version: 1.0.0 -appVersion: 1.0.0 -description: | - PowerStore CSI (Container Storage Interface) driver Kubernetes - integration. This chart includes everything required to provision via CSI as - well as a PowerStore StorageClass. -keywords: -- csi -- storage -home: https://github.com/dell/csi-powerstore -sources: -- https://github.com/dell/csi-powerstore -maintainers: -- name: DellEMC -engine: gotpl diff --git a/tests/sanity/helm/sanity-csi-powerstore/templates/controller.yaml b/tests/sanity/helm/sanity-csi-powerstore/templates/controller.yaml deleted file mode 100644 index 6329e67a..00000000 --- a/tests/sanity/helm/sanity-csi-powerstore/templates/controller.yaml +++ /dev/null @@ -1,164 +0,0 @@ -# -# -# Copyright © 2020-2022 Dell Inc. or its subsidiaries. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ .Release.Name }}-controller - namespace: {{ .Release.Namespace }} - ---- - -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: {{ .Release.Name }}-controller -rules: - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "watch", "list", "delete", "update", "create"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["create", "get", "list", "watch", "update", "delete"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents/status"] - verbs: ["update"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots", "volumesnapshots/status"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments/status"] - verbs: ["patch"] - - apiGroups: [""] - resources: ["pods"] - verbs: ["get", "list", "watch"] - - apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["create", "list", "watch", "delete"] - - apiGroups: ["storage.k8s.io"] - resources: ["csinodes"] - verbs: ["get", "list", "watch"] - # below for resizer - - apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["update", "patch"] - ---- - -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: {{ .Release.Name }}-controller -subjects: - - kind: ServiceAccount - name: {{ .Release.Name }}-controller - namespace: {{ .Release.Namespace }} -roleRef: - kind: ClusterRole - name: {{ .Release.Name }}-controller - apiGroup: rbac.authorization.k8s.io - ---- - -kind: Deployment -apiVersion: apps/v1 -metadata: - name: {{ .Release.Name }}-controller - namespace: {{ .Release.Namespace }} -spec: - selector: - matchLabels: - name: {{ .Release.Name }}-controller - replicas: {{ required "Must provide the number of controller replicas." .Values.controller.replicas }} - template: - metadata: - labels: - name: {{ .Release.Name }}-controller - spec: - {{ if .Values.controller.nodeSelector }} - nodeSelector: - {{- toYaml .Values.controller.nodeSelector | nindent 8 }} - {{ end }} - {{ if .Values.controller.tolerations }} - tolerations: - {{- toYaml .Values.controller.tolerations | nindent 6 }} - {{ end }} - serviceAccountName: {{ .Release.Name }}-controller - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: "name" - operator: In - values: - - {{ .Release.Name }}-controller - topologyKey: "kubernetes.io/hostname" - containers: - - name: driver - image: {{ required "Must provide the PowerStore driver container image." .Values.images.driver }} - imagePullPolicy: Always - command: [ "/csi-powerstore" ] - env: - - name: CSI_ENDPOINT - value: /var/run/csi/controller-csi.sock - - name: X_CSI_MODE - value: controller - - name: X_CSI_DEBUG - value: "true" - - name: X_CSI_DRIVER_NAME - value: {{ .Values.driverName }} - - name: X_CSI_DRIVER_EXTERNAL_ACCESS - value: {{ .Values.externalAccess }} - - name: X_CSI_POWERSTORE_CONFIG_PATH - value: /powerstore-config/config - volumeMounts: - - name: socket-dir - mountPath: /var/run/csi - - name: powerstore-config - mountPath: /powerstore-config - volumes: - - name: socket-dir - hostPath: - path: /var/run/csi - type: DirectoryOrCreate - - name: powerstore-config - secret: - secretName: powerstore-config diff --git a/tests/sanity/helm/sanity-csi-powerstore/templates/node.yaml b/tests/sanity/helm/sanity-csi-powerstore/templates/node.yaml deleted file mode 100644 index 64d824dd..00000000 --- a/tests/sanity/helm/sanity-csi-powerstore/templates/node.yaml +++ /dev/null @@ -1,231 +0,0 @@ -# -# -# Copyright © 2020-2022 Dell Inc. or its subsidiaries. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ .Release.Name }}-node - namespace: {{ .Release.Namespace }} - ---- - -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: {{ .Release.Name }}-node -rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["create", "delete", "get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["persistentvolumesclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["events"] - verbs: ["get", "list", "watch", "create", "update", "patch"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["security.openshift.io"] - resourceNames: ["privileged"] - resources: ["securitycontextconstraints"] - verbs: ["use"] - - ---- - -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: {{ .Release.Name }}-node -subjects: - - kind: ServiceAccount - name: {{ .Release.Name }}-node - namespace: {{ .Release.Namespace }} -roleRef: - kind: ClusterRole - name: {{ .Release.Name }}-node - apiGroup: rbac.authorization.k8s.io - ---- - -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: {{ .Release.Name }}-node - namespace: {{ .Release.Namespace }} -spec: - selector: - matchLabels: - app: {{ .Release.Name }}-node - template: - metadata: - labels: - app: {{ .Release.Name }}-node - spec: - {{ if .Values.node.nodeSelector }} - nodeSelector: - {{- toYaml .Values.node.nodeSelector | nindent 8 }} - {{ end }} - {{ if .Values.node.tolerations }} - tolerations: - {{- toYaml .Values.node.tolerations | nindent 6 }} - {{ end }} - serviceAccount: {{ .Release.Name }}-node - hostNetwork: true - hostIPC: true - containers: - - name: driver - securityContext: - privileged: true - capabilities: - add: ["SYS_ADMIN"] - allowPrivilegeEscalation: true - image: {{ required "Must provide the PowerStore driver container image." .Values.images.driver }} - imagePullPolicy: Always - command: [ "/csi-powerstore" ] - env: - - name: CSI_ENDPOINT - value: /var/run/csi/node-csi.sock - - name: X_CSI_MODE - value: node - - name: X_CSI_DEBUG - value: "true" - - name: X_CSI_POWERSTORE_KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: X_CSI_POWERSTORE_NODE_NAME_PREFIX - value: {{ .Values.nodeNamePrefix }} - - name: X_CSI_POWERSTORE_NODE_ID_PATH - value: /node-id - - name: X_CSI_POWERSTORE_NODE_CHROOT_PATH - value: /noderoot - - name: X_CSI_POWERSTORE_TMP_DIR - value: /var/lib/kubelet/plugins/{{ .Values.driverName }}/tmp - - name: X_CSI_DRIVER_NAME - value: {{ .Values.driverName }} - - name: X_CSI_FC_PORTS_FILTER_FILE_PATH - value: {{ .Values.nodeFCPortsFilterFile }} - - name: X_CSI_ENABLE_TRACING - value: "false" - - name: X_CSI_DRIVER_NAME - value: {{ .Values.driverName }} - {{- if eq .Values.connection.enableCHAP true }} - - name: X_CSI_POWERSTORE_ENABLE_CHAP - value: "true" - {{- else }} - - name: X_CSI_POWERSTORE_ENABLE_CHAP - value: "false" - {{- end }} - - name: X_CSI_POWERSTORE_CONFIG_PATH - value: /powerstore-config/config - volumeMounts: - - name: driver-path - mountPath: /var/lib/kubelet/plugins/{{ .Values.driverName }} - - name: csi-path - mountPath: /var/lib/kubelet/plugins/kubernetes.io/csi - mountPropagation: "Bidirectional" - - name: pods-path - mountPath: /var/lib/kubelet/pods - mountPropagation: "Bidirectional" - - name: stg - mountPath: /dev/stg - mountPropagation: "Bidirectional" - - name: mnt - mountPath: /dev/mnt - mountPropagation: "Bidirectional" - - name: dev - mountPath: /dev - - name: sys - mountPath: /sys - - name: run - mountPath: /run - - name: node-id - mountPath: /node-id - - name: etciscsi - mountPath: /etc/iscsi - - name: mpath - mountPath: /etc/multipath.conf - - name: noderoot - mountPath: /noderoot - - name: powerstore-config - mountPath: /powerstore-config - volumes: - - name: registration-dir - hostPath: - path: /var/lib/kubelet/plugins_registry/ - type: DirectoryOrCreate - - name: driver-path - hostPath: - path: /var/lib/kubelet/plugins/{{ .Values.driverName }} - type: DirectoryOrCreate - - name: csi-path - hostPath: - path: /var/lib/kubelet/plugins/kubernetes.io/csi - - name: pods-path - hostPath: - path: /var/lib/kubelet/pods - type: Directory - - name: stg - hostPath: - path: /dev/stg - type: Directory - - name: mnt - hostPath: - path: /dev/mnt - type: Directory - - name: dev - hostPath: - path: /dev - type: Directory - - name: node-id - hostPath: - path: {{ required "Must provide the path to file with node identifier." .Values.nodeIDPath }} - type: File - - name: etciscsi - hostPath: - path: /etc/iscsi - type: DirectoryOrCreate - - name: mpath - hostPath: - path: /etc/multipath.conf - type: FileOrCreate - - name: noderoot - hostPath: - path: / - type: Directory - - name: sys - hostPath: - path: /sys - type: Directory - - name: run - hostPath: - path: /run - type: Directory - - name: powerstore-config - secret: - secretName: powerstore-config diff --git a/tests/sanity/helm/sanity-csi-powerstore/values.yaml b/tests/sanity/helm/sanity-csi-powerstore/values.yaml deleted file mode 100644 index 06c664e8..00000000 --- a/tests/sanity/helm/sanity-csi-powerstore/values.yaml +++ /dev/null @@ -1,68 +0,0 @@ -# -# -# Copyright © 2020-2022 Dell Inc. or its subsidiaries. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# - -# "driverName" defines the name of driver (provisioner) -driverName: "csi-powerstore.dellemc.com" - -# "powerStoreApi" defines the full path to PowerStore API -# Example: https://127.0.0.1/api/rest -powerStoreApi: - -# "powerStoreApiUser" defines username for PowerStore API -powerStoreApiUser: - -# "powerStoreApiPassword" defines password for PowerStore API -powerStoreApiPassword: - -# "volumeNamePrefix" defines a string prepended to each volume created by the CSI driver. -volumeNamePrefix: csi - -# "nodeNamePrefix" defines a string prepended to each node registered by the CSI driver. -nodeNamePrefix: csi-node - -# "nodeIDPath" defines the path to file with node identifier (e.g. /etc/machine-id, /etc/hostname). -nodeIDPath: /etc/machine-id - -# "transportProtocol" enables you to be able to force the transport protocol. -# Valid values are: "FC", "ISCSI", "auto", "". If "" or "auto", will choose FC if both are available. -transportProtocol: ISCSI - -# "nodeFCPortsFilterFile" is the name of the environment variable which store path to the file which -# provide list of WWPN which should be used by the driver for FC connection on this node -# example: -# content of the file: -# 21:00:00:29:ff:48:9f:6e,21:00:00:29:ff:48:9f:6e -# If file not exist or empty or in invalid format, then the driver will use all available FC ports -nodeFCPortsFilterFile: /etc/fc-ports-filter - -# The installation process will generate multiple storageclasses based on these parameters. -# Only the primary storageclass for the driver will be marked default if specified. -storageClass: - # "storageClass.name" defines the name of the storage class to be defined. - name: powerstore - - # "storageClass.isDefault" defines whether the primary storage class should be the # default. - isDefault: "true" - - # "storageClass.reclaimPolicy" defines what will happen when a volume is - # removed from the Kubernetes API. Valid values are "Retain" and "Delete". - reclaimPolicy: Delete - -# IT IS RECOMMENDED YOU DO NOT CHANGE THE IMAGES TO BE DOWNLOADED. -images: - # "images.driver" defines the container images used for the driver container. - driver: - diff --git a/tests/sanity/install-sanity.sh b/tests/sanity/install-sanity.sh deleted file mode 100755 index 4b55eebb..00000000 --- a/tests/sanity/install-sanity.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/sh -# -# -# Copyright © 2020-2022 Dell Inc. or its subsidiaries. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# - -IMAGE=$1 - -kubectl create ns sanity -sed "s/CONFIG_YAML/`cat helm/config.yaml|base64 -w0`/g" helm/new-secret.yaml | kubectl apply -f - -# Create controller and noce driver instances -helm_command="helm install --values ./myvalues.yaml --name-template csi-sanity-pstore --namespace sanity ./helm/sanity-csi-powerstore --wait --timeout 180s" -echo "Helm install command:" -echo " ${helm_command}" -${helm_command} - -# Run tests from using csi-sanity container -./test.sh $1 - -# Delete sanity test chart -helm delete --namespace sanity csi-sanity-pstore -kubectl delete ns sanity diff --git a/tests/sanity/Dockerfile b/tests/sanity/params.yaml similarity index 66% rename from tests/sanity/Dockerfile rename to tests/sanity/params.yaml index 20ac3174..bb6f4128 100644 --- a/tests/sanity/Dockerfile +++ b/tests/sanity/params.yaml @@ -1,6 +1,4 @@ -# -# -# Copyright © 2020-2022 Dell Inc. or its subsidiaries. All Rights Reserved. +# Copyright © 2020-2025 Dell Inc. or its subsidiaries. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,14 +10,9 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# - -FROM golang:1.13.4 as build-env - -RUN go get -u github.com/kubernetes-csi/csi-test/... - -FROM frolvlad/alpine-glibc -WORKDIR /app/csi-sanity/ -COPY --from=build-env /go/bin/csi-sanity . -ENTRYPOINT [ "./csi-sanity" ] +# put your storage class parameters here, csi-sanity test will pass them to any createVolume request +arrayID: "REPLACE_ARRAY_ID" +csi.storage.k8s.io/fstype: "nfs" +nasName: "REPLACE_NAS" +allowRoot: "false" diff --git a/tests/e2e/run.sh b/tests/sanity/run-csi-sanity.sh old mode 100644 new mode 100755 similarity index 54% rename from tests/e2e/run.sh rename to tests/sanity/run-csi-sanity.sh index 7bd74464..3851d856 --- a/tests/e2e/run.sh +++ b/tests/sanity/run-csi-sanity.sh @@ -1,7 +1,6 @@ +#!/bin/bash -# -# -# Copyright © 2022 Dell Inc. or its subsidiaries. All Rights Reserved. +# Copyright © 2020-2025 Dell Inc. or its subsidiaries. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,11 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# -# supress ginkgo 2.0 upgrade hints -export ACK_GINKGO_DEPRECATIONS=1.16.5 -# run all tests -go test -timeout=25m -v ./ -ginkgo.v=1 +# remove left over from any uncompleted previous run +rm -rf /tmp/csi-mount +rm -rf /tmp/csi-staging + + +csi-sanity --ginkgo.v --csi.controllerendpoint=controller.sock --csi.endpoint=node.sock --csi.testvolumeparameters=params.yaml --ginkgo.junit-report=report.xml +# to run specific tests, add optional focus arguments like so: +#--ginkgo.focus="should remove target path" diff --git a/tests/sanity/setup-driver-controller-sanity.sh b/tests/sanity/setup-driver-controller-sanity.sh new file mode 100755 index 00000000..eb713c20 --- /dev/null +++ b/tests/sanity/setup-driver-controller-sanity.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# Copyright © 2020-2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +echo "Make sure binary exists!" +echo "Starting driver!" + +export X_CSI_POWERSTORE_CONFIG_PATH=$(pwd)/config.yaml +export X_CSI_POWERSTORE_CONFIG_PARAMS_PATH=$(pwd)/driver-config-params.yaml +export X_CSI_MODE=controller +export X_CSI_DRIVER_NAMESPACE=powerstore +export CSI_ENDPOINT=$(pwd)/controller.sock +export X_CSI_VOL_PREFIX=sanity +export X_CSI_MAX_VOLUMES_PER_NODE=0 +export X_CSI_NODE_IP=REPLACE_IP +export X_CSI_POWERSTORE_KUBE_NODE_NAME=REPLACE_NAME +export X_CSI_HEALTH_MONITOR_ENABLED=true +export CSI_AUTO_ROUND_OFF_FILESYSTEM_SIZE=true + +#assume binary is in csi-powerstore/ dir + ../../csi-powerstore --array-config=/root/csi-powerstore/tests/sanity/config.yaml --driver-config-params=/root/csi-powerstore/tests/sanity/driver-config-params.yaml diff --git a/tests/sanity/setup-driver-node-sanity.sh b/tests/sanity/setup-driver-node-sanity.sh new file mode 100755 index 00000000..a7889d1a --- /dev/null +++ b/tests/sanity/setup-driver-node-sanity.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Copyright © 2020-2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +echo "Make sure binary exists!" +echo "Starting driver!" + +export X_CSI_POWERSTORE_CONFIG_PATH=$(pwd)/config.yaml +export X_CSI_POWERSTORE_CONFIG_PARAMS_PATH=$(pwd)/driver-config-params.yaml +export X_CSI_MODE=node +export X_CSI_DRIVER_NAMESPACE=powerstore +export CSI_ENDPOINT=$(pwd)/node.sock +export X_CSI_VOL_PREFIX=node +export X_CSI_MAX_VOLUMES_PER_NODE=0 +export X_CSI_NODE_IP=REPLACE_IP +export X_CSI_POWERSTORE_KUBE_NODE_NAME=REPLACE_NAME +export X_CSI_POWERSTORE_NODE_ID_PATH=/etc/machine-id +export X_CSI_POWERSTORE_NODE_CHROOT_PATH=/ +export X_CSI_HEALTH_MONITOR_ENABLED=true +export CSI_AUTO_ROUND_OFF_FILESYSTEM_SIZE=true + +#assume binary is in csi-powerstore/ dir +../../csi-powerstore --array-config=/root/csi-powerstore/tests/sanity/config.yaml --driver-config-params=/root/csi-powerstore/tests/sanity/driver-config-params.yaml diff --git a/tests/sanity/test.sh b/tests/sanity/test.sh deleted file mode 100755 index 56b96759..00000000 --- a/tests/sanity/test.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/sh -# -# -# Copyright © 2020-2022 Dell Inc. or its subsidiaries. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# - -IMAGE=$1 -kubectl run csi-sanity --image=$IMAGE --overrides=' -{ - "apiVersion": "v1", - "spec": { - "containers": [ - { - "name": "csi-sanity", - "image": "'$IMAGE'", - "stdin": true, - "stdinOnce": true, - "tty": true, - "command": ["/app/csi-sanity/csi-sanity"], - "args": ["--ginkgo.v", "--csi.endpoint=/node.sock", "--csi.controllerendpoint=/controller.sock", "--csi.mountdir=/dev/mnt", "--csi.stagingdir=/dev/stg"], - "volumeMounts": [{ - "name": "controller", - "mountPath": "/controller.sock" - }, - { - "name": "node", - "mountPath": "/node.sock" - }] - } - ], - "volumes": [{ - "name":"controller", - "hostPath":{ - "path": "/var/run/csi/controller-csi.sock" - } - }, - { - "name":"node", - "hostPath":{ - "path": "/var/run/csi/node-csi.sock" - } - }] - } -} -' --rm -ti --attach --restart=Never \ No newline at end of file diff --git a/tests/scale/volumes/templates/test.yaml b/tests/scale/volumes/templates/test.yaml index 7f0241e6..cbf0686c 100644 --- a/tests/scale/volumes/templates/test.yaml +++ b/tests/scale/volumes/templates/test.yaml @@ -1,4 +1,5 @@ -# +# yamllint disable-file +# This file is not valid YAML because it is a Helm template # # Copyright © 2020-2022 Dell Inc. or its subsidiaries. All Rights Reserved. # @@ -32,7 +33,7 @@ spec: spec: containers: - name: test - image: docker.io/centos:latest + image: quay.io/centos/centos:latest imagePullPolicy: IfNotPresent volumeMounts: {{ range $i, $e := until (int .Values.volumeCount) }} diff --git a/tests/scale/volumes/values.yaml b/tests/scale/volumes/values.yaml index 7e15d050..d484c8fa 100644 --- a/tests/scale/volumes/values.yaml +++ b/tests/scale/volumes/values.yaml @@ -17,4 +17,4 @@ name: scaleStatefulSet replicas: 1 storageClass: default -volumeCount: 5 \ No newline at end of file +volumeCount: 5 diff --git a/tests/simple/simple.yaml b/tests/simple/simple.yaml index 21578b77..0df21669 100644 --- a/tests/simple/simple.yaml +++ b/tests/simple/simple.yaml @@ -1,6 +1,4 @@ -# -# -# Copyright © 2020-2022 Dell Inc. or its subsidiaries. All Rights Reserved. +# Copyright © 2020-2024 Dell Inc. or its subsidiaries. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,11 +10,12 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# -# This test creates 2 different PersistentVolumeClaims -# These PVCs use ext4 and xfs default storage classes -# PVs are mounted to pod from StatefulSet +# This test creates three different PersistentVolumeClaims using default +# ext4, xfs, and nfs storage classes and automatically mounts them to the pod. +# +# It assumes that you’ve created the same basic three storage classes from +# samples/storageclass folder without changing their names. # # To test the driver just run from root directory of repository # > kubectl create -f ./tests/simple/ @@ -29,7 +28,7 @@ # # After that you can uninstall the testing PVCs and StatefulSet # > kubectl delete -f ./tests/simple/ -# +# apiVersion: v1 kind: Namespace metadata: @@ -42,12 +41,12 @@ metadata: namespace: testpowerstore spec: accessModes: - - ReadWriteOnce + - ReadWriteOnce volumeMode: Filesystem resources: requests: storage: 8Gi - storageClassName: powerstore + storageClassName: powerstore-ext4 --- kind: PersistentVolumeClaim apiVersion: v1 @@ -56,7 +55,7 @@ metadata: namespace: testpowerstore spec: accessModes: - - ReadWriteOnce + - ReadWriteOnce volumeMode: Filesystem resources: requests: @@ -80,44 +79,44 @@ spec: apiVersion: v1 kind: ServiceAccount metadata: - name: powerstoretest - namespace: testpowerstore + name: powerstoretest + namespace: testpowerstore --- kind: StatefulSet apiVersion: apps/v1 metadata: - name: powerstoretest - namespace: testpowerstore + name: powerstoretest + namespace: testpowerstore spec: - serviceName: powerstoretest - selector: - matchLabels: - app: powerstoretest - template: - metadata: - labels: - app: powerstoretest - spec: - serviceAccount: powerstoretest - hostNetwork: true - containers: - - name: test - image: quay.io/centos/centos:latest - command: [ "/bin/sleep", "3600" ] - volumeMounts: - - mountPath: "/data0" - name: pvol0 - - mountPath: "/data1" - name: pvol1 - - mountPath: "/data2" - name: pvol2 - volumes: - - name: pvol0 - persistentVolumeClaim: - claimName: pvol0 - - name: pvol1 - persistentVolumeClaim: - claimName: pvol1 - - name: pvol2 - persistentVolumeClaim: - claimName: pvol2 + serviceName: powerstoretest + selector: + matchLabels: + app: powerstoretest + template: + metadata: + labels: + app: powerstoretest + spec: + serviceAccount: powerstoretest + hostNetwork: true + containers: + - name: test + image: quay.io/centos/centos:latest + command: ["/bin/sleep", "3600"] + volumeMounts: + - mountPath: "/data0" + name: pvol0 + - mountPath: "/data1" + name: pvol1 + - mountPath: "/data2" + name: pvol2 + volumes: + - name: pvol0 + persistentVolumeClaim: + claimName: pvol0 + - name: pvol1 + persistentVolumeClaim: + claimName: pvol1 + - name: pvol2 + persistentVolumeClaim: + claimName: pvol2