diff --git a/.gitignore b/.gitignore index 4f079119..411f753a 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ csi-powermax !csi-powermax/ semver.mk vendor/ +scinib # files created by IDEs .vscode diff --git a/Dockerfile b/Dockerfile index 22bcaa0a..2a8e259e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ # ARG GOIMAGE ARG BASEIMAGE -ARG VERSION="2.16.2" +ARG VERSION="2.17.0" # Stage to build the driver FROM $GOIMAGE as builder @@ -35,7 +35,7 @@ LABEL vendor="Dell Technologies" \ name="csi-powermax" \ summary="CSI Driver for Dell EMC PowerMax" \ description="CSI Driver for provisioning persistent storage from Dell EMC PowerMax" \ - release="1.16.0" \ + release="1.17.0" \ version=$VERSION \ license="Apache-2.0" COPY ./licenses /licenses diff --git a/Dockerfile.dev b/Dockerfile.dev new file mode 100644 index 00000000..0e430a99 --- /dev/null +++ b/Dockerfile.dev @@ -0,0 +1,35 @@ +# Copyright © 2020-2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +ARG BASEIMAGE + +# Development image - copy pre-built binary +FROM $BASEIMAGE AS final +ARG VERSION="dev" + +# Copy the pre-built binary from local build +COPY csi-powermax /csi-powermax +COPY csi-powermax.sh /csi-powermax.sh + +ENTRYPOINT ["/csi-powermax.sh"] +RUN chmod +x /csi-powermax.sh + +LABEL vendor="Dell Technologies" \ + maintainer="Dell Technologies" \ + name="csi-powermax" \ + summary="CSI Driver for Dell EMC PowerMax" \ + description="CSI Driver for provisioning persistent storage from Dell EMC PowerMax" \ + release="dev" \ + version=$VERSION \ + license="Apache-2.0" + +COPY ./licenses /licenses diff --git a/Makefile b/Makefile index 2be55551..8de66f99 100644 --- a/Makefile +++ b/Makefile @@ -14,6 +14,8 @@ include images.mk all: build +.PHONY: all build clean unit-test bdd-test integration-test gosec go-code-tester mocks dev-images help + # This will be overridden during image build. IMAGE_VERSION ?= 0.0.0 LDFLAGS = "-X main.ManifestSemver=$(IMAGE_VERSION)" @@ -64,3 +66,28 @@ go-code-tester: mocks: go generate ./... + +# Build images for development with dev tag and outside container build +dev-images: build + @echo "Building development images with dev tag..." + $(eval include csm-common.mk) + @echo "Building: $(IMAGE_REGISTRY)/$(IMAGE_NAME):dev" + $(BUILDER) build --pull -f Dockerfile.dev -t "$(IMAGE_REGISTRY)/$(IMAGE_NAME):dev" \ + --build-arg BASEIMAGE=$(CSM_BASEIMAGE) \ + --build-arg VERSION="dev" . + +# Show help for available targets +help: + @echo "Available targets:" + @echo " build - Build the Go binary" + @echo " images - Build container images with timestamp tag" + @echo " dev-images - Build container images with 'dev' tag for development (builds binary first, then copies to image)" + @echo " images-no-cache- Build container images with --no-cache" + @echo " push - Push container images to registry" + @echo " unit-test - Run unit tests" + @echo " bdd-test - Run BDD tests" + @echo " integration-test - Run integration tests" + @echo " gosec - Run security scan" + @echo " clean - Clean build artifacts" + @echo " mocks - Generate mocks" + @echo " help - Show this help message" diff --git a/core/semver/semver_test.go b/core/semver/semver_test.go index 37dc3200..fab60833 100644 --- a/core/semver/semver_test.go +++ b/core/semver/semver_test.go @@ -244,7 +244,7 @@ func TestErrorExit(t *testing.T) { return } // call the test again with INVOKE_ERROR_EXIT=1 so the errorExit function is invoked and we can check the return code - cmd := exec.Command(os.Args[0], "-test.run=TestErrorExit") // #nosec + cmd := exec.Command(os.Args[0], "-test.run=TestErrorExit") // #nosec G204,G702 cmd.Env = append(os.Environ(), "INVOKE_ERROR_EXIT=1") stderr, err := cmd.StderrPipe() diff --git a/csireverseproxy/.gitignore b/csireverseproxy/.gitignore index cee67233..7babd221 100644 --- a/csireverseproxy/.gitignore +++ b/csireverseproxy/.gitignore @@ -3,3 +3,4 @@ revproxy.exe certs/* !certs/.gitkeep c.out +csireverseproxy diff --git a/csireverseproxy/Dockerfile b/csireverseproxy/Dockerfile index 139afc8c..adc731d3 100644 --- a/csireverseproxy/Dockerfile +++ b/csireverseproxy/Dockerfile @@ -15,7 +15,7 @@ ############################ ARG GOIMAGE ARG BASEIMAGE -ARG VERSION="2.15.0" +ARG VERSION="2.16.0" FROM $GOIMAGE as builder @@ -55,7 +55,7 @@ LABEL vendor="Dell Technologies" \ name="csipowermax-reverseproxy" \ summary="CSI PowerMax Reverse Proxy" \ description="CSI PowerMax Reverse Proxy which helps manage connections with Unisphere for PowerMax" \ - release="1.16.0" \ + release="1.17.0" \ version=$VERSION \ license="Apache-2.0" COPY licenses /licenses diff --git a/csireverseproxy/Dockerfile.dev b/csireverseproxy/Dockerfile.dev new file mode 100644 index 00000000..e141bea2 --- /dev/null +++ b/csireverseproxy/Dockerfile.dev @@ -0,0 +1,66 @@ +# Copyright © 2020-2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +############################ +# STEP 1 build executable binary +############################ +ARG GOIMAGE +ARG BASEIMAGE +ARG GOPROXY + +FROM $GOIMAGE as builder + +# Install git + SSL ca certificates. +# Git is required for fetching the dependencies. +# Ca-certificates is required to call HTTPS endpoints. +RUN apt-get update && apt-get -y install tzdata && update-ca-certificates + +# Create revproxy +ENV USER=revproxy +ENV UID=10001 + +# See https://stackoverflow.com/a/55757473/12429735 +RUN adduser \ + --disabled-password \ + --gecos "" \ + --home "/nonexistent" \ + --shell "/sbin/nologin" \ + --no-create-home \ + --uid "${UID}" \ + "${USER}" + +############################ +# STEP 2 build a small image +############################ +FROM $BASEIMAGE as final +LABEL vendor="Dell Technologies" \ + maintainer="Dell Technologies" \ + name="csipowermax-reverseproxy" \ + summary="CSI PowerMax Reverse Proxy" \ + description="CSI PowerMax Reverse Proxy which helps manage connections with Unisphere for PowerMax" \ + release="1.13.0" \ + version="2.12.0" \ + license="Apache-2.0" +COPY licenses /licenses +COPY csireverseproxy /app/revproxy +# Import from builder. +COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ +COPY --from=builder /etc/passwd /etc/passwd +COPY --from=builder /etc/group /etc/group + +# Use an unprivileged user. +USER revproxy:revproxy + +WORKDIR /app +CMD ["/app/revproxy"] + +EXPOSE 2222 diff --git a/csireverseproxy/Makefile b/csireverseproxy/Makefile index c24f5464..ca670311 100644 --- a/csireverseproxy/Makefile +++ b/csireverseproxy/Makefile @@ -16,7 +16,7 @@ clean: go clean build: vendor - CGO_ENABLED=0 go build -mod=vendor + GOOS=linux CGO_ENABLED=0 go build -mod=vendor unit-test: go test -v -coverprofile c1.out ./... diff --git a/csireverseproxy/go.mod b/csireverseproxy/go.mod index 6975c670..1e5f5864 100644 --- a/csireverseproxy/go.mod +++ b/csireverseproxy/go.mod @@ -1,11 +1,11 @@ module github.com/dell/csi-powermax/csireverseproxy/v2 -go 1.25.0 +go 1.26.0 require ( - github.com/dell/csmlog v1.0.0 - github.com/dell/gopowermax/v2 v2.12.2 - github.com/fsnotify/fsnotify v1.9.0 + github.com/dell/csmlog v1.1.0 + github.com/dell/gopowermax/v2 v2.13.0 + github.com/fsnotify/fsnotify v1.10.1 github.com/gorilla/mux v1.8.1 github.com/kubernetes-csi/csi-lib-utils v0.23.0 github.com/mitchellh/mapstructure v1.5.0 @@ -15,9 +15,9 @@ require ( github.com/stretchr/testify v1.11.1 go.uber.org/mock v0.6.0 gopkg.in/yaml.v2 v2.4.0 - k8s.io/api v0.34.2 - k8s.io/apimachinery v0.34.2 - k8s.io/client-go v0.34.2 + k8s.io/api v0.36.0 + k8s.io/apimachinery v0.36.0 + k8s.io/client-go v0.36.0 ) require ( @@ -40,19 +40,14 @@ require ( github.com/go-openapi/swag/typeutils v0.25.4 // indirect github.com/go-openapi/swag/yamlutils v0.25.4 // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect - github.com/gogo/protobuf v1.3.2 // indirect github.com/google/gnostic-models v0.7.1 // indirect - github.com/google/go-cmp v0.7.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/onsi/ginkgo/v2 v2.25.3 // indirect - github.com/onsi/gomega v1.39.0 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/sagikazarmark/locafero v0.12.0 // indirect github.com/spf13/afero v1.15.0 // indirect github.com/spf13/cast v1.10.0 // indirect @@ -62,22 +57,22 @@ require ( go.uber.org/automaxprocs v1.6.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/net v0.48.0 // indirect + golang.org/x/net v0.52.0 // indirect golang.org/x/oauth2 v0.34.0 // indirect - golang.org/x/sys v0.39.0 // indirect - golang.org/x/term v0.38.0 // indirect - golang.org/x/text v0.32.0 // indirect + golang.org/x/sys v0.43.0 // indirect + golang.org/x/term v0.41.0 // indirect + golang.org/x/text v0.35.0 // indirect golang.org/x/time v0.14.0 // indirect - google.golang.org/grpc v1.79.1 // indirect - google.golang.org/protobuf v1.36.10 // indirect + google.golang.org/grpc v1.80.0 // indirect + google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/klog/v2 v2.130.1 // indirect - k8s.io/kube-openapi v0.0.0-20251125145642-4e65d59e963e // indirect - k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 // indirect + k8s.io/klog/v2 v2.140.0 // indirect + k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a // indirect + k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect - sigs.k8s.io/structured-merge-diff/v6 v6.3.1 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect sigs.k8s.io/yaml v1.6.0 // indirect ) diff --git a/csireverseproxy/go.sum b/csireverseproxy/go.sum index cf4ea981..262a02e3 100644 --- a/csireverseproxy/go.sum +++ b/csireverseproxy/go.sum @@ -1,9 +1,7 @@ -github.com/dell/csmlog v1.0.0 h1:EzW+nMJBD0QTNP88OoaAJUOMVilS9cWkO248BTcJt/4= -github.com/dell/csmlog v1.0.0/go.mod h1:7rBzSv9xF5t233+J+9vkStjFsmyYO3L/B9tDTy3+9ZU= -github.com/dell/gopowermax/v2 v2.12.2 h1:b50XR/a67H8zJHbKVnRqqvtrdTquvawXT4r1IqzwQc0= -github.com/dell/gopowermax/v2 v2.12.2/go.mod h1:f/wnjHWZ6H4nJE/w2VTu547yBvZxMQjlz3oc2twb2Fk= -github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= -github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/dell/csmlog v1.1.0 h1:M2pi9a/xTI2FtQNruOoCs2LPACsdJIqPmtUYIH3Y2cI= +github.com/dell/csmlog v1.1.0/go.mod h1:+I49NJrSPD8DhS7PY37vfIwM0lIHAvGm22zRoMG2ygU= +github.com/dell/gopowermax/v2 v2.13.0 h1:bsEUfKyEeLCFksZ86I1kJJNmSEELzB8lZi9j0qyT6jk= +github.com/dell/gopowermax/v2 v2.13.0/go.mod h1:F2DSQ9MlqryFRu9tmX/p1V4bw2ZBZbRhEa/kut+Qfg8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -12,8 +10,8 @@ github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bF github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= -github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= -github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho= +github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= @@ -52,27 +50,19 @@ github.com/go-openapi/testify/enable/yaml/v2 v2.0.2 h1:0+Y41Pz1NkbTHz8NngxTuAXxE github.com/go-openapi/testify/enable/yaml/v2 v2.0.2/go.mod h1:kme83333GCtJQHXQ8UKX3IBZu6z8T5Dvy5+CW3NLUUg= github.com/go-openapi/testify/v2 v2.0.2 h1:X999g3jeLcoY8qctY/c/Z8iBHTbwLz7R2WXd6Ub6wls= github.com/go-openapi/testify/v2 v2.0.2/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54= -github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= -github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/google/gnostic-models v0.7.1 h1:SisTfuFKJSKM5CPZkffwi6coztzzeYUhc3v4yxLWH8c= github.com/google/gnostic-models v0.7.1/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8= -github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -89,10 +79,6 @@ github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFd github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.25.3 h1:Ty8+Yi/ayDAGtk4XxmmfUy4GabvM+MegeB4cDLRi6nw= -github.com/onsi/ginkgo/v2 v2.25.3/go.mod h1:43uiyQC4Ed2tkOzLsEYm7hnrb7UJTWHYNsuy3bG/snE= -github.com/onsi/gomega v1.39.0 h1:y2ROC3hKFmQZJNFeGAMeHZKkjBL65mIZcvrLQBF9k6Q= -github.com/onsi/gomega v1.39.0/go.mod h1:ZCU1pkQcXDO5Sl9/VVEGlDyp+zm0m1cmeG5TOzLgdh4= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -127,8 +113,6 @@ github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -139,50 +123,23 @@ go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= -golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= +golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= +golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= -golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= -golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= -golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= +golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= +golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU= +golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A= +golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= +golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= -golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/grpc v1.79.1 h1:zGhSi45ODB9/p3VAawt9a+O/MULLl9dpizzNNpq7flY= -google.golang.org/grpc v1.79.1/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= -google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= -google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= +google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= +google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= +google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= @@ -195,23 +152,23 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.34.2 h1:fsSUNZhV+bnL6Aqrp6O7lMTy6o5x2C4XLjnh//8SLYY= -k8s.io/api v0.34.2/go.mod h1:MMBPaWlED2a8w4RSeanD76f7opUoypY8TFYkSM+3XHw= -k8s.io/apimachinery v0.34.2 h1:zQ12Uk3eMHPxrsbUJgNF8bTauTVR2WgqJsTmwTE/NW4= -k8s.io/apimachinery v0.34.2/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= -k8s.io/client-go v0.34.2 h1:Co6XiknN+uUZqiddlfAjT68184/37PS4QAzYvQvDR8M= -k8s.io/client-go v0.34.2/go.mod h1:2VYDl1XXJsdcAxw7BenFslRQX28Dxz91U9MWKjX97fE= -k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= -k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20251125145642-4e65d59e963e h1:iW9ChlU0cU16w8MpVYjXk12dqQ4BPFBEgif+ap7/hqQ= -k8s.io/kube-openapi v0.0.0-20251125145642-4e65d59e963e/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ= -k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 h1:SjGebBtkBqHFOli+05xYbK8YF1Dzkbzn+gDM4X9T4Ck= -k8s.io/utils v0.0.0-20251002143259-bc988d571ff4/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/api v0.36.0 h1:SgqDhZzHdOtMk40xVSvCXkP9ME0H05hPM3p9AB1kL80= +k8s.io/api v0.36.0/go.mod h1:m1LVrGPNYax5NBHdO+QuAedXyuzTt4RryI/qnmNvs34= +k8s.io/apimachinery v0.36.0 h1:jZyPzhd5Z+3h9vJLt0z9XdzW9VzNzWAUw+P1xZ9PXtQ= +k8s.io/apimachinery v0.36.0/go.mod h1:FklypaRJt6n5wUIwWXIP6GJlIpUizTgfo1T/As+Tyxc= +k8s.io/client-go v0.36.0 h1:pOYi7C4RHChYjMiHpZSpSbIM6ZxVbRXBy7CuiIwqA3c= +k8s.io/client-go v0.36.0/go.mod h1:ZKKcpwF0aLYfkHFCjillCKaTK/yBkEDHTDXCFY6AS9Y= +k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= +k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= +k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a h1:xCeOEAOoGYl2jnJoHkC3hkbPJgdATINPMAxaynU2Ovg= +k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a/go.mod h1:uGBT7iTA6c6MvqUvSXIaYZo9ukscABYi2btjhvgKGZ0= +k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 h1:AZYQSJemyQB5eRxqcPky+/7EdBj0xi3g0ZcxxJ7vbWU= +k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= -sigs.k8s.io/structured-merge-diff/v6 v6.3.1 h1:JrhdFMqOd/+3ByqlP2I45kTOZmTRLBUm5pvRjeheg7E= -sigs.k8s.io/structured-merge-diff/v6 v6.3.1/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/structured-merge-diff/v6 v6.3.2 h1:kwVWMx5yS1CrnFWA/2QHyRVJ8jM6dBA80uLmm0wJkk8= +sigs.k8s.io/structured-merge-diff/v6 v6.3.2/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/csireverseproxy/images.mk b/csireverseproxy/images.mk index 21cefeac..d7a4b8d9 100644 --- a/csireverseproxy/images.mk +++ b/csireverseproxy/images.mk @@ -19,3 +19,9 @@ images-no-cache: push: @echo "Pushing: $(IMAGE_REGISTRY)/$(IMAGE_NAME):$(IMAGE_TAG)" $(BUILDER) push "$(IMAGE_REGISTRY)/$(IMAGE_NAME):$(IMAGE_TAG)" + +# Build images for development with dev tag +dev-images: download-csm-common build + $(eval include csm-common.mk) + @echo "Building: $(IMAGE_REGISTRY)/$(IMAGE_NAME):dev" + $(BUILDER) build --pull -f Dockerfile.dev -t "$(IMAGE_REGISTRY)/$(IMAGE_NAME):dev" --build-arg BASEIMAGE=$(CSM_BASEIMAGE) --build-arg GOIMAGE=$(DEFAULT_GOIMAGE) . diff --git a/csireverseproxy/pkg/config/config.go b/csireverseproxy/pkg/config/config.go index 27ff1e69..f5347641 100644 --- a/csireverseproxy/pkg/config/config.go +++ b/csireverseproxy/pkg/config/config.go @@ -661,7 +661,9 @@ func (pc *ProxyConfig) ParseConfig(proxyConfigMap ProxyConfigMap, k8sUtils k8sut return err } var certFile string - if managementServer.CertSecret != "" { + // user may have certSecret set and SkipCertificationValidation set to true + // so if SkipCertificationValidation is true, ignore certSecret + if managementServer.CertSecret != "" && !managementServer.SkipCertificateValidation { certFile, err = k8sUtils.GetCertFileFromSecretName(managementServer.CertSecret) if err != nil { return err @@ -751,7 +753,9 @@ func (pc *ProxyConfig) ParseConfigFromSecret(proxySecret ProxySecret, k8sUtils k return err } var certFile string - if managementServer.CertSecret != "" { + // user may have certSecret set and SkipCertificationValidation set to true + // so if SkipCertificationValidation is true, ignore certSecret + if managementServer.CertSecret != "" && !managementServer.SkipCertificateValidation { certFile, err = k8sUtils.GetCertFileFromSecretName(managementServer.CertSecret) if err != nil { return err diff --git a/csireverseproxy/pkg/config/config_test.go b/csireverseproxy/pkg/config/config_test.go index c741f0e4..bdd9095b 100644 --- a/csireverseproxy/pkg/config/config_test.go +++ b/csireverseproxy/pkg/config/config_test.go @@ -215,6 +215,96 @@ func TestParseConfig(t *testing.T) { } } +func TestProxyConfig_ParseConfig_SkipCertificateValidation(t *testing.T) { + k8sUtils := k8smock.Init() + testCases := []struct { + name string + proxyConfigMap ProxyConfigMap + expectedErrorMessage string + }{ + { + name: "Valid config with one array and one management server", + proxyConfigMap: ProxyConfigMap{ + Config: &Config{ + StorageArrayConfig: []StorageArrayConfig{ + { + StorageArrayID: "test-symm-id", + PrimaryEndpoint: "https://management.example.com", + }, + }, + ManagementServerConfig: []ManagementServerConfig{ + { + Endpoint: "https://management.example.com", + Username: "test-username", + Password: "password", + SkipCertificateValidation: true, + }, + }, + }, + }, + expectedErrorMessage: "", + }, + { + name: "Valid config with one array where SkipCertificateValidation is true and CertSecret is provided", + proxyConfigMap: ProxyConfigMap{ + Config: &Config{ + StorageArrayConfig: []StorageArrayConfig{ + { + StorageArrayID: "test-symm-id", + PrimaryEndpoint: "https://management.example.com", + }, + }, + ManagementServerConfig: []ManagementServerConfig{ + { + Endpoint: "https://management.example.com", + Username: "test-username", + Password: "password", + SkipCertificateValidation: true, + // even though resource isn't mocked, test should not fail since SkipCertificateValidation is true + CertSecret: "not-mocked", + }, + }, + }, + }, + expectedErrorMessage: "", + }, + { + name: "Valid config with one array where SkipCertificateValidation is false but CertSecret cannot be found", + proxyConfigMap: ProxyConfigMap{ + Config: &Config{ + StorageArrayConfig: []StorageArrayConfig{ + { + StorageArrayID: "test-symm-id", + PrimaryEndpoint: "https://management.example.com", + }, + }, + ManagementServerConfig: []ManagementServerConfig{ + { + Endpoint: "https://management.example.com", + Username: "test-username", + Password: "password", + SkipCertificateValidation: false, + // this should return error since this secret will not be found and SkipCertificateValidation is false + CertSecret: "not-mocked", + }, + }, + }, + }, + expectedErrorMessage: "secrets \"not-mocked\" not found", + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var config ProxyConfig + err := config.ParseConfig(tc.proxyConfigMap, k8sUtils) + if err != nil { + assert.Contains(t, err.Error(), tc.expectedErrorMessage) + return + } + }) + } +} + func TestProxyConfig_UpdateCerts(t *testing.T) { config, err := getProxyConfig(t) if err != nil { @@ -609,11 +699,11 @@ func getProxyConfigFromSecret(t *testing.T) (*ProxyConfig, error) { } func TestProxyConfig_ParseConfigFromSecret(t *testing.T) { + k8sUtils := k8smock.Init() testCases := []struct { - name string - proxySecret ProxySecret - expectedConfig *ProxyConfig - expectedError error + name string + proxySecret ProxySecret + expectedErrorMessage string }{ { name: "Valid config with one array and one management server", @@ -633,7 +723,29 @@ func TestProxyConfig_ParseConfigFromSecret(t *testing.T) { }, }, }, - expectedError: nil, + expectedErrorMessage: "", + }, + { + name: "Valid config with one array where SkipCertificateValidation is true and CertSecret is provided", + proxySecret: ProxySecret{ + StorageArrayConfig: []StorageArrayConfig{ + { + StorageArrayID: "test-symm-id", + PrimaryEndpoint: "https://management.example.com", + }, + }, + ManagementServerConfig: []ManagementServerConfig{ + { + Endpoint: "https://management.example.com", + Username: "test-username", + Password: "password", + SkipCertificateValidation: true, + // even though resource isn't mocked, test should not fail since SkipCertificateValidation is true + CertSecret: "not-mocked", + }, + }, + }, + expectedErrorMessage: "", }, { name: "Invalid config with no arrays", @@ -648,7 +760,7 @@ func TestProxyConfig_ParseConfigFromSecret(t *testing.T) { }, }, }, - expectedError: fmt.Errorf("no storage arrays configured"), + expectedErrorMessage: "no storage arrays configured", }, { name: "Invalid config with no endpoints configured for a storage array", @@ -667,7 +779,7 @@ func TestProxyConfig_ParseConfigFromSecret(t *testing.T) { }, }, }, - expectedError: fmt.Errorf("primary endpoint not configured for array: test-symm-id"), + expectedErrorMessage: "primary endpoint not configured for array: test-symm-id", }, { name: "Invalid config with primary endpoint not among management servers", @@ -687,7 +799,7 @@ func TestProxyConfig_ParseConfigFromSecret(t *testing.T) { }, }, }, - expectedError: fmt.Errorf("primary endpoint: %s for array: %s not present among management endpoint addresses", "https://example.com", "test-symm-id"), + expectedErrorMessage: fmt.Sprintf("primary endpoint: %s for array: %s not present among management endpoint addresses", "https://example.com", "test-symm-id"), }, { name: "Valid config with multiple arrays and multiple management servers", @@ -719,18 +831,39 @@ func TestProxyConfig_ParseConfigFromSecret(t *testing.T) { }, }, }, - expectedError: nil, + expectedErrorMessage: "", + }, + { + name: "Valid config with one array where SkipCertificateValidation is false but CertSecret cannot be found", + proxySecret: ProxySecret{ + StorageArrayConfig: []StorageArrayConfig{ + { + StorageArrayID: "test-symm-id", + PrimaryEndpoint: "https://management.example.com", + }, + }, + ManagementServerConfig: []ManagementServerConfig{ + { + Endpoint: "https://management.example.com", + Username: "test-username", + Password: "password", + SkipCertificateValidation: false, + // this should return error since this secret will not be found and SkipCertificateValidation is false + CertSecret: "not-mocked", + }, + }, + }, + expectedErrorMessage: "secrets \"not-mocked\" not found", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { var config ProxyConfig - err := config.ParseConfigFromSecret(tc.proxySecret, nil) + err := config.ParseConfigFromSecret(tc.proxySecret, k8sUtils) if err != nil { - assert.Equal(t, tc.expectedError, err) + assert.Contains(t, err.Error(), tc.expectedErrorMessage) return } - // assert.Equal(t, tc.expectedConfig, &config) }) } } diff --git a/csireverseproxy/pkg/proxy/proxy.go b/csireverseproxy/pkg/proxy/proxy.go index 0926249e..a0132c64 100644 --- a/csireverseproxy/pkg/proxy/proxy.go +++ b/csireverseproxy/pkg/proxy/proxy.go @@ -286,7 +286,15 @@ func (revProxy *Proxy) getResponseIfAuthorised(res http.ResponseWriter, req *htt } requestID := req.Header.Get("RequestID") - req, err = http.NewRequest(req.Method, path, req.Body) // #nosec G704 - Not an user input + + // Validate backend URL scheme before making request to prevent SSRF + // Wrong: req.URL.Scheme (incoming request URL - always empty for relative paths) + // Correct: proxy.URL.Scheme (backend URL that actually needs validation) + if proxy.URL.Scheme != "http" && proxy.URL.Scheme != "https" { + utils.WriteHTTPError(res, "unsupported URL scheme: "+proxy.URL.Scheme, utils.StatusInternalError) + return nil, fmt.Errorf("unsupported URL scheme: %s", proxy.URL.Scheme) + } + req, err = http.NewRequest(req.Method, path, req.Body) // #nosec G704 - URL scheme validation implemented above if err != nil { http.Error(res, err.Error(), 500) return nil, err @@ -312,7 +320,13 @@ func (revProxy *Proxy) getResponseIfAuthorised(res http.ResponseWriter, req *htt return nil, err } defer lock.Release() - return client.Do(req) // #nosec G704 - Not an user input + + // Validate URL scheme before making request to prevent SSRF + if req.URL.Scheme != "http" && req.URL.Scheme != "https" { + utils.WriteHTTPError(res, "unsupported URL scheme: "+req.URL.Scheme, utils.StatusInternalError) + return nil, fmt.Errorf("unsupported URL scheme: %s", req.URL.Scheme) + } + return client.Do(req) // #nosec G704 - URL scheme validation implemented above } func (revProxy *Proxy) modifyHTTPRequest(res http.ResponseWriter, req *http.Request, targetURL url.URL) { @@ -434,6 +448,7 @@ func (revProxy *Proxy) GetRouter() http.Handler { router.PathPrefix(utils.Prefix + "/{version}/sloprovisioning/symmetrix/{symid}").HandlerFunc(revProxy.ServeReverseProxy) router.PathPrefix(utils.PrivatePrefix + "/{version}/replication/symmetrix/{symid}").HandlerFunc(revProxy.ServeReverseProxy) router.PathPrefix(utils.PrivatePrefix + "/{version}/sloprovisioning/symmetrix/{symid}").HandlerFunc(revProxy.ServeReverseProxy) + router.PathPrefix(utils.PrivateV1Prefix + "/systems/{symid}").HandlerFunc(revProxy.ServeReverseProxy) router.PathPrefix(utils.Prefix + "/{version}/replication/symmetrix/{symid}").HandlerFunc(revProxy.ServeReverseProxy) // endpoints without symmetrix id @@ -572,7 +587,12 @@ func (revProxy *Proxy) ServeReverseProxy(res http.ResponseWriter, req *http.Requ } defer utils.Elapsed(requestID, "Unisphere RESTAPI response")() defer lock.Release() - proxy.ReverseProxy.ServeHTTP(res, req) // #nosec G704 - Suppress error handling check as ReverseProxy.ServeHTTP handles errors internally + // Validate URL scheme before making request to prevent SSRF + if req.URL.Scheme != "http" && req.URL.Scheme != "https" { + utils.WriteHTTPError(res, "unsupported URL scheme: "+req.URL.Scheme, utils.StatusInternalError) + return + } + proxy.ReverseProxy.ServeHTTP(res, req) // #nosec G704 - URL scheme validation implemented above } } @@ -583,9 +603,27 @@ func (revProxy *Proxy) ServeVersions(res http.ResponseWriter, req *http.Request) return } for _, symID := range symIDs { - _, err := revProxy.getResponseIfAuthorised(res, req, symID) + resp, err := revProxy.getResponseIfAuthorised(res, req, symID) if err != nil { log.Errorf("Authorisation step fails for: (%s) symID with error (%s)", symID, err.Error()) + continue + } + if resp != nil { + defer resp.Body.Close() + err = utils.IsValidResponse(resp) + if err != nil { + log.Errorf("Get version step fails for: (%s) symID with error (%s)", symID, err.Error()) + utils.WriteHTTPError(res, err.Error(), resp.StatusCode) + } else { + versionDetails := new(types.VersionDetails) + if err := json.NewDecoder(resp.Body).Decode(versionDetails); err != nil { + utils.WriteHTTPError(res, "decoding error: "+err.Error(), 400) + log.Errorf("decoding error: %s", err.Error()) + } else { + utils.WriteHTTPResponse(res, versionDetails) + } + } + return } } } @@ -704,7 +742,12 @@ func (revProxy *Proxy) ServeIterator(res http.ResponseWriter, req *http.Request) utils.WriteHTTPError(res, "failed to obtain lock", utils.StatusInternalError) } defer lock.Release() - proxy.ReverseProxy.ServeHTTP(res, req) // #nosec G704 - Suppress error handling check as ReverseProxy.ServeHTTP handles errors internally + // Validate URL scheme before making request to prevent SSRF + if req.URL.Scheme != "http" && req.URL.Scheme != "https" { + utils.WriteHTTPError(res, "unsupported URL scheme: "+req.URL.Scheme, utils.StatusInternalError) + return + } + proxy.ReverseProxy.ServeHTTP(res, req) // #nosec G704 - URL scheme validation implemented above } // ServeSymmetrix - handler function for symmetrix list endpoint diff --git a/csireverseproxy/pkg/proxy/proxy_test.go b/csireverseproxy/pkg/proxy/proxy_test.go index dcba310f..827ef5d7 100644 --- a/csireverseproxy/pkg/proxy/proxy_test.go +++ b/csireverseproxy/pkg/proxy/proxy_test.go @@ -759,6 +759,59 @@ func TestGetRouter_ServeReverseProxy(t *testing.T) { } } +func TestGetRouter_ServeReverseProxy_PrivateV1SystemsRoute(t *testing.T) { + server := fakeServer(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, err := w.Write([]byte(`{"status":"ok"}`)) + if err != nil { + t.Errorf("expected nil error, got %v", err) + } + })) + + proxy, err := createValidProxyConfig(t, server) + if err != nil { + t.Fatalf("Failed to create proxy: %v", err) + } + + utils.InitializeLock() + router := proxy.GetRouter() + + arrayID := "000000000001" + url := fmt.Sprintf("%s/systems/%s/volumes", utils.PrivateV1Prefix, arrayID) + req, _ := http.NewRequest(http.MethodPost, url, bytes.NewBufferString(`{"volumes":[{}]}`)) + req = mux.SetURLVars(req, map[string]string{"symid": arrayID}) + req.SetBasicAuth("test-username", "test-password") + + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + + assert.Equal(t, http.StatusOK, rec.Code) +} + +func TestGetRouter_ServeReverseProxy_OldPrivateSloprovisioningSystemsPathNotRouted(t *testing.T) { + server := fakeServer(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + + proxy, err := createValidProxyConfig(t, server) + if err != nil { + t.Fatalf("Failed to create proxy: %v", err) + } + + router := proxy.GetRouter() + + arrayID := "000000000001" + url := fmt.Sprintf("%s/%s/sloprovisioning/systems/%s/volumes", utils.PrivatePrefix, "104", arrayID) + req, _ := http.NewRequest(http.MethodPost, url, nil) + req = mux.SetURLVars(req, map[string]string{"symid": arrayID}) + req.SetBasicAuth("test-username", "test-password") + + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + + assert.Equal(t, http.StatusNotFound, rec.Code) +} + func TestGetRouter_ServeVersions(t *testing.T) { testCases := []struct { name string @@ -2312,3 +2365,92 @@ func TestGetRouter_GetPortGroups(t *testing.T) { }) } } + +// TestURLSchemeValidation_Simple tests the SSRF protection by validating URL schemes directly +func TestURLSchemeValidation_Simple(t *testing.T) { + utils.InitializeLock() + + testCases := []struct { + name string + urlScheme string + shouldPass bool + skipURLCreation bool + }{ + { + name: "Valid HTTP scheme", + urlScheme: "http", + shouldPass: true, + skipURLCreation: false, + }, + { + name: "Valid HTTPS scheme", + urlScheme: "https", + shouldPass: true, + skipURLCreation: false, + }, + { + name: "Invalid FTP scheme - SSRF protection", + urlScheme: "ftp", + shouldPass: false, + skipURLCreation: false, + }, + { + name: "Invalid file scheme - SSRF protection", + urlScheme: "file", + shouldPass: false, + skipURLCreation: false, + }, + { + name: "Invalid gopher scheme - SSRF protection", + urlScheme: "gopher", + shouldPass: false, + skipURLCreation: false, + }, + { + name: "Empty scheme - SSRF protection", + urlScheme: "", + shouldPass: false, + skipURLCreation: true, // Skip URL creation for empty scheme test + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var req *http.Request + var err error + + if !tc.skipURLCreation { + // Create a mock HTTP request with the specified scheme + targetURL := fmt.Sprintf("%s://example.com/test", tc.urlScheme) + req, err = http.NewRequest("GET", targetURL, nil) + assert.NoError(t, err) + } else { + // For empty scheme test, create a request with a valid URL and manually set empty scheme + req, err = http.NewRequest("GET", "http://example.com/test", nil) + assert.NoError(t, err) + req.URL.Scheme = "" // Manually set empty scheme + } + + // Create a mock response writer + recorder := httptest.NewRecorder() + + // Test the URL scheme validation logic directly + if req.URL.Scheme != "http" && req.URL.Scheme != "https" { + // This should trigger the SSRF protection + if tc.shouldPass { + t.Errorf("Expected scheme %s to be valid, but it was rejected", tc.urlScheme) + } else { + // Expected to fail - write the error response as the proxy does + utils.WriteHTTPError(recorder, "unsupported URL scheme: "+req.URL.Scheme, utils.StatusInternalError) + assert.Equal(t, http.StatusInternalServerError, recorder.Code) + assert.Contains(t, recorder.Body.String(), "unsupported URL scheme: "+tc.urlScheme) + } + } else { + // This should pass the validation + if !tc.shouldPass { + t.Errorf("Expected scheme %s to be rejected, but it was accepted", tc.urlScheme) + } + } + }) + } +} diff --git a/csireverseproxy/pkg/utils/utils.go b/csireverseproxy/pkg/utils/utils.go index 2a254965..034cfcf8 100644 --- a/csireverseproxy/pkg/utils/utils.go +++ b/csireverseproxy/pkg/utils/utils.go @@ -41,6 +41,7 @@ const ( Prefix = "/univmax/restapi" PrefixV1 = "/univmax/rest/v1" PrivatePrefix = "/univmax/restapi/private" + PrivateV1Prefix = "/univmax/rest/private/v1" InternalPrefix = Prefix + "/internal" ) diff --git a/dell-csi-helm-installer/README.md b/dell-csi-helm-installer/README.md index efc38f1b..2e756918 100644 --- a/dell-csi-helm-installer/README.md +++ b/dell-csi-helm-installer/README.md @@ -36,7 +36,7 @@ This project provides the following capabilitites, each one is discussed in deta Most of these usages require the creation/specification of a values file. These files specify configuration settings that are passed into the driver and configure it for use. To create one of these files, the following steps should be followed: -1. Download a template file for the driver to a new location, naming this new file is at the users discretion. The template files are always found at `https://github.com/dell/helm-charts/raw/csi-powermax-2.16.2/charts/csi-powermax/values.yaml` +1. Download a template file for the driver to a new location, naming this new file is at the users discretion. The template files are always found at `https://github.com/dell/helm-charts/raw/csi-powermax-2.17.0/charts/csi-powermax/values.yaml` 2. Edit the file such that it contains the proper configuration settings for the specific environment. These files are yaml formatted so maintaining the file structure is important. For example, to create a values file for the PowerMax driver the following steps can be executed @@ -45,7 +45,7 @@ For example, to create a values file for the PowerMax driver the following steps cd dell-csi-helm-installer # Download the template file -wget -O my-powermax-settings.yaml https://github.com/dell/helm-charts/raw/csi-powermax-2.16.2/charts/csi-powermax/values.yaml +wget -O my-powermax-settings.yaml https://github.com/dell/helm-charts/raw/csi-powermax-2.17.0/charts/csi-powermax/values.yaml # edit the newly created values file vi my-powermax-settings.yaml diff --git a/dell-csi-helm-installer/csi-install.sh b/dell-csi-helm-installer/csi-install.sh index b7d46669..b31dc624 100755 --- a/dell-csi-helm-installer/csi-install.sh +++ b/dell-csi-helm-installer/csi-install.sh @@ -20,7 +20,7 @@ PROG="${0}" NODE_VERIFY=1 VERIFY=1 MODE="install" -DEFAULT_DRIVER_VERSION="v2.16.2" +DEFAULT_VERSION="v2.17.0" WATCHLIST="" # @@ -49,8 +49,6 @@ function usage() { exit 0 } -DRIVERVERSION="csi-powermax-2.16.2" - while getopts ":h-:" optchar; do case "${optchar}" in -) @@ -130,10 +128,6 @@ while getopts ":h-:" optchar; do esac done -if [ -n "$HELMCHARTVERSION" ]; then - DRIVERVERSION=$HELMCHARTVERSION -fi - if [ ! -d "$DRIVERDIR/helm-charts" ]; then if [ ! -d "$SCRIPTDIR/helm-charts" ]; then @@ -149,6 +143,14 @@ DRIVERDIR="${SCRIPTDIR}/../helm-charts/charts" DRIVER="csi-powermax" VERIFYSCRIPT="${SCRIPTDIR}/verify.sh" +# Derive helm chart version from DEFAULT_VERSION (single source of truth) +DRIVERVERSION="${DRIVER}-${DEFAULT_VERSION#v}" + +# Allow override via --helm-charts-version +if [ -n "$HELMCHARTVERSION" ]; then + DRIVERVERSION=$HELMCHARTVERSION +fi + # export the name of the debug log, so child processes will see it export DEBUGLOG="${SCRIPTDIR}/install-debug.log" declare -a VALIDDRIVERS @@ -407,7 +409,7 @@ RELEASE=$(get_release_name "${DRIVER}") # by default, NODEUSER is root NODEUSER="${NODEUSER:-root}" if [[ -z ${DRIVER_VERSION} ]]; then - DRIVER_VERSION=${DEFAULT_DRIVER_VERSION} + DRIVER_VERSION=${DEFAULT_VERSION} fi diff --git a/dell-csi-helm-installer/csi-offline-bundle.md b/dell-csi-helm-installer/csi-offline-bundle.md index c18e6ce5..1211765c 100644 --- a/dell-csi-helm-installer/csi-offline-bundle.md +++ b/dell-csi-helm-installer/csi-offline-bundle.md @@ -78,30 +78,9 @@ For example, here is the output of a request to build an offline bundle for the * * Pulling and saving container images - quay.io/dell/container-storage-modules/csi-isilon:v2.16.0 - quay.io/dell/container-storage-modules/csi-metadata-retriever:v1.13.0 - quay.io/dell/container-storage-modules/csipowermax-reverseproxy:v2.15.2 - quay.io/dell/container-storage-modules/csi-powermax:v2.16.2 - quay.io/dell/container-storage-modules/csi-powerstore:v2.16.0 - quay.io/dell/container-storage-modules/csi-unity:v2.16.0 - quay.io/dell/container-storage-modules/csi-vxflexos:v2.16.1 - quay.io/dell/container-storage-modules/csm-authorization-sidecar:v2.4.0 - quay.io/dell/container-storage-modules/csm-metrics-powerflex:v1.14.0 - quay.io/dell/container-storage-modules/csm-metrics-powerscale:v1.11.0 - quay.io/dell/container-storage-modules/csm-topology:v1.12.0 - quay.io/dell/container-storage-modules/dell-csi-replicator:v1.14.0 - quay.io/dell/container-storage-modules/dell-replication-controller:v1.14.0 - quay.io/dell/container-storage-modules/sdc:4.5.2.1 - quay.io/dell/container-storage-modules/dell-csm-operator:v1.11.3 - registry.redhat.io/openshift4/ose-kube-rbac-proxy-rhel9:v4.16.0-202409051837.p0.g8ea2c99.assembly.stream.el9 - nginxinc/nginx-unprivileged:1.29 - otel/opentelemetry-collector:0.142.0 - registry.k8s.io/sig-storage/csi-attacher:v4.10.0 - registry.k8s.io/sig-storage/csi-external-health-monitor-controller:v0.16.0 - registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.15.0 - registry.k8s.io/sig-storage/csi-provisioner:v6.1.0 - registry.k8s.io/sig-storage/csi-resizer:v2.0.0 - registry.k8s.io/sig-storage/csi-snapshotter:v8.4.0 +... + quay.io/dell/container-storage-modules/csi-powermax:v2.17.0 +... * * Copying necessary files @@ -176,32 +155,20 @@ Preparing a offline bundle for installation * * Loading docker images -Loaded image: quay.io/dell/container-storage-modules/csi-powerstore:v2.16.0 -Loaded image: quay.io/dell/container-storage-modules/csi-isilon:v2.16.0 -... +Loaded image: quay.io/dell/container-storage-modules/csi-powermax:v2.17.0 ... -Loaded image: registry.k8s.io/sig-storage/csi-resizer:v2.0.0 -Loaded image: registry.k8s.io/sig-storage/csi-snapshotter:v8.4.0 * * Tagging and pushing images - quay.io/dell/container-storage-modules/csi-isilon:v2.16.0 -> localregistry:5000/dell-csm-operator/csi-isilon:v2.16.0 - quay.io/dell/container-storage-modules/csi-metadata-retriever:v1.13.0 -> localregistry:5000/dell-csm-operator/csi-metadata-retriever:v1.13.0 + quay.io/dell/container-storage-modules/csi-powermax:v2.17.0 -> localregistry:5000/dell-csm-operator/csi-powermax:v2.17.0 ... - ... - registry.k8s.io/sig-storage/csi-resizer:v2.0.0 -> localregistry:5000/dell-csm-operator/csi-resizer:v2.0.0 - registry.k8s.io/sig-storage/csi-snapshotter:v8.4.0 -> localregistry:5000/dell-csm-operator/csi-snapshotter:v8.4.0 * * Preparing files within /root/dell-csm-operator-bundle - changing: quay.io/dell/container-storage-modules/csi-isilon:v2.16.0 -> localregistry:5000/dell-csm-operator/csi-isilon:v2.16.0 - changing: quay.io/dell/container-storage-modules/csi-metadata-retriever:v1.13.0 -> localregistry:5000/dell-csm-operator/csi-metadata-retriever:v1.13.0 - ... + changing: quay.io/dell/container-storage-modules/csi-powermax:v2.17.0 -> localregistry:5000/dell-csm-operator/csi-powermax:v2.17.0 ... - changing: registry.k8s.io/sig-storage/csi-resizer:v2.0.0 -> localregistry:5000/dell-csm-operator/csi-resizer:v2.0.0 - changing: registry.k8s.io/sig-storage/csi-snapshotter:v8.4.0 -> localregistry:5000/dell-csm-operator/csi-snapshotter:v8.4.0 * * Complete diff --git a/dell-csi-helm-installer/csi-offline-bundle.sh b/dell-csi-helm-installer/csi-offline-bundle.sh index de2624f5..12933f4e 100755 --- a/dell-csi-helm-installer/csi-offline-bundle.sh +++ b/dell-csi-helm-installer/csi-offline-bundle.sh @@ -232,13 +232,12 @@ CREATE="false" PREPARE="false" REGISTRY="" DRIVER="csi-powermax" +DEFAULT_VERSION="v2.17.0" # some directories SCRIPTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" REPODIR="$( dirname "${SCRIPTDIR}" )" -DRIVERVERSION="csi-powermax-2.16.2" - while getopts "cprv:h" opt; do case $opt in c) @@ -269,6 +268,10 @@ while getopts "cprv:h" opt; do esac done +# Derive DRIVERVERSION from DEFAULT_VERSION (single source of truth) +DRIVERVERSION="${DRIVER}-${DEFAULT_VERSION#v}" + +# Allow override via -v option if [ -n "$HELMCHARTVERSION" ]; then DRIVERVERSION=$HELMCHARTVERSION fi diff --git a/dell-csi-helm-installer/verify-csi-powermax.sh b/dell-csi-helm-installer/verify-csi-powermax.sh index bf909c34..213d23af 100644 --- a/dell-csi-helm-installer/verify-csi-powermax.sh +++ b/dell-csi-helm-installer/verify-csi-powermax.sh @@ -15,8 +15,8 @@ # verify-csi-powermax method function verify-csi-powermax() { - verify_k8s_versions "1.33" "1.35" - verify_openshift_versions "4.18" "4.20" + verify_k8s_versions "1.34" "1.36" + verify_openshift_versions "4.18" "4.21" verify_helm_values_version "${DRIVER_VERSION}" verify_namespace "${NS}" verify_required_secrets "${RELEASE}-creds" diff --git a/go.mod b/go.mod index 7be3f332..b28e9916 100644 --- a/go.mod +++ b/go.mod @@ -1,38 +1,38 @@ module github.com/dell/csi-powermax/v2 -go 1.25.0 +go 1.26 require ( - github.com/dell/csmlog v1.0.0 - github.com/dell/dell-csi-extensions/common v1.10.0 - github.com/dell/dell-csi-extensions/migration v1.10.0 - github.com/dell/dell-csi-extensions/podmon v1.10.0 - github.com/dell/dell-csi-extensions/replication v1.13.0 - github.com/dell/gobrick v1.16.0 - github.com/dell/gocsi v1.16.0 - github.com/dell/gofsutil v1.21.0 - github.com/dell/goiscsi v1.14.0 - github.com/dell/gonvme v1.13.0 - github.com/dell/gopowermax/v2 v2.12.2 + github.com/dell/csmlog v1.1.0 + github.com/dell/dell-csi-extensions/common v1.11.0 + github.com/dell/dell-csi-extensions/migration v1.11.0 + github.com/dell/dell-csi-extensions/podmon v1.11.0 + github.com/dell/dell-csi-extensions/replication v1.14.0 + github.com/dell/gobrick v1.17.0 + github.com/dell/gocsi v1.17.0 + github.com/dell/gofsutil v1.22.0 + github.com/dell/goiscsi v1.15.0 + github.com/dell/gonvme v1.14.0 + github.com/dell/gopowermax/v2 v2.13.0 github.com/akutz/goof v0.1.2 - github.com/container-storage-interface/spec v1.6.0 - github.com/coreos/go-systemd/v22 v22.6.0 + github.com/container-storage-interface/spec v1.11.0 + github.com/coreos/go-systemd/v22 v22.7.0 github.com/cucumber/godog v0.15.1 github.com/cucumber/messages-go/v10 v10.0.3 github.com/fsnotify/fsnotify v1.9.0 github.com/golang/mock v1.6.0 github.com/gorilla/mux v1.8.1 github.com/kubernetes-csi/csi-lib-utils v0.11.0 + github.com/robfig/cron/v3 v3.0.1 github.com/sirupsen/logrus v1.9.3 github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.11.1 - github.com/vmware/govmomi v0.52.0 - golang.org/x/net v0.48.0 - google.golang.org/grpc v1.79.1 - google.golang.org/protobuf v1.36.10 - k8s.io/api v0.34.2 - k8s.io/apimachinery v0.34.2 - k8s.io/client-go v0.34.2 + github.com/vmware/govmomi v0.53.0 + google.golang.org/grpc v1.80.0 + google.golang.org/protobuf v1.36.11 + k8s.io/api v0.35.3 + k8s.io/apimachinery v0.35.3 + k8s.io/client-go v0.35.3 ) require ( @@ -86,26 +86,26 @@ require ( go.etcd.io/etcd/api/v3 v3.6.6 // indirect go.etcd.io/etcd/client/pkg/v3 v3.6.6 // indirect go.etcd.io/etcd/client/v3 v3.6.6 // indirect - go.opentelemetry.io/otel v1.40.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.1 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/net v0.52.0 // indirect golang.org/x/oauth2 v0.34.0 // indirect - golang.org/x/sync v0.19.0 // indirect - golang.org/x/sys v0.39.0 // indirect - golang.org/x/term v0.38.0 // indirect - golang.org/x/text v0.32.0 // indirect + golang.org/x/sync v0.20.0 // indirect + golang.org/x/sys v0.43.0 // indirect + golang.org/x/term v0.41.0 // indirect + golang.org/x/text v0.35.0 // indirect golang.org/x/time v0.14.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260120221211-b8f7ae30c516 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260120221211-b8f7ae30c516 // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/klog/v2 v2.130.1 // indirect - k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect - k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect - sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect + k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 // indirect + k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v6 v6.3.1 // indirect sigs.k8s.io/yaml v1.6.0 // indirect diff --git a/go.sum b/go.sum index 0d985275..59105916 100644 --- a/go.sum +++ b/go.sum @@ -22,28 +22,28 @@ cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiy cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/dell/csmlog v1.0.0 h1:EzW+nMJBD0QTNP88OoaAJUOMVilS9cWkO248BTcJt/4= -github.com/dell/csmlog v1.0.0/go.mod h1:7rBzSv9xF5t233+J+9vkStjFsmyYO3L/B9tDTy3+9ZU= -github.com/dell/dell-csi-extensions/common v1.10.0 h1:WIFPWVEBUyzOTCOPAlcgQsiRRGAufyKJYbIATbNZXIY= -github.com/dell/dell-csi-extensions/common v1.10.0/go.mod h1:zRHzmPX5SQQnqQ1LEIxG4hYqLBeQOSiD8TkEhU0eWTY= -github.com/dell/dell-csi-extensions/migration v1.10.0 h1:TwMCI91zZNuDUqr3TcL1BCWL1cNkQMwkupyRbVbYJAo= -github.com/dell/dell-csi-extensions/migration v1.10.0/go.mod h1:BN7Mlxt3CrpqrxxliH3BadAtmAyTzdBKDid7IDyfoi4= -github.com/dell/dell-csi-extensions/podmon v1.10.0 h1:YeM9OmgJHE+n6aNaeEC96EuVev5x3pddggcM7Ws7pkk= -github.com/dell/dell-csi-extensions/podmon v1.10.0/go.mod h1:+g7fdyw1Zx74NBJQgi1BCtsywqk37MJd9JN86IPJJu0= -github.com/dell/dell-csi-extensions/replication v1.13.0 h1:DSpoZ3vX65a3KDxUv0OinLkY2qUAQtRX3E1c1e3fnvA= -github.com/dell/dell-csi-extensions/replication v1.13.0/go.mod h1:aJBwd55amqbY3kk8SG7NjwH7nxBscceDwc1rKesUG1g= -github.com/dell/gobrick v1.16.0 h1:z/a9qXnT3hx3D4I+SJUMnIgJtcCx0j3gzmPPDUWtoYs= -github.com/dell/gobrick v1.16.0/go.mod h1:9uoH8EsNi9yAsUZj2gZFgB5kqdlyvArqx0tYC7Qg9IM= -github.com/dell/gocsi v1.16.0 h1:avhQPD11rYzT6/dPxpZfFsJV+T/T0x1GJqqbco45W8c= -github.com/dell/gocsi v1.16.0/go.mod h1:Fz5dQv/kWf5Y1EXZEzxLBQSsnW2HE/WY95R0WCDQLO4= -github.com/dell/gofsutil v1.21.0 h1:SeusAYjiO/1ogvg/TapvCyHcrM9z+OvdaMU5i9Ijn3M= -github.com/dell/gofsutil v1.21.0/go.mod h1:qBGEz1wMOtqTODuJfiBZhUHT0JjexBblu2oa+sEclNs= -github.com/dell/goiscsi v1.14.0 h1:kNDqOlpJ3cLSJh7Hfyn/Kz/FMCKHzV0s/xx4EqnelFw= -github.com/dell/goiscsi v1.14.0/go.mod h1:SCSC8dJCqTosU7SspaoLv6ICTKNEz08rt/I8nZ3+ptc= -github.com/dell/gonvme v1.13.0 h1:j8A1BzYA48gelih3xWd/J6LQ71CbC8Lbdyv0jG8uUNU= -github.com/dell/gonvme v1.13.0/go.mod h1:L5K7V4JZTf12m3k2wdwKwP+/eA6pr8DvlCsJU1QTGOQ= -github.com/dell/gopowermax/v2 v2.12.2 h1:b50XR/a67H8zJHbKVnRqqvtrdTquvawXT4r1IqzwQc0= -github.com/dell/gopowermax/v2 v2.12.2/go.mod h1:f/wnjHWZ6H4nJE/w2VTu547yBvZxMQjlz3oc2twb2Fk= +github.com/dell/csmlog v1.1.0 h1:M2pi9a/xTI2FtQNruOoCs2LPACsdJIqPmtUYIH3Y2cI= +github.com/dell/csmlog v1.1.0/go.mod h1:+I49NJrSPD8DhS7PY37vfIwM0lIHAvGm22zRoMG2ygU= +github.com/dell/dell-csi-extensions/common v1.11.0 h1:G3chBDrKZN61XFJwY42AMQ9Rd7uIYrIoktRb//2pGEs= +github.com/dell/dell-csi-extensions/common v1.11.0/go.mod h1:zjvMAs9sJ6rxDKsNShd/TxZVmiQDMkZ6jIyOuaNADTQ= +github.com/dell/dell-csi-extensions/migration v1.11.0 h1:e8gjMH0zNUFXLsTExKDNbnd0xfj////ICrnR70DJ7ko= +github.com/dell/dell-csi-extensions/migration v1.11.0/go.mod h1:U0TxEJYo6RoTOJwMp2VMm8Fg4xi9FXJuJy+LSQnDsPA= +github.com/dell/dell-csi-extensions/podmon v1.11.0 h1:K7THnQ6ckhuHwAO59hM/kjpqZVi99XYIdgzLT6Eq35U= +github.com/dell/dell-csi-extensions/podmon v1.11.0/go.mod h1:vQrJUiJhxysGLl7TN5edroct3bXrnTn+12wbDwL4NQs= +github.com/dell/dell-csi-extensions/replication v1.14.0 h1:9ZlrHTO5AzvH2QNahPW+1acXPiVBE6G3OoveYTebV9I= +github.com/dell/dell-csi-extensions/replication v1.14.0/go.mod h1:dM5xa34Qt3nN5I1GczwFrLcc0RB5lQVVOPEWKp5ZIZU= +github.com/dell/gobrick v1.17.0 h1:bfbNMhHmoiDLLrQbAlXxbSwCYp8RXO85OXKKF+XkpcM= +github.com/dell/gobrick v1.17.0/go.mod h1:omT0QLeai8b7NP+e/bQLUcuhaRQiKDgZQX6TOfweaLg= +github.com/dell/gocsi v1.17.0 h1:nrpnwiVgi0d+1Babpo+mFTbHRw/J8bBAIw9jcb/2zWE= +github.com/dell/gocsi v1.17.0/go.mod h1:GNJIfINdAzlScBdy2kkqcJNDL272dgJtrk5Xe/9/7I8= +github.com/dell/gofsutil v1.22.0 h1:g1ALo2Y7xbljPw3nCGZ6S0VXf4WR1HIuz1dP4fh8M7E= +github.com/dell/gofsutil v1.22.0/go.mod h1:dnFY+zuE79FGv76g8RdUMqmhBNllvu5e/crZt56xJx0= +github.com/dell/goiscsi v1.15.0 h1:71QzLLm4X8XrEkGLnZshpGEDdkgbFuZ8NiwARFwaCtY= +github.com/dell/goiscsi v1.15.0/go.mod h1:jlkRplXgeJHMZZ/dLUkWAnNcOrkIXxuibi9vDbPKYk4= +github.com/dell/gonvme v1.14.0 h1:dRyS0o+3B+cnnncgblb/H0qUJkNzjkPAq/82oqt/eMc= +github.com/dell/gonvme v1.14.0/go.mod h1:bx/tqYBKuY8SHxEpw9b8SiD/98+4TQdMYkYWES39Dgw= +github.com/dell/gopowermax/v2 v2.13.0 h1:bsEUfKyEeLCFksZ86I1kJJNmSEELzB8lZi9j0qyT6jk= +github.com/dell/gopowermax/v2 v2.13.0/go.mod h1:F2DSQ9MlqryFRu9tmX/p1V4bw2ZBZbRhEa/kut+Qfg8= github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= @@ -55,6 +55,8 @@ github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= @@ -96,16 +98,16 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/container-storage-interface/spec v1.5.0/go.mod h1:8K96oQNkJ7pFcC2R9Z1ynGGBB1I93kcS6PGg3SsOk8s= -github.com/container-storage-interface/spec v1.6.0 h1:vwN9uCciKygX/a0toYryoYD5+qI9ZFeAMuhEEKO+JBA= -github.com/container-storage-interface/spec v1.6.0/go.mod h1:8K96oQNkJ7pFcC2R9Z1ynGGBB1I93kcS6PGg3SsOk8s= +github.com/container-storage-interface/spec v1.11.0 h1:H/YKTOeUZwHtyPOr9raR+HgFmGluGCklulxDYxSdVNM= +github.com/container-storage-interface/spec v1.11.0/go.mod h1:DtUvaQszPml1YJfIK7c00mlv6/g4wNMLanLgiUbKFRI= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.6.0 h1:aGVa/v8B7hpb0TKl0MWoAavPDmHvobFe5R5zn0bCJWo= -github.com/coreos/go-systemd/v22 v22.6.0/go.mod h1:iG+pp635Fo7ZmV/j14KUcmEyWF+0X7Lua8rrTWzYgWU= +github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA= +github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= @@ -278,8 +280,8 @@ github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OI github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= -github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -422,13 +424,13 @@ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108 github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= -github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns= +github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.39.0 h1:y2ROC3hKFmQZJNFeGAMeHZKkjBL65mIZcvrLQBF9k6Q= -github.com/onsi/gomega v1.39.0/go.mod h1:ZCU1pkQcXDO5Sl9/VVEGlDyp+zm0m1cmeG5TOzLgdh4= +github.com/onsi/gomega v1.40.0 h1:Vtol0e1MghCD2ZVIilPDIg44XSL9l2QAn8ZNaljWcJc= +github.com/onsi/gomega v1.40.0/go.mod h1:M/Uqpu/8qTjtzCLUA2zJHX9Iilrau25x1PdoSRbWh5A= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= @@ -469,6 +471,8 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -537,8 +541,8 @@ github.com/thecodeteam/gosync v0.1.0/go.mod h1:43QHsngcnWc8GE1aCmi7PEypslflHjCzX github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/vmware/govmomi v0.52.0 h1:JyxQ1IQdllrY7PJbv2am9mRsv3p9xWlIQ66bv+XnyLw= -github.com/vmware/govmomi v0.52.0/go.mod h1:Yuc9xjznU3BH0rr6g7MNS1QGvxnJlE1vOvTJ7Lx7dqI= +github.com/vmware/govmomi v0.53.0 h1:e1bZCotAq7wm4xy95ePN2uoWwz28pNp/ewZZhpBY7/4= +github.com/vmware/govmomi v0.53.0/go.mod h1:EWfuzPfxT5NV+aS2we02SLFdhvJkgeY7t7+TszgBSMY= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= @@ -573,8 +577,8 @@ go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.5 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0/go.mod h1:ijPqXp5P6IRRByFVVg9DY8P5HkxkHE5ARIa+86aXPf4= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= -go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms= -go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= go.opentelemetry.io/otel/exporters/otlp v0.20.0 h1:PTNgq9MRmQqqJY0REVbZFvwkYOA85vbdQU/nVfxDyqg= go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 h1:OeNbIYk/2C15ckl7glBlOBp5+WlYsOElzTNmiPW/x60= @@ -582,19 +586,19 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0/go.mod h1:7Bept48yIeqx go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 h1:tgJ0uaNS4c98WRNUEx5U3aDlrDOI5Rs+1Vifcw4DJ8U= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0/go.mod h1:U7HYyW0zt/a9x5J1Kjs+r1f/d4ZHnYFclhYY2+YbeoE= go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= -go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g= -go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= -go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= -go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= -go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw= -go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= @@ -623,8 +627,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= -golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= +golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= +golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -657,6 +661,8 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= +golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -688,8 +694,8 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= -golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= +golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= +golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -706,8 +712,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= -golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -752,13 +758,13 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= -golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= +golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= -golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= +golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU= +golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -766,8 +772,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= -golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= +golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= +golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -813,14 +819,14 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= -golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= +golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= +golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= -gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -858,10 +864,10 @@ google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 h1:fCvbg86sFXwdrl5LgVcTEvNC+2txB5mgROGmRL5mrls= -google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:+rXWjjaukWZun3mLfjmVnQi18E1AsFbDN9QdJ5YXLto= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 h1:gRkg/vSppuSQoDjxyiGfN4Upv/h/DQmIR10ZU8dh4Ww= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= +google.golang.org/genproto/googleapis/api v0.0.0-20260120221211-b8f7ae30c516 h1:vmC/ws+pLzWjj/gzApyoZuSVrDtF1aod4u/+bbj8hgM= +google.golang.org/genproto/googleapis/api v0.0.0-20260120221211-b8f7ae30c516/go.mod h1:p3MLuOwURrGBRoEyFHBT3GjUwaCQVKeNqqWxlcISGdw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260120221211-b8f7ae30c516 h1:sNrWoksmOyF5bvJUcnmbeAmQi8baNhqg5IWaI3llQqU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260120221211-b8f7ae30c516/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -874,8 +880,8 @@ google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTp google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.79.1 h1:zGhSi45ODB9/p3VAawt9a+O/MULLl9dpizzNNpq7flY= -google.golang.org/grpc v1.79.1/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= +google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= +google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -888,8 +894,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= -google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -932,14 +938,14 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.22.0/go.mod h1:0AoXXqst47OI/L0oGKq9DG61dvGRPXs7X4/B7KyjBCU= -k8s.io/api v0.34.2 h1:fsSUNZhV+bnL6Aqrp6O7lMTy6o5x2C4XLjnh//8SLYY= -k8s.io/api v0.34.2/go.mod h1:MMBPaWlED2a8w4RSeanD76f7opUoypY8TFYkSM+3XHw= +k8s.io/api v0.35.3 h1:pA2fiBc6+N9PDf7SAiluKGEBuScsTzd2uYBkA5RzNWQ= +k8s.io/api v0.35.3/go.mod h1:9Y9tkBcFwKNq2sxwZTQh1Njh9qHl81D0As56tu42GA4= k8s.io/apimachinery v0.22.0/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= -k8s.io/apimachinery v0.34.2 h1:zQ12Uk3eMHPxrsbUJgNF8bTauTVR2WgqJsTmwTE/NW4= -k8s.io/apimachinery v0.34.2/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= +k8s.io/apimachinery v0.35.3 h1:MeaUwQCV3tjKP4bcwWGgZ/cp/vpsRnQzqO6J6tJyoF8= +k8s.io/apimachinery v0.35.3/go.mod h1:jQCgFZFR1F4Ik7hvr2g84RTJSZegBc8yHgFWKn//hns= k8s.io/client-go v0.22.0/go.mod h1:GUjIuXR5PiEv/RVK5OODUsm6eZk7wtSWZSaSJbpFdGg= -k8s.io/client-go v0.34.2 h1:Co6XiknN+uUZqiddlfAjT68184/37PS4QAzYvQvDR8M= -k8s.io/client-go v0.34.2/go.mod h1:2VYDl1XXJsdcAxw7BenFslRQX28Dxz91U9MWKjX97fE= +k8s.io/client-go v0.35.3 h1:s1lZbpN4uI6IxeTM2cpdtrwHcSOBML1ODNTCCfsP1pg= +k8s.io/client-go v0.35.3/go.mod h1:RzoXkc0mzpWIDvBrRnD+VlfXP+lRzqQjCmKtiwZ8Q9c= k8s.io/component-base v0.22.0/go.mod h1:SXj6Z+V6P6GsBhHZVbWCw9hFjUdUYnJerlhhPnYCBCg= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= @@ -947,16 +953,16 @@ k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= -k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b h1:MloQ9/bdJyIu9lb1PzujOPolHyvO06MXG5TUIj2mNAA= -k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 h1:Y3gxNAuB0OBLImH611+UDZcmKS3g6CthxToOb37KgwE= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ= k8s.io/utils v0.0.0-20210707171843-4b05e18ac7d9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y= -k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 h1:SjGebBtkBqHFOli+05xYbK8YF1Dzkbzn+gDM4X9T4Ck= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= -sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= diff --git a/k8smock/k8sutils_mock.go b/k8smock/k8sutils_mock.go index 369102e7..7bb03f77 100644 --- a/k8smock/k8sutils_mock.go +++ b/k8smock/k8sutils_mock.go @@ -15,18 +15,21 @@ package k8smock import ( + "context" "reflect" "strings" "github.com/golang/mock/gomock" - kubernetes "k8s.io/client-go/kubernetes/fake" + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes" + kubernetesFake "k8s.io/client-go/kubernetes/fake" ) var mockUtils *MockUtils // MockUtils - mock kubernetes utils type MockUtils struct { - KubernetesClient *kubernetes.Clientset + KubernetesClient *kubernetesFake.Clientset } // Init - initializes the mock k8s utils @@ -34,7 +37,7 @@ func Init() *MockUtils { if mockUtils != nil { return mockUtils } - kubernetesClient := kubernetes.NewSimpleClientset() + kubernetesClient := kubernetesFake.NewSimpleClientset() mockUtils = &MockUtils{ KubernetesClient: kubernetesClient, } @@ -56,6 +59,16 @@ func (m *MockUtils) GetNodeIPs(nodeID string) string { return nodeElem[1] } +// GetPVCForVolume is mock implementation for GetPVCForVolume +func (m *MockUtils) GetPVCForVolume(_ context.Context, _ string, _ string) (*corev1.PersistentVolumeClaim, error) { + return nil, nil +} + +// GetClient is mock implementation for GetClient +func (m *MockUtils) GetClient() kubernetes.Interface { + return m.KubernetesClient +} + // Added a new mocking capability to help enable mocking this dynamically // MockUtilsInterface is a mock of UtilsInterface interface @@ -109,3 +122,32 @@ func (mr *MockUtilsInterfaceMockRecorder) GetNodeIPs(arg0 interface{}) *gomock.C mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNodeIPs", reflect.TypeOf((*MockUtilsInterface)(nil).GetNodeIPs), arg0) } + +// GetPVCForVolume mocks base method +func (m *MockUtilsInterface) GetPVCForVolume(ctx context.Context, pvName string, volumeID string) (*corev1.PersistentVolumeClaim, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetPVCForVolume", ctx, pvName, volumeID) + ret0, _ := ret[0].(*corev1.PersistentVolumeClaim) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetPVCForVolume indicates an expected call of GetPVCForVolume +func (mr *MockUtilsInterfaceMockRecorder) GetPVCForVolume(ctx, pvName, volumeID interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPVCForVolume", reflect.TypeOf((*MockUtilsInterface)(nil).GetPVCForVolume), ctx, pvName, volumeID) +} + +// GetClient mocks base method +func (m *MockUtilsInterface) GetClient() kubernetes.Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetClient") + ret0, _ := ret[0].(kubernetes.Interface) + return ret0 +} + +// GetClient indicates an expected call of GetClient +func (mr *MockUtilsInterfaceMockRecorder) GetClient() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetClient", reflect.TypeOf((*MockUtilsInterface)(nil).GetClient)) +} diff --git a/k8sutils/k8sutils.go b/k8sutils/k8sutils.go index b6d298c7..a03459f6 100644 --- a/k8sutils/k8sutils.go +++ b/k8sutils/k8sutils.go @@ -18,6 +18,7 @@ package k8sutils import ( "context" + "fmt" "strings" corev1 "k8s.io/api/core/v1" @@ -37,6 +38,8 @@ var log = csmlog.GetLogger() type UtilsInterface interface { GetNodeLabels(string) (map[string]string, error) GetNodeIPs(string) string + GetPVCForVolume(ctx context.Context, pvName string, volumeID string) (*corev1.PersistentVolumeClaim, error) + GetClient() kubernetes.Interface } // K8sUtils stores the configuration of the k8s client, k8s client and the informer @@ -69,6 +72,9 @@ func Init(kubeConfig string) (*K8sUtils, error) { return k8sUtils, nil } +// set this function as a var so that it can be mocked +var getInClusterConfigFunc = rest.InClusterConfig + // CreateKubeClientSet - Returns kubeClient set func CreateKubeClientSet(kubeConfig string) (*kubernetes.Clientset, error) { var clientSet *kubernetes.Clientset @@ -81,7 +87,7 @@ func CreateKubeClientSet(kubeConfig string) (*kubernetes.Clientset, error) { return nil, err } } else { - config, err = rest.InClusterConfig() + config, err = getInClusterConfigFunc() if err != nil { return nil, err } @@ -114,6 +120,14 @@ func (c *K8sUtils) GetNodeLabels(nodeFullName string) (map[string]string, error) return node.Labels, nil } +// GetClient returns the underlying kubernetes.Interface client. +func (c *K8sUtils) GetClient() kubernetes.Interface { + if c.KubernetesClient != nil { + return c.KubernetesClient.ClientSet + } + return nil +} + // GetNodeIPs returns cluster IP of the node object func (c *K8sUtils) GetNodeIPs(nodeID string) string { // access the API to fetch node object @@ -132,3 +146,30 @@ func (c *K8sUtils) GetNodeIPs(nodeID string) string { } return "" } + +// GetPVCForVolume retrieves the PVC bound to the given PV. +// It validates that the PV's CSI volume handle matches the provided volumeID. +func (c *K8sUtils) GetPVCForVolume(ctx context.Context, pvName string, volumeID string) (*corev1.PersistentVolumeClaim, error) { + // Get PV + pv, err := c.KubernetesClient.ClientSet.CoreV1().PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get PV %s: %w", pvName, err) + } + + // Validate CSI volume handle + if pv.Spec.CSI == nil || pv.Spec.CSI.VolumeHandle != volumeID { + return nil, fmt.Errorf("PV %s CSI volume handle does not match volume ID %s", pvName, volumeID) + } + + // Get PVC via ClaimRef + if pv.Spec.ClaimRef == nil { + return nil, fmt.Errorf("PV %s has no ClaimRef", pvName) + } + + pvc, err := c.KubernetesClient.ClientSet.CoreV1().PersistentVolumeClaims(pv.Spec.ClaimRef.Namespace).Get(ctx, pv.Spec.ClaimRef.Name, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get PVC %s/%s: %w", pv.Spec.ClaimRef.Namespace, pv.Spec.ClaimRef.Name, err) + } + + return pvc, nil +} diff --git a/k8sutils/k8sutils_test.go b/k8sutils/k8sutils_test.go index ec725f36..a077d861 100644 --- a/k8sutils/k8sutils_test.go +++ b/k8sutils/k8sutils_test.go @@ -18,6 +18,7 @@ package k8sutils import ( "context" + "errors" "os" "reflect" "testing" @@ -28,6 +29,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/rest" ) const kubeconfigFilepath = "./fake-kubeconfig" @@ -305,6 +307,10 @@ func Test_Init(t *testing.T) { want *K8sUtils wantErr bool } + + // store the original value for InClusterConfigFunc, so that we can restore it later + originalInClusterConfigFunc := getInClusterConfigFunc + tests := []test{ { name: "successfully initializes the kube client", @@ -319,11 +325,15 @@ func Test_Init(t *testing.T) { wantErr: false, }, { - name: "fails when given an empty kubeconfig file path", - args: args{kubeConfig: ""}, - before: func(_ test) error { return nil }, - after: func() {}, - wantErr: false, + name: "fails when given an empty kubeconfig file path", + args: args{kubeConfig: ""}, + // we want to TEMPORARILY make rest.InClusterConfig() return an error, to test that path + before: func(_ test) error { + getInClusterConfigFunc = func() (*rest.Config, error) { return nil, errors.New("error") } + return nil + }, + after: func() { getInClusterConfigFunc = originalInClusterConfigFunc }, + wantErr: true, }, { name: "returns existing k8s clientset", @@ -459,3 +469,116 @@ users: err := os.WriteFile(filepath, []byte(kubeconfig), 0o600) return err } + +// Test ID: U-028 +func TestGetPVCForVolume(t *testing.T) { + // Create fake PV and PVC + pv := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{Name: "test-pv"}, + Spec: corev1.PersistentVolumeSpec{ + PersistentVolumeSource: corev1.PersistentVolumeSource{ + CSI: &corev1.CSIPersistentVolumeSource{ + VolumeHandle: "vol-123", + Driver: "csi-powermax.dellemc.com", + }, + }, + ClaimRef: &corev1.ObjectReference{ + Name: "test-pvc", + Namespace: "default", + }, + }, + } + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pvc", + Namespace: "default", + Labels: map[string]string{ + "csi.dell.com/fs_check_enabled": "true", + "csi.dell.com/fs_check_mode": "checkAndRepair", + }, + }, + } + + fakeClient := fake.NewSimpleClientset(pv, pvc) + utils := &K8sUtils{ + KubernetesClient: &KubernetesClient{ + ClientSet: fakeClient, + }, + } + + result, err := utils.GetPVCForVolume(context.Background(), "test-pv", "vol-123") + assert.NoError(t, err, "GetPVCForVolume should succeed") + if assert.NotNil(t, result, "PVC should not be nil") { + assert.Equal(t, "test-pvc", result.Name) + assert.Equal(t, "default", result.Namespace) + assert.Equal(t, "true", result.Labels["csi.dell.com/fs_check_enabled"]) + assert.Equal(t, "checkAndRepair", result.Labels["csi.dell.com/fs_check_mode"]) + } +} + +// Test ID: U-029 +func TestGetPVCForVolumePVNotFound(t *testing.T) { + fakeClient := fake.NewSimpleClientset() // empty - no PV or PVC + utils := &K8sUtils{ + KubernetesClient: &KubernetesClient{ + ClientSet: fakeClient, + }, + } + + result, err := utils.GetPVCForVolume(context.Background(), "nonexistent-pv", "vol-999") + assert.Error(t, err, "GetPVCForVolume should fail when PV not found") + assert.Nil(t, result, "PVC should be nil when PV not found") +} + +// Test ID: U-030 +func TestGetPVCForVolumeIDMismatch(t *testing.T) { + pv := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{Name: "test-pv"}, + Spec: corev1.PersistentVolumeSpec{ + PersistentVolumeSource: corev1.PersistentVolumeSource{ + CSI: &corev1.CSIPersistentVolumeSource{ + VolumeHandle: "vol-123", + Driver: "csi-powermax.dellemc.com", + }, + }, + }, + } + + fakeClient := fake.NewSimpleClientset(pv) + utils := &K8sUtils{ + KubernetesClient: &KubernetesClient{ + ClientSet: fakeClient, + }, + } + + result, err := utils.GetPVCForVolume(context.Background(), "test-pv", "wrong-vol-id") + assert.Error(t, err, "GetPVCForVolume should fail when volume ID doesn't match") + assert.Nil(t, result, "PVC should be nil on volume ID mismatch") +} + +// Test ID: U-031 +func TestGetPVCForVolumeNoClaimRef(t *testing.T) { + pv := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{Name: "test-pv"}, + Spec: corev1.PersistentVolumeSpec{ + PersistentVolumeSource: corev1.PersistentVolumeSource{ + CSI: &corev1.CSIPersistentVolumeSource{ + VolumeHandle: "vol-123", + Driver: "csi-powermax.dellemc.com", + }, + }, + // No ClaimRef + }, + } + + fakeClient := fake.NewSimpleClientset(pv) + utils := &K8sUtils{ + KubernetesClient: &KubernetesClient{ + ClientSet: fakeClient, + }, + } + + result, err := utils.GetPVCForVolume(context.Background(), "test-pv", "vol-123") + assert.Error(t, err, "GetPVCForVolume should fail when PV has no ClaimRef") + assert.Nil(t, result, "PVC should be nil when no ClaimRef") +} diff --git a/pkg/symmetrix/mocks/pmaxclient.go b/pkg/symmetrix/mocks/pmaxclient.go index bce83a18..a7b85750 100644 --- a/pkg/symmetrix/mocks/pmaxclient.go +++ b/pkg/symmetrix/mocks/pmaxclient.go @@ -331,6 +331,26 @@ func (mr *MockPmaxClientMockRecorder) CreateStorageGroupSnapshot(arg0, arg1, arg return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateStorageGroupSnapshot", reflect.TypeOf((*MockPmaxClient)(nil).CreateStorageGroupSnapshot), arg0, arg1, arg2, arg3) } +// CreateVolume mocks base method. +func (m *MockPmaxClient) CreateVolume(arg0 context.Context, arg1 string, arg2 v100.CreateVolumesRequest, arg3 ...http.Header) (*v100.CreateVolumesResponse, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1, arg2} + for _, a := range arg3 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "CreateVolume", varargs...) + ret0, _ := ret[0].(*v100.CreateVolumesResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateVolume indicates an expected call of CreateVolume. +func (mr *MockPmaxClientMockRecorder) CreateVolume(arg0, arg1, arg2 interface{}, arg3 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1, arg2}, arg3...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateVolume", reflect.TypeOf((*MockPmaxClient)(nil).CreateVolume), varargs...) +} + // CreateVolumeInProtectedStorageGroupS mocks base method. func (m *MockPmaxClient) CreateVolumeInProtectedStorageGroupS(arg0 context.Context, arg1, arg2, arg3, arg4, arg5 string, arg6 interface{}, arg7 map[string]interface{}, arg8 ...http.Header) (*v100.Volume, error) { m.ctrl.T.Helper() @@ -796,6 +816,44 @@ func (mr *MockPmaxClientMockRecorder) GetHTTPClient() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetHTTPClient", reflect.TypeOf((*MockPmaxClient)(nil).GetHTTPClient)) } +// GetCustomHTTPHeaders mocks base method. +func (m *MockPmaxClient) GetCustomHTTPHeaders() http.Header { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCustomHTTPHeaders") + ret0, _ := ret[0].(http.Header) + return ret0 +} + +// GetCustomHTTPHeaders indicates an expected call of GetCustomHTTPHeaders. +func (mr *MockPmaxClientMockRecorder) GetCustomHTTPHeaders() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCustomHTTPHeaders", reflect.TypeOf((*MockPmaxClient)(nil).GetCustomHTTPHeaders)) +} + +// SetCustomHTTPHeaders mocks base method. +func (m *MockPmaxClient) SetCustomHTTPHeaders(arg0 http.Header) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "SetCustomHTTPHeaders", arg0) +} + +// SetCustomHTTPHeaders indicates an expected call of SetCustomHTTPHeaders. +func (mr *MockPmaxClientMockRecorder) SetCustomHTTPHeaders(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetCustomHTTPHeaders", reflect.TypeOf((*MockPmaxClient)(nil).SetCustomHTTPHeaders), arg0) +} + +// SetToken mocks base method. +func (m *MockPmaxClient) SetToken(arg0 string) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "SetToken", arg0) +} + +// SetToken indicates an expected call of SetToken. +func (mr *MockPmaxClientMockRecorder) SetToken(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetToken", reflect.TypeOf((*MockPmaxClient)(nil).SetToken), arg0) +} + // GetHostByID mocks base method. func (m *MockPmaxClient) GetHostByID(arg0 context.Context, arg1, arg2 string) (*v100.Host, error) { m.ctrl.T.Helper() @@ -871,6 +929,21 @@ func (mr *MockPmaxClientMockRecorder) GetISCSITargets(arg0, arg1 interface{}) *g return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetISCSITargets", reflect.TypeOf((*MockPmaxClient)(nil).GetISCSITargets), arg0, arg1) } +// GetISCSIEndpoints mocks base method. +func (m *MockPmaxClient) GetISCSIEndpoints(arg0 context.Context, arg1 string) ([]pmax.ISCSITarget, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetISCSIEndpoints", arg0, arg1) + ret0, _ := ret[0].([]pmax.ISCSITarget) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetISCSIEndpoints indicates an expected call of GetISCSIEndpoints. +func (mr *MockPmaxClientMockRecorder) GetISCSIEndpoints(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetISCSIEndpoints", reflect.TypeOf((*MockPmaxClient)(nil).GetISCSIEndpoints), arg0, arg1) +} + // GetInitiatorByID mocks base method. func (m *MockPmaxClient) GetInitiatorByID(arg0 context.Context, arg1, arg2 string) (*v100.Initiator, error) { m.ctrl.T.Helper() @@ -2082,6 +2155,21 @@ func (mr *MockPmaxClientMockRecorder) ModifyStorageGroupSnapshot(arg0, arg1, arg return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ModifyStorageGroupSnapshot", reflect.TypeOf((*MockPmaxClient)(nil).ModifyStorageGroupSnapshot), arg0, arg1, arg2, arg3, arg4, arg5) } +// PublishMaskingViews mocks base method. +func (m *MockPmaxClient) PublishMaskingViews(arg0 context.Context, arg1 string, arg2 *v100.PublishMaskingViewsParam) (*v100.PublishMaskingViewResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PublishMaskingViews", arg0, arg1, arg2) + ret0, _ := ret[0].(*v100.PublishMaskingViewResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// PublishMaskingViews indicates an expected call of PublishMaskingViews. +func (mr *MockPmaxClientMockRecorder) PublishMaskingViews(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PublishMaskingViews", reflect.TypeOf((*MockPmaxClient)(nil).PublishMaskingViews), arg0, arg1, arg2) +} + // RefreshSymmetrix mocks base method. func (m *MockPmaxClient) RefreshSymmetrix(arg0 context.Context, arg1 string) error { m.ctrl.T.Helper() diff --git a/provider/provider.go b/provider/provider.go index dc97607d..68e22ec3 100644 --- a/provider/provider.go +++ b/provider/provider.go @@ -1,5 +1,5 @@ /* - Copyright © 2021 Dell Inc. or its subsidiaries. All Rights Reserved. + Copyright © 2021-2026 Dell Inc. or its subsidiaries. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -47,6 +47,7 @@ func New() gocsi.StoragePluginProvider { svc := service.New() return &gocsi.StoragePlugin{ Controller: svc, + GroupController: svc, Identity: svc, Node: svc, BeforeServe: svc.BeforeServe, diff --git a/service/backend_selection.go b/service/backend_selection.go new file mode 100644 index 00000000..ec407a70 --- /dev/null +++ b/service/backend_selection.go @@ -0,0 +1,234 @@ +package service + +import ( + "context" + "path" + "strconv" + "strings" + + pmax "github.com/dell/gopowermax/v2" + "github.com/container-storage-interface/spec/lib/go/csi" +) + +// --------------------------------------------------------------------------- +// Backend path selection +// --------------------------------------------------------------------------- + +type backendPath int + +const ( + backendLegacy backendPath = iota + backendU4P104 +) + +// Unisphere API version constants. Use these instead of raw integers +// when branching on the API version so that every version-dependent +// check references a single, well-known constant. +const ( + APIVersion101 = 101 + APIVersion103 = 103 + APIVersion104 = 104 +) + +func (p backendPath) String() string { + switch p { + case backendLegacy: + return "legacy" + case backendU4P104: + return "u4p104" + default: + return "unknown" + } +} + +var minU4PBackendVersion = []int{10, 4, 0, 4} + +func parseUnisphereVersion(version string) ([]int, bool) { + trimmed := strings.TrimSpace(version) + if trimmed == "" { + return nil, false + } + trimmed = strings.TrimPrefix(strings.TrimPrefix(trimmed, "V"), "v") + parts := strings.Split(trimmed, ".") + if len(parts) < len(minU4PBackendVersion) { + return nil, false + } + parsed := make([]int, len(parts)) + for i, part := range parts { + value, err := strconv.Atoi(part) + if err != nil { + return nil, false + } + parsed[i] = value + } + return parsed, true +} + +func isVersionAtLeast(version string, minimum []int) bool { + parsed, ok := parseUnisphereVersion(version) + if !ok { + return false + } + for i := 0; i < len(minimum); i++ { + if parsed[i] > minimum[i] { + return true + } + if parsed[i] < minimum[i] { + return false + } + } + return true +} + +// selectBackendPath determines whether to use the 10.4 or legacy code path +// based on the Unisphere version and the operation profile. NFS, +// replication, and thick provisioning are not yet supported on the 10.4 path, +// so any request that involves them is routed to legacy regardless of the version. +func selectBackendPath(profile backendSelectionProfile, version string) backendPath { + if profile.IsFile || profile.ReplicationEnabled || profile.IsThick { + return backendLegacy + } + if isVersionAtLeast(version, minU4PBackendVersion) { + return backendU4P104 + } + return backendLegacy +} + +// --------------------------------------------------------------------------- +// Selection profiles +// --------------------------------------------------------------------------- + +type backendSelectionProfile struct { + IsFile bool + HasContentSource bool + ReplicationEnabled bool + ReplicationMode string + IsThick bool +} + +func createVolumeSelectionProfile(isFile bool, contentSource *csi.VolumeContentSource, replicationEnabled bool, repMode string, isThick bool) backendSelectionProfile { + return backendSelectionProfile{ + IsFile: isFile, + HasContentSource: contentSource != nil, + ReplicationEnabled: replicationEnabled, + ReplicationMode: repMode, + IsThick: isThick, + } +} + +func publishVolumeSelectionProfile(isFile bool, remoteSymID string) backendSelectionProfile { + return backendSelectionProfile{ + IsFile: isFile, + HasContentSource: false, + ReplicationEnabled: remoteSymID != "", + ReplicationMode: "", + } +} + +// --------------------------------------------------------------------------- +// Interfaces +// --------------------------------------------------------------------------- + +type volumeCreator interface { + Create(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) +} + +type volumePublisher interface { + Publish(ctx context.Context, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) +} + +type U4P104ServiceDeps interface { + resolveParameter(params map[string]string, symID, key, defaultVal string) string + isDynamicSGEnabled() bool + getReplicationPrefix() string + getReplicationContextPrefix() string + getClusterPrefix() string + parseCsiID(csiID string) (volName, arrayID, devID, remoteSymID, remoteVolID string, err error) + isSnapshotLicensed(ctx context.Context, symID string, pmaxClient pmax.Pmax) error + getDynamicSG(ctx context.Context, arrayID, baseSGName string) (string, bool, error) + getStorageArrayLabels(arrayID string) map[string]string + isBlockEnabled() bool +} + +// --------------------------------------------------------------------------- +// Factory functions +// --------------------------------------------------------------------------- + +func creatorFor(s *service, pmaxClient pmax.Pmax, pmaxClient104 pmax.Pmax, symmetrixID, reqID string, params map[string]string, symmIDFoundInAZ bool, version string, apiVersion int, vcs []*csi.VolumeCapability, contentSource *csi.VolumeContentSource) (backendPath, volumeCreator) { + isFile := vcs != nil && accTypeIsNFS(vcs) + replicationEnabled := params != nil && params[path.Join(s.getReplicationPrefix(), RepEnabledParam)] == "true" + isThick := params != nil && params[ThickVolumesParam] == "true" + profile := createVolumeSelectionProfile(isFile, contentSource, replicationEnabled, "", isThick) + selectedPath := selectBackendPath(profile, version) + + if selectedPath == backendU4P104 { + u4p104 := &u4p104VolumeCreator{ + s: s, + pmaxClient: pmaxClient, + pmaxClient104: pmaxClient104, + symmetrixID: symmetrixID, + reqID: reqID, + params: params, + symmIDFoundInAZ: symmIDFoundInAZ, + apiVersion: apiVersion, + } + return selectedPath, u4p104 + } + legacy := &legacyVolumeCreator{ + s: s, + pmaxClient: pmaxClient, + symmetrixID: symmetrixID, + reqID: reqID, + params: params, + symmIDFoundInAZ: symmIDFoundInAZ, + apiVersion: apiVersion, + } + return selectedPath, legacy +} + +func publisherFor(ctx context.Context, s *service, pmaxClient pmax.Pmax, pmaxClient104 pmax.Pmax, symID, devID, volID, volumeName, remoteSymID, remoteVolumeID, reqID string, version string, vc *csi.VolumeCapability) volumePublisher { + isFile := vc != nil && accTypeIsNFS([]*csi.VolumeCapability{vc}) + profile := publishVolumeSelectionProfile(isFile, remoteSymID) + selectedPath := selectBackendPath(profile, version) + + // vSphere uses host groups and fixed naming conventions that don't map + // cleanly to the 10.4 PublishMaskingViews API — route to legacy. + if selectedPath == backendU4P104 && s.opts.IsVsphereEnabled { + selectedPath = backendLegacy + } + + // Scope gate: static provisioning of a file system — when fsType is empty + // and a file system already exists on the device, fall back to legacy. + if selectedPath == backendU4P104 && vc != nil && vc.GetMount().GetFsType() == "" { + _, err := pmaxClient.GetFileSystemByID(ctx, symID, devID) + if err == nil { + selectedPath = backendLegacy + } + } + + if selectedPath == backendU4P104 { + return &u4p104VolumePublisher{ + s: s, + legacyClient: pmaxClient, + client104: pmaxClient104, + symID: symID, + devID: devID, + volID: volID, + volumeName: volumeName, + remoteSymID: remoteSymID, + remoteVolumeID: remoteVolumeID, + reqID: reqID, + } + } + return &legacyVolumePublisher{ + s: s, + pmaxClient: pmaxClient, + symID: symID, + devID: devID, + volID: volID, + volumeName: volumeName, + remoteSymID: remoteSymID, + remoteVolumeID: remoteVolumeID, + reqID: reqID, + } +} diff --git a/service/backend_selection_test.go b/service/backend_selection_test.go new file mode 100644 index 00000000..9cb6e441 --- /dev/null +++ b/service/backend_selection_test.go @@ -0,0 +1,356 @@ +package service + +import ( + "context" + "testing" + + "github.com/dell/csi-powermax/v2/pkg/symmetrix/mocks" + types "github.com/dell/gopowermax/v2/types/v100" + "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" +) + +func TestCreateVolumeSelectionProfile(t *testing.T) { + tests := []struct { + name string + isFile bool + contentSource *csi.VolumeContentSource + replicationEnabled bool + repMode string + isThick bool + expected backendSelectionProfile + }{ + { + name: "block volume no content source no replication", + isFile: false, + contentSource: nil, + replicationEnabled: false, + repMode: "", + isThick: false, + expected: backendSelectionProfile{ + IsFile: false, + HasContentSource: false, + ReplicationEnabled: false, + ReplicationMode: "", + IsThick: false, + }, + }, + { + name: "block volume with content source", + isFile: false, + contentSource: &csi.VolumeContentSource{Type: &csi.VolumeContentSource_Snapshot{}}, + replicationEnabled: false, + repMode: "", + isThick: false, + expected: backendSelectionProfile{ + IsFile: false, + HasContentSource: true, + ReplicationEnabled: false, + ReplicationMode: "", + IsThick: false, + }, + }, + { + name: "file volume replication", + isFile: true, + contentSource: nil, + replicationEnabled: true, + repMode: "METRO", + isThick: false, + expected: backendSelectionProfile{ + IsFile: true, + HasContentSource: false, + ReplicationEnabled: true, + ReplicationMode: "METRO", + IsThick: false, + }, + }, + { + name: "thick volume", + isFile: false, + contentSource: nil, + replicationEnabled: false, + repMode: "", + isThick: true, + expected: backendSelectionProfile{ + IsFile: false, + HasContentSource: false, + ReplicationEnabled: false, + ReplicationMode: "", + IsThick: true, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + profile := createVolumeSelectionProfile(tt.isFile, tt.contentSource, tt.replicationEnabled, tt.repMode, tt.isThick) + assert.Equal(t, tt.expected, profile) + }) + } +} + +func TestPublishVolumeSelectionProfile(t *testing.T) { + tests := []struct { + name string + isFile bool + remoteSymID string + expected backendSelectionProfile + }{ + { + name: "block volume not replicated", + isFile: false, + remoteSymID: "", + expected: backendSelectionProfile{ + IsFile: false, + HasContentSource: false, + ReplicationEnabled: false, + ReplicationMode: "", + }, + }, + { + name: "block volume replicated", + isFile: false, + remoteSymID: "000123456789", + expected: backendSelectionProfile{ + IsFile: false, + HasContentSource: false, + ReplicationEnabled: true, + ReplicationMode: "", + }, + }, + { + name: "file volume", + isFile: true, + remoteSymID: "", + expected: backendSelectionProfile{ + IsFile: true, + HasContentSource: false, + ReplicationEnabled: false, + ReplicationMode: "", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + profile := publishVolumeSelectionProfile(tt.isFile, tt.remoteSymID) + assert.Equal(t, tt.expected, profile) + }) + } +} + +func TestSelectBackendPath(t *testing.T) { + empty := backendSelectionProfile{} + assert.Equal(t, backendLegacy, selectBackendPath(empty, "10.4.0.3")) + assert.Equal(t, backendU4P104, selectBackendPath(empty, "10.4.0.4")) + assert.Equal(t, backendU4P104, selectBackendPath(empty, "10.4.1.0")) + assert.Equal(t, backendLegacy, selectBackendPath(empty, "")) +} + +func TestSelectBackendPath_ForcesLegacyForNFS(t *testing.T) { + nfs := backendSelectionProfile{IsFile: true} + assert.Equal(t, backendLegacy, selectBackendPath(nfs, "10.4.0.4")) + assert.Equal(t, backendLegacy, selectBackendPath(nfs, "99.0.0.0")) +} + +func TestSelectBackendPath_ForcesLegacyForReplication(t *testing.T) { + repl := backendSelectionProfile{ReplicationEnabled: true} + assert.Equal(t, backendLegacy, selectBackendPath(repl, "10.4.0.4")) + assert.Equal(t, backendLegacy, selectBackendPath(repl, "99.0.0.0")) +} + +func TestCreatorFor_SelectsU4P104CreatorWhenVersionIs104OrHigher(t *testing.T) { + path, creator := creatorFor(&service{}, nil, nil, "000000000001", "req-1", map[string]string{}, false, "10.4.0.4", APIVersion104, nil, nil) + assert.Equal(t, backendU4P104, path) + assert.IsType(t, &u4p104VolumeCreator{}, creator) +} + +func TestCreatorFor_SelectsLegacyCreatorWhenVersionIsLessThan104OrUnknown(t *testing.T) { + path, creator := creatorFor(&service{}, nil, nil, "000000000001", "req-1", map[string]string{}, false, "10.4.0.3", 103, nil, nil) + assert.Equal(t, backendLegacy, path) + assert.NotNil(t, creator) + + path, creator = creatorFor(&service{}, nil, nil, "000000000001", "req-1", map[string]string{}, false, "", 0, nil, nil) + assert.Equal(t, backendLegacy, path) + assert.NotNil(t, creator) +} + +func TestCreatorFor_SelectsLegacyForNFS(t *testing.T) { + nfsCaps := []*csi.VolumeCapability{{ + AccessType: &csi.VolumeCapability_Mount{Mount: &csi.VolumeCapability_MountVolume{FsType: NFS}}, + }} + path, creator := creatorFor(&service{}, nil, nil, "000000000001", "req-1", map[string]string{}, false, "10.4.0.4", APIVersion104, nfsCaps, nil) + assert.Equal(t, backendLegacy, path) + assert.IsType(t, &legacyVolumeCreator{}, creator) +} + +func TestCreatorFor_SelectsLegacyForReplication(t *testing.T) { + params := map[string]string{RepEnabledParam: "true"} + path, creator := creatorFor(&service{}, nil, nil, "000000000001", "req-1", params, false, "10.4.0.4", APIVersion104, nil, nil) + assert.Equal(t, backendLegacy, path) + assert.IsType(t, &legacyVolumeCreator{}, creator) +} + +func TestPublisherFor_SelectsU4P104PublisherWhenVersionIs104OrHigher(t *testing.T) { + vc := &csi.VolumeCapability{ + AccessType: &csi.VolumeCapability_Mount{Mount: &csi.VolumeCapability_MountVolume{FsType: "ext4"}}, + } + publisher := publisherFor(context.Background(), &service{}, nil, nil, "000000000001", "00001", "vol-id", "vol-name", "", "", "req-1", "10.4.0.4", vc) + assert.IsType(t, &u4p104VolumePublisher{}, publisher) +} + +func TestPublisherFor_SelectsLegacyPublisherWhenVersionIsLessThan104OrUnknown(t *testing.T) { + publisher := publisherFor(context.Background(), &service{}, nil, nil, "000000000001", "00001", "vol-id", "vol-name", "", "", "req-1", "10.4.0.3", nil) + assert.IsType(t, &legacyVolumePublisher{}, publisher) + + publisher = publisherFor(context.Background(), &service{}, nil, nil, "000000000001", "00001", "vol-id", "vol-name", "", "", "req-1", "", nil) + assert.IsType(t, &legacyVolumePublisher{}, publisher) +} + +func TestPublisherFor_SelectsLegacyForNFS(t *testing.T) { + nfsCap := &csi.VolumeCapability{ + AccessType: &csi.VolumeCapability_Mount{Mount: &csi.VolumeCapability_MountVolume{FsType: NFS}}, + } + publisher := publisherFor(context.Background(), &service{}, nil, nil, "000000000001", "00001", "vol-id", "vol-name", "", "", "req-1", "10.4.0.4", nfsCap) + assert.IsType(t, &legacyVolumePublisher{}, publisher) +} + +func TestPublisherFor_SelectsLegacyForReplication(t *testing.T) { + publisher := publisherFor(context.Background(), &service{}, nil, nil, "000000000001", "00001", "vol-id", "vol-name", "000120000002", "011BC", "req-1", "10.4.0.4", nil) + assert.IsType(t, &legacyVolumePublisher{}, publisher) +} + +func TestPublisherFor_SelectsLegacyForStaticFileSystem(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockClient := mocks.NewMockPmaxClient(ctrl) + // GetFileSystemByID returns success → file system exists → should select legacy + mockClient.EXPECT().GetFileSystemByID(gomock.Any(), "000120000001", "011AB"). + Return(&types.FileSystem{}, nil).Times(1) + // fsType is empty → triggers static FS check + vc := &csi.VolumeCapability{ + AccessType: &csi.VolumeCapability_Mount{Mount: &csi.VolumeCapability_MountVolume{FsType: ""}}, + } + publisher := publisherFor(context.Background(), &service{}, mockClient, nil, "000120000001", "011AB", "vol-id", "vol-name", "", "", "req-1", "10.4.0.4", vc) + assert.IsType(t, &legacyVolumePublisher{}, publisher) +} + +func TestPublisherFor_InjectsDependenciesIntoLegacyPublisher(t *testing.T) { + svc := &service{} + publisher := publisherFor(context.Background(), svc, nil, nil, "SYM001", "DEV01", "vol-1", "myVol", "RSYM", "RVOL", "req-42", "10.4.0.3", nil) + lp, ok := publisher.(*legacyVolumePublisher) + assert.True(t, ok) + assert.Equal(t, svc, lp.s) + assert.Equal(t, "SYM001", lp.symID) + assert.Equal(t, "DEV01", lp.devID) + assert.Equal(t, "vol-1", lp.volID) + assert.Equal(t, "myVol", lp.volumeName) + assert.Equal(t, "RSYM", lp.remoteSymID) + assert.Equal(t, "RVOL", lp.remoteVolumeID) + assert.Equal(t, "req-42", lp.reqID) +} + +func TestPublisherFor_InjectsDependenciesIntoU4P104Publisher(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + svc := &service{} + mockClient := mocks.NewMockPmaxClient(ctrl) + mockClient104 := mocks.NewMockPmaxClient(ctrl) + vc := &csi.VolumeCapability{ + AccessType: &csi.VolumeCapability_Mount{Mount: &csi.VolumeCapability_MountVolume{FsType: "ext4"}}, + } + publisher := publisherFor(context.Background(), svc, mockClient, mockClient104, "SYM001", "DEV01", "vol-1", "myVol", "", "", "req-42", "10.4.0.4", vc) + up, ok := publisher.(*u4p104VolumePublisher) + assert.True(t, ok) + assert.Equal(t, svc, up.s) + assert.Equal(t, mockClient, up.legacyClient) + assert.Equal(t, mockClient104, up.client104) + assert.Equal(t, "SYM001", up.symID) + assert.Equal(t, "DEV01", up.devID) + assert.Equal(t, "vol-1", up.volID) + assert.Equal(t, "myVol", up.volumeName) + assert.Equal(t, "", up.remoteSymID) + assert.Equal(t, "", up.remoteVolumeID) + assert.Equal(t, "req-42", up.reqID) +} + +func TestSelectBackendPath_ForcesLegacyForThickVolumes(t *testing.T) { + thick := backendSelectionProfile{IsThick: true} + assert.Equal(t, backendLegacy, selectBackendPath(thick, "10.4.0.4")) + assert.Equal(t, backendLegacy, selectBackendPath(thick, "99.0.0.0")) +} + +func TestCreatorFor_SelectsLegacyForThickVolumes(t *testing.T) { + params := map[string]string{ThickVolumesParam: "true"} + path, creator := creatorFor(&service{}, nil, nil, "000000000001", "req-1", params, false, "10.4.0.4", 104, nil, nil) + assert.Equal(t, backendLegacy, path) + assert.IsType(t, &legacyVolumeCreator{}, creator) +} + +func TestCreatorFor_SelectsU4P104WhenThickIsFalseOrAbsent(t *testing.T) { + // ThickVolumesParam explicitly false + params := map[string]string{ThickVolumesParam: "false"} + path, creator := creatorFor(&service{}, nil, nil, "000000000001", "req-1", params, false, "10.4.0.4", 104, nil, nil) + assert.Equal(t, backendU4P104, path) + assert.IsType(t, &u4p104VolumeCreator{}, creator) + + // ThickVolumesParam absent + path2, creator2 := creatorFor(&service{}, nil, nil, "000000000001", "req-1", map[string]string{}, false, "10.4.0.4", 104, nil, nil) + assert.Equal(t, backendU4P104, path2) + assert.IsType(t, &u4p104VolumeCreator{}, creator2) +} + +func TestPublisherFor_SelectsLegacyForVSphere(t *testing.T) { + svc := &service{} + svc.opts.IsVsphereEnabled = true + vc := &csi.VolumeCapability{ + AccessType: &csi.VolumeCapability_Mount{Mount: &csi.VolumeCapability_MountVolume{FsType: "ext4"}}, + } + publisher := publisherFor(context.Background(), svc, nil, nil, "000000000001", "00001", "vol-id", "vol-name", "", "", "req-1", "10.4.0.4", vc) + assert.IsType(t, &legacyVolumePublisher{}, publisher) +} + +func TestPublisherFor_SelectsU4P104WhenVSphereDisabled(t *testing.T) { + svc := &service{} + svc.opts.IsVsphereEnabled = false + vc := &csi.VolumeCapability{ + AccessType: &csi.VolumeCapability_Mount{Mount: &csi.VolumeCapability_MountVolume{FsType: "ext4"}}, + } + publisher := publisherFor(context.Background(), svc, nil, nil, "000000000001", "00001", "vol-id", "vol-name", "", "", "req-1", "10.4.0.4", vc) + assert.IsType(t, &u4p104VolumePublisher{}, publisher) +} + +func TestParseUnisphereVersion(t *testing.T) { + tests := []struct { + name string + version string + expected []int + ok bool + }{ + {name: "prefixed version", version: "V10.4.0.4", expected: []int{10, 4, 0, 4}, ok: true}, + {name: "non-prefixed version", version: "10.4.0.4", expected: []int{10, 4, 0, 4}, ok: true}, + {name: "extra components", version: "10.4.0.4.99", expected: []int{10, 4, 0, 4, 99}, ok: true}, + {name: "empty version", version: "", expected: nil, ok: false}, + {name: "malformed version", version: "10.4.x.4", expected: nil, ok: false}, + {name: "too short version", version: "10.4.0", expected: nil, ok: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actual, ok := parseUnisphereVersion(tt.version) + assert.Equal(t, tt.ok, ok) + assert.Equal(t, tt.expected, actual) + }) + } +} + +func TestIsVersionAtLeast(t *testing.T) { + minimum := []int{10, 4, 0, 4} + assert.True(t, isVersionAtLeast("10.4.0.4", minimum)) + assert.True(t, isVersionAtLeast("V10.4.1.0", minimum)) + assert.False(t, isVersionAtLeast("10.4.0.3", minimum)) + assert.False(t, isVersionAtLeast("", minimum)) + assert.False(t, isVersionAtLeast("10.4.bad.4", minimum)) +} diff --git a/service/controller.go b/service/controller.go index 71f4c33a..3fa3d32e 100644 --- a/service/controller.go +++ b/service/controller.go @@ -15,6 +15,7 @@ package service import ( + "context" "encoding/base64" "errors" "fmt" @@ -35,7 +36,6 @@ import ( csiext "github.com/dell/dell-csi-extensions/replication" - "golang.org/x/net/context" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" @@ -174,14 +174,12 @@ type pmaxCachedInformation struct { // Pair of a map containing dirPortKey to portIdentifier mapping // and a timestamp indicating when this map was created portIdentifiers *Pair - uCodeVersion *Pair } // Initializes a pmaxCachedInformation type func (p *pmaxCachedInformation) initialize() { p.knownStoragePools = make(map[string]time.Time) p.portIdentifiers = nil - p.uCodeVersion = nil } func getPmaxCache(symID string) *pmaxCachedInformation { @@ -215,6 +213,16 @@ var ( getMVConnectionsDelay = 30 * time.Second ) +// isValidSLO checks whether the given service level is one of the known-valid values. +func isValidSLO(slo string) bool { + for _, val := range validSLO { + if slo == val { + return true + } + } + return false +} + func (s *service) GetPortIdentifier(ctx context.Context, symID string, dirPortKey string, pmaxClient pmax.Pmax) (string, error) { s.cacheMutex.Lock() defer s.cacheMutex.Unlock() @@ -437,824 +445,32 @@ func (s *service) CreateVolume( return nil, err } - thick := params[ThickVolumesParam] - applicationPrefix := s.resolveParameter(params, symmetrixID, ApplicationPrefixParam, "") - - // Storage (resource) Pool. Validate it against exist Pools - storagePoolID := s.resolveParameter(params, symmetrixID, StoragePoolParam, "") - err = s.validateStoragePoolID(ctx, symmetrixID, storagePoolID, pmaxClient) - if err != nil { - log.Error(err.Error()) - return nil, status.Errorf(codes.InvalidArgument, "%s", err.Error()) - } - - // SLO is optional - serviceLevel := s.resolveParameter(params, symmetrixID, ServiceLevelParam, "Optimized") - found := false - for _, val := range validSLO { - if serviceLevel == val { - found = true - break - } - } - if !found { - log.Error("An invalid Service Level parameter was specified") - return nil, status.Errorf(codes.InvalidArgument, "An invalid Service Level parameter was specified") - } - - storageGroupName := s.resolveParameter(params, symmetrixID, StorageGroupParam, "") - hostLimitName := s.resolveParameter(params, symmetrixID, HostLimitNameParam, "") - hostMBsec := s.resolveParameter(params, symmetrixID, HostIOLimitMBSecParam, "") - hostIOsec := s.resolveParameter(params, symmetrixID, HostIOLimitIOSecParam, "") - hostDynDistribution := s.resolveParameter(params, symmetrixID, DynamicDistributionParam, "") - namespace := s.resolveParameter(params, "", CSIPVCNamespace, "") - - versionDetails, err := pmaxClient.GetVersionDetails(ctx) + versionDetails, err := s.getVersionCache().getOrFetchVersionDetails(ctx, symmetrixID, pmaxClient) if err != nil { log.Error("Error in getversion API " + err.Error()) return nil, status.Errorf(codes.Internal, "Error in getVersion API: %s", err.Error()) } - var version int + version := versionDetails.Version + var apiVersion int if versionDetails.APIVersion != "" { - version, err = strconv.Atoi(versionDetails.APIVersion) + apiVersion, err = strconv.Atoi(versionDetails.APIVersion) if err != nil { log.Error("Error parsing getVersion" + err.Error()) return nil, status.Errorf(codes.Internal, "Error in parsing getVersion: %s", err.Error()) } } - // Dynamic SG check is available only from 10.1 - if s.opts.dynamicSGEnabled && version < 101 { - log.Errorf("Dynamic SG is enabled, but not supported for array %s with version %d. Minimum expected array version is 10.1", symmetrixID, version) - return nil, status.Errorf(codes.Internal, "Dynamic SG is enabled, but not supported for array %s with version %d. Minimum expected array version is 10.1", symmetrixID, version) - } - - // File related params - useNFS := false - nasServer := "" - allowRoot := "" - if params[NASServerName] != "" { - nasServer = params[NASServerName] - } - if params[AllowRootParam] != "" { - allowRoot = params[AllowRootParam] - } - - // Validate volume capabilities - vcs := req.GetVolumeCapabilities() - if vcs != nil { - isBlock := accTypeIsBlock(vcs) - if isBlock && !s.opts.EnableBlock { - return nil, status.Error(codes.InvalidArgument, "Block Volume Capability is not supported") - } - useNFS = accTypeIsNFS(vcs) - if isBlock && useNFS { - return nil, status.Errorf(codes.InvalidArgument, "NFS with Block is not supported") - } - } - - // Remote Replication based paramsMes - var replicationEnabled string - var remoteSymID string - var localRDFGrpNo string - var remoteRDFGrpNo string - var remoteServiceLevel string - var remoteSRPID string - var repMode string - var bias string - - if params[path.Join(s.opts.ReplicationPrefix, RepEnabledParam)] == "true" { - if s.opts.IsVsphereEnabled { - return nil, status.Errorf(codes.Unavailable, "Replication on a vSphere volume is not supported") - } - if useNFS { - return nil, status.Errorf(codes.Unavailable, "Replication on a NFS volume is not supported") - } - replicationEnabled = params[path.Join(s.opts.ReplicationPrefix, RepEnabledParam)] - // remote symmetrix ID and rdf group name are mandatory params when replication is enabled - remoteSymID = params[path.Join(s.opts.ReplicationPrefix, RemoteSymIDParam)] - // check if storage class contains SRDG details - if params[path.Join(s.opts.ReplicationPrefix, LocalRDFGroupParam)] != "" { - localRDFGrpNo = params[path.Join(s.opts.ReplicationPrefix, LocalRDFGroupParam)] - } - if params[path.Join(s.opts.ReplicationPrefix, RemoteRDFGroupParam)] != "" { - remoteRDFGrpNo = params[path.Join(s.opts.ReplicationPrefix, RemoteRDFGroupParam)] - } - repMode = params[path.Join(s.opts.ReplicationPrefix, ReplicationModeParam)] - - if symmIDFoundInAZ && repMode == Metro { - return nil, status.Errorf(codes.InvalidArgument, "The use of Availability Zones with Metro volumes is not supported") - } - - remoteServiceLevel = params[path.Join(s.opts.ReplicationPrefix, RemoteServiceLevelParam)] - remoteSRPID = params[path.Join(s.opts.ReplicationPrefix, RemoteSRPParam)] - bias = params[path.Join(s.opts.ReplicationPrefix, BiasParam)] - - // Get Local and remote RDFg Numbers from a rest call - // Create RDFg for a namespace if it doens't exist? - // Create RDFg when the volume gets added first time for a replication sssn - if localRDFGrpNo == "" && remoteRDFGrpNo == "" { - localRDFGrpNo, remoteRDFGrpNo, err = s.GetOrCreateRDFGroup(ctx, symmetrixID, remoteSymID, repMode, namespace, pmaxClient) - if err != nil { - return nil, status.Errorf(codes.NotFound, "Received error get/create RDFG, err: %s", err.Error()) - } - if localRDFGrpNo == "" || remoteRDFGrpNo == "" { - return nil, status.Errorf(codes.Unavailable, "Can not fetch RDF Group for volume creation get/create RDFG") - } - log.Debugf("RDF group for given array pair and RDF mode: local(%s), remote(%s)", localRDFGrpNo, remoteRDFGrpNo) - } - if repMode == Metro { - return s.createMetroVolume(ctx, req, reqID, storagePoolID, symmetrixID, storageGroupName, serviceLevel, thick, remoteSymID, localRDFGrpNo, remoteRDFGrpNo, remoteServiceLevel, remoteSRPID, namespace, applicationPrefix, bias, hostLimitName, hostMBsec, hostIOsec, hostDynDistribution) - } - if repMode != Async && repMode != Sync { - log.Errorf("Unsupported Replication Mode: (%s)", repMode) - return nil, status.Errorf(codes.InvalidArgument, "Unsupported Replication Mode: (%s)", repMode) - } - } - - // Get the required capacity - cr := req.GetCapacityRange() - requiredCylinders, err := s.validateVolSize(ctx, cr, symmetrixID, storagePoolID, pmaxClient) - if err != nil { - return nil, err - } - - var srcVolID, srcSnapID string - var symID, SrcDevID, snapID string - var srcVol *types.Volume - var volContent string - // When content source is specified, the size of the new volume - // is determined based on the size of the source volume in the - // snapshot. The size of the new volume to be created should be - // greater than or equal to the size of snapshot source - contentSource := req.GetVolumeContentSource() - if contentSource != nil { - if useNFS { - return nil, status.Errorf(codes.Unavailable, "Cloning on a NFS volume is not supported") - } - switch req.GetVolumeContentSource().GetType().(type) { - case *csi.VolumeContentSource_Volume: - srcVolID = req.GetVolumeContentSource().GetVolume().GetVolumeId() - if srcVolID != "" { - _, symID, SrcDevID, _, _, err = s.parseCsiID(srcVolID) - if err != nil { - // We couldn't comprehend the identifier. - log.Error("Could not parse CSI VolumeId: " + srcVolID) - return nil, status.Error(codes.InvalidArgument, "Source volume identifier not in supported format") - } - volContent = srcVolID - } - break - case *csi.VolumeContentSource_Snapshot: - srcSnapID = req.GetVolumeContentSource().GetSnapshot().GetSnapshotId() - if srcSnapID != "" { - snapID, symID, SrcDevID, _, _, err = s.parseCsiID(srcSnapID) - if err != nil { - // We couldn't comprehend the identifier. - log.Error("Snapshot identifier not in supported format: " + srcSnapID) - return nil, status.Error(codes.InvalidArgument, "Snapshot identifier not in supported format") - } - volContent = snapID - } - break - default: - return nil, status.Error(codes.InvalidArgument, "VolumeContentSource is missing volume and snapshot source") - } - // check snapshot is licensed - if err := s.IsSnapshotLicensed(ctx, symID, pmaxClient); err != nil { - log.Error("Error - " + err.Error()) - return nil, status.Error(codes.Internal, err.Error()) - } - } - - if SrcDevID != "" && symID != "" { - if symID != symmetrixID { - log.Error("The volume content source is in different PowerMax array") - return nil, status.Errorf(codes.InvalidArgument, "The volume content source is in different PowerMax array") - } - srcVol, err = pmaxClient.GetVolumeByID(ctx, symmetrixID, SrcDevID) - if err != nil { - log.Error("Volume content source volume couldn't be found in the array: " + err.Error()) - return nil, status.Errorf(codes.InvalidArgument, "Volume content source volume couldn't be found in the array: %s", err.Error()) - } - // reset the volume size to match with source - if requiredCylinders < srcVol.CapacityCYL { - log.Error("Capacity specified is smaller than the source") - return nil, status.Error(codes.InvalidArgument, "Requested capacity is smaller than the source") - } - } - - // Get the volume name - volumeName := req.GetName() - if volumeName == "" { - log.Error("Name cannot be empty") - return nil, status.Error(codes.InvalidArgument, - "Name cannot be empty") - } - - // Get the Volume prefix from environment - volumePrefix := s.getClusterPrefix() - maxLength := MaxVolIdentifierLength - len(volumePrefix) - len(s.getClusterPrefix()) - len(CsiVolumePrefix) - 1 - // First get the short volume name - shortVolumeName := truncateString(volumeName, maxLength) - // Form the volume identifier using short volume name and namespace - var namespaceSuffix string - if namespace != "" { - namespaceSuffix = "-" + namespace - } - volumeIdentifier := fmt.Sprintf("%s%s-%s%s", CsiVolumePrefix, s.getClusterPrefix(), shortVolumeName, namespaceSuffix) - - if useNFS { - // calculate size in MiB - reqSizeInMiB := (cr.GetRequiredBytes() + MiBSizeInBytes - 1) / MiBSizeInBytes - return file.CreateFileSystem(ctx, reqID, accessibility, params, symmetrixID, storagePoolID, serviceLevel, nasServer, volumeIdentifier, allowRoot, reqSizeInMiB, pmaxClient) - } - // Storage Group is required to be derived from the parameters (such as service level and storage resource pool which are supplied in parameters) - // Storage Group Name can optionally be supplied in the parameters (for testing) to over-ride the default. - if storageGroupName == "" { - if applicationPrefix == "" { - storageGroupName = fmt.Sprintf("%s-%s-%s-%s-SG", CSIPrefix, s.getClusterPrefix(), - serviceLevel, storagePoolID) - } else { - storageGroupName = fmt.Sprintf("%s-%s-%s-%s-%s-SG", CSIPrefix, s.getClusterPrefix(), - applicationPrefix, serviceLevel, storagePoolID) - } - if hostLimitName != "" { - storageGroupName = fmt.Sprintf("%s-%s", storageGroupName, hostLimitName) - } - } - - var dynamicSGName string - var needCreation bool - if s.opts.dynamicSGEnabled { - if dynamicSGName, needCreation, err = getDynamicSG(ctx, symmetrixID, storageGroupName, s); err != nil { - log.Error("failed to get dynamic SG: " + err.Error()) - return nil, status.Error(codes.Internal, err.Error()) - } - log.Infof("####### dynamic storage group name: %s, base SG name: %s, needCreation: %v", dynamicSGName, storageGroupName, needCreation) - storageGroupName = dynamicSGName - } - - // localProtectionGroupID refers to name of Storage Group which has protected local volumes - // remoteProtectionGroupID refers to name of Storage Group which has protected remote volumes - var localProtectionGroupID string - var remoteProtectionGroupID string - if replicationEnabled == "true" { - localProtectionGroupID = buildProtectionGroupID(namespace, localRDFGrpNo, repMode) - remoteProtectionGroupID = buildProtectionGroupID(namespace, remoteRDFGrpNo, repMode) - } - - // log all parameters used in CreateVolume call - fields := map[string]interface{}{ - "SymmetrixID": symmetrixID, - "SRP": storagePoolID, - "Accessibility": accessibility, - "ApplicationPrefix": applicationPrefix, - "volumeIdentifier": volumeIdentifier, - "requiredCylinders": requiredCylinders, - "storageGroupName": storageGroupName, - "CSIRequestID": reqID, - "SourceVolume": srcVolID, - "SourceSnapshot": srcSnapID, - "ReplicationEnabled": replicationEnabled, - "RemoteSymID": remoteSymID, - "LocalRDFGroup": localRDFGrpNo, - "RemoteRDFGroup": remoteRDFGrpNo, - "SRDFMode": repMode, - "PVCNamespace": namespace, - "LocalProtectionGroupID": localProtectionGroupID, - "RemoteProtectionGroupID": remoteProtectionGroupID, - HeaderPersistentVolumeName: params[CSIPersistentVolumeName], - HeaderPersistentVolumeClaimName: params[CSIPersistentVolumeClaimName], - HeaderPersistentVolumeClaimNamespace: params[CSIPVCNamespace], - HostIOLimitMBSec: hostMBsec, - HostIOLimitIOSec: hostIOsec, - DynamicDistribution: hostDynDistribution, - } - log.WithFields(fields).Info("Executing CreateVolume with following fields") - - // isSGUnprotected is set to true only if SG has a replica, eg if the SG is new - isSGUnprotected := false - if replicationEnabled == "true" { - sg, err := s.getOrCreateProtectedStorageGroup(ctx, symmetrixID, localProtectionGroupID, namespace, localRDFGrpNo, repMode, reqID, pmaxClient) - if err != nil { - return nil, status.Errorf(codes.Internal, "Error in getOrCreateProtectedStorageGroup: (%s)", err.Error()) - } - if sg != nil && sg.Rdf == true { - // Check the direction of SG - // Creation of replicated volume is allowed in an SG of type R1 - err := s.VerifyProtectedGroupDirection(ctx, symmetrixID, localProtectionGroupID, localRDFGrpNo, pmaxClient) - if err != nil { - return nil, err - } - } else { - isSGUnprotected = true - } - } - - // Check existence of the Storage Group and create if necessary. - if !s.opts.dynamicSGEnabled { - sg, err := pmaxClient.GetStorageGroup(ctx, symmetrixID, storageGroupName) - if err != nil || sg == nil { - log.Debug(fmt.Sprintf("Unable to find storage group: %s", storageGroupName)) - needCreation = true - } - } - - if needCreation { - hostLimitsParam := &types.SetHostIOLimitsParam{ - HostIOLimitMBSec: hostMBsec, - HostIOLimitIOSec: hostIOsec, - DynamicDistribution: hostDynDistribution, - } - optionalPayload := make(map[string]interface{}) - optionalPayload[HostLimits] = hostLimitsParam - if *hostLimitsParam == (types.SetHostIOLimitsParam{}) { - optionalPayload = nil - } - _, err := pmaxClient.CreateStorageGroup(ctx, symmetrixID, storageGroupName, storagePoolID, - serviceLevel, thick == "true", optionalPayload) + selectedPath, creator := creatorFor(s, pmaxClient, s.adminClient104, symmetrixID, reqID, params, symmIDFoundInAZ, version, apiVersion, req.GetVolumeCapabilities(), req.GetVolumeContentSource()) + if selectedPath == backendU4P104 { + log.Infof("Attempting 10.4 CreateVolume path for array %s", symmetrixID) + resp, err := creator.Create(ctx, req) if err != nil { - log.Error("Error creating storage group: " + err.Error()) - return nil, status.Errorf(codes.Internal, "Error creating storage group: %s", err.Error()) - } - } - alreadyExists := false - isLocalVolumePresent := false - - var vol *types.Volume - var volumeList *types.Volumev1 - - if version >= 103 { - log.Debug("API version is greater than or equal to 103. Using enhanced API") - // Idempotency test. We will read the volume and check for: - // 1. Existence of a volume with matching volume name - // 2. Matching cylinderSize - // 3. Is a member of the storage group - // 4. Check if snapshot/volume target - log.Debug("Calling GetVolumeIDList for idempotency test") - volumeList, err = pmaxClient.GetVolumesByIdentifier(ctx, symmetrixID, volumeIdentifier) - if err != nil { - log.Error("Error getting the volumes for idempotence check: " + err.Error()) - return nil, status.Errorf(codes.Internal, "Error getting the volumes for idempotence check: %s", err.Error()) - } - - // isLocalVolumePresent restrict CreateVolumeInProtectedSG call if the volume is present in local SG but not in remote SG - // isLocalVolumePresent := false - // Look up the volume(s), if any, returned for the idempotency check to see if there are any matches - // We ignore any volume not in the desired storage group (even though they have the same name). - for _, eachVol := range volumeList.Volumes { - if len(eachVol.StorageGroups) < 1 { - log.Error("Idempotence check: StorageGroupIDList is empty for (%s): " + eachVol.ID) - return nil, status.Errorf(codes.Internal, "Idempotence check: StorageGroupIDList is empty for (%s)", eachVol.ID) - } - matchesStorageGroup := false - for _, sgid := range eachVol.StorageGroups { - if strings.Contains(sgid.StorageGroupID, storageGroupName) { - matchesStorageGroup = true - storageGroupName = sgid.StorageGroupID - } - } - - // with Authorization, a tenant prefix is applied to the volume identifier on the array - // csi-CSM-pmax-69298b3d3d-namespace -> tn1-csi-CSM-pmax-69298b3d3d-namespace - // since we don't know the tenant prefix, the volume identifier on the array is checked to contain the standard volume identifier - if matchesStorageGroup && (eachVol.Identifier == volumeIdentifier || strings.Contains(eachVol.Identifier, volumeIdentifier)) { - // A volume with the same name exists and has the same size - if eachVol.CapCyl != float64(requiredCylinders) { - log.Error("A volume with the same name exists but has a different size than required.") - alreadyExists = true - continue - } - var remoteVolumeID string - if replicationEnabled == "true" { - remoteVolumeID, _, err = s.GetRemoteVolumeID(ctx, symmetrixID, localRDFGrpNo, eachVol.ID, pmaxClient) - if err != nil && !strings.Contains(err.Error(), "The device must be an RDF device") { - return nil, status.Errorf(codes.Internal, "Failed to fetch rdf pair information for (%s) - Error (%s)", eachVol.ID, err.Error()) - } - if remoteVolumeID == "" { - // Missing corresponding Remote Volume Name for existing local volume - // The SG is unprotected as Local volume and Local SG exists but missing corresponding SRDF info - // If the SG was protected, there must exist a corresponding remote replica volume - log.Debugf("Local Volume already exist, skipping creation (%s)", eachVol.ID) - isLocalVolumePresent = true - vol = &types.Volume{} - vol.VolumeID = eachVol.ID - vol.CapacityGB = eachVol.CapCyl - vol.VolumeIdentifier = eachVol.Identifier - var sgIDs []string - for _, sg := range eachVol.StorageGroups { - if sg.StorageGroupID != "" { - sgIDs = append(sgIDs, sg.StorageGroupID) - } - } - vol.StorageGroupIDList = sgIDs - continue - } - } - if volContent != "" { - if replicationEnabled == "true" { - if srcSnapID != "" { - err = s.LinkSRDFVolToSnapshot(ctx, reqID, symID, srcVol.VolumeID, snapID, localProtectionGroupID, localRDFGrpNo, vol, bias, false, pmaxClient) - if err != nil { - return nil, err - } - } else if srcVolID != "" { - // Check if the array is V4 or above - isV4 := s.isV4OrAbove(ctx, symID, pmaxClient) - if isV4 { - // V4 and above, use clone with "establish_terminate" option - err = s.LinkSRDFCloneVolume(ctx, reqID, symID, srcVol, vol, localProtectionGroupID, localRDFGrpNo, "false", pmaxClient) - if err != nil { - return nil, status.Errorf(codes.Internal, "Failed to create SRDF volume from volume (%s)", err.Error()) - } - } else { - // V3 or below, use legacy SnapVx clone - tmpSnapID := fmt.Sprintf("%s%s-%d", TempSnap, s.getClusterPrefix(), time.Now().Nanosecond()) - err = s.LinkSRDFVolToVolume(ctx, reqID, symID, srcVol, vol, tmpSnapID, localProtectionGroupID, localRDFGrpNo, "false", false, pmaxClient) - if err != nil { - return nil, status.Errorf(codes.Internal, "Failed to create SRDF volume from volume (%s)", err.Error()) - } - } - } - } else { // replication is not enabled - if srcSnapID != "" { - err = s.UnlinkTargets(ctx, symID, SrcDevID, pmaxClient) - if err != nil { - return nil, status.Errorf(codes.Internal, "Failed unlink existing target from snapshot (%s)", err.Error()) - } - err = s.LinkVolumeToSnapshot(ctx, symID, srcVol.VolumeID, eachVol.ID, snapID, reqID, false, pmaxClient) - if err != nil { - return nil, status.Errorf(codes.Internal, "Failed to create volume from snapshot (%s)", err.Error()) - } - } else if srcVolID != "" && eachVol.ID != "" { - // Check if the array is V4 or above - isV4 := s.isV4OrAbove(ctx, symID, pmaxClient) - if isV4 { - // V4 and above, use clone with "establish_terminate" option - replicaRequest := types.ReplicationRequest{ - ReplicationPair: []types.ReplicationPair{ - { - SourceVolumeName: srcVol.VolumeID, - TargetVolumeName: eachVol.ID, - }, - }, - Establish: true, - EstablishTerminate: true, - } - err = pmaxClient.CloneVolumeFromVolume(ctx, symID, replicaRequest) - if err != nil { - return nil, status.Errorf(codes.Internal, "Failed to create volume from volume (%s)", err.Error()) - } - } else { - // V3 or below, use legacy SnapVx clone - tmpSnapID := fmt.Sprintf("%s%s-%d", TempSnap, s.getClusterPrefix(), time.Now().Nanosecond()) - err = s.LinkVolumeToVolume(ctx, symID, srcVol, eachVol.ID, tmpSnapID, reqID, false, pmaxClient) - if err != nil { - return nil, status.Errorf(codes.Internal, "Failed to create volume from volume (%s)", err.Error()) - } - } - } - } - } - - log.WithFields(fields).Info("Idempotent volume detected, returning success") - eachVol.ID = fmt.Sprintf("%s-%s-%s", eachVol.Identifier, symmetrixID, eachVol.ID) - volResp := s.buildCSIVolume(&eachVol) - // Set the volume context - attributes := map[string]string{ - ServiceLevelParam: serviceLevel, - StoragePoolParam: storagePoolID, - path.Join(s.opts.ReplicationContextPrefix, SymmetrixIDParam): symmetrixID, - CapacityGB: fmt.Sprintf("%.2f", eachVol.CapCyl), - ContentSource: volContent, - StorageGroup: storageGroupName, - // Format the time output - "CreationTime": time.Now().Format("20060102150405"), - } - if replicationEnabled == "true" { - addReplicationParamsToVolumeAttributes(attributes, s.opts.ReplicationContextPrefix, remoteSymID, repMode, remoteVolumeID, localRDFGrpNo, remoteRDFGrpNo) - } - volResp.VolumeContext = attributes - csiResp := &csi.CreateVolumeResponse{ - Volume: volResp, - } - volResp.ContentSource = contentSource - if accessibility != nil { - volResp.AccessibleTopology = accessibility.Preferred - } - return csiResp, nil - } - } - } else { - // Idempotency test. We will read the volume and check for: - // 1. Existence of a volume with matching volume name - // 2. Matching cylinderSize - // 3. Is a member of the storage group - // 4. Check if snapshot/volume target - log.Debug("Calling GetVolumeIDList for idempotency test") - // For now an exact match - volumeIDList, err := pmaxClient.GetVolumeIDList(ctx, symmetrixID, volumeIdentifier, false) - if err != nil { - log.Error("Error looking up volume for idempotence check: " + err.Error()) - return nil, status.Errorf(codes.Internal, "Error looking up volume for idempotence check: %s", err.Error()) - } - // isLocalVolumePresent restrict CreateVolumeInProtectedSG call if the volume is present in local SG but not in remote SG - // isLocalVolumePresent := false - // Look up the volume(s), if any, returned for the idempotency check to see if there are any matches - // We ignore any volume not in the desired storage group (even though they have the same name). - for _, volumeID := range volumeIDList { - // Fetch the volume - log.WithFields(fields).Info("Calling GetVolumeByID for idempotence check") - vol, err = pmaxClient.GetVolumeByID(ctx, symmetrixID, volumeID) - if err != nil { - log.Error("Error fetching volume for idempotence check: " + err.Error()) - return nil, status.Errorf(codes.Internal, "Error fetching volume for idempotence check: %s", err.Error()) - } - if len(vol.StorageGroupIDList) < 1 { - log.Error("Idempotence check: StorageGroupIDList is empty for (%s): " + volumeID) - return nil, status.Errorf(codes.Internal, "Idempotence check: StorageGroupIDList is empty for (%s)", volumeID) - } - matchesStorageGroup := false - for _, sgid := range vol.StorageGroupIDList { - if strings.Contains(sgid, storageGroupName) { - matchesStorageGroup = true - storageGroupName = sgid - } - } - - // with Authorization, a tenant prefix is applied to the volume identifier on the array - // csi-CSM-pmax-69298b3d3d-namespace -> tn1-csi-CSM-pmax-69298b3d3d-namespace - // since we don't know the tenant prefix, the volume identifier on the array is checked to contain the standard volume identifier - if matchesStorageGroup && (vol.VolumeIdentifier == volumeIdentifier || strings.Contains(vol.VolumeIdentifier, volumeIdentifier)) { - // A volume with the same name exists and has the same size - if vol.CapacityCYL != requiredCylinders { - log.Error("A volume with the same name exists but has a different size than required.") - alreadyExists = true - continue - } - var remoteVolumeID string - if replicationEnabled == "true" { - remoteVolumeID, _, err = s.GetRemoteVolumeID(ctx, symmetrixID, localRDFGrpNo, vol.VolumeID, pmaxClient) - if err != nil && !strings.Contains(err.Error(), "The device must be an RDF device") { - return nil, status.Errorf(codes.Internal, "Failed to fetch rdf pair information for (%s) - Error (%s)", vol.VolumeID, err.Error()) - } - if remoteVolumeID == "" { - // Missing corresponding Remote Volume Name for existing local volume - // The SG is unprotected as Local volume and Local SG exists but missing corresponding SRDF info - // If the SG was protected, there must exist a corresponding remote replica volume - log.Debugf("Local Volume already exist, skipping creation (%s)", vol.VolumeID) - isLocalVolumePresent = true - continue - } - } - if volContent != "" { - if replicationEnabled == "true" { - if srcSnapID != "" { - err = s.LinkSRDFVolToSnapshot(ctx, reqID, symID, srcVol.VolumeID, snapID, localProtectionGroupID, localRDFGrpNo, vol, bias, false, pmaxClient) - if err != nil { - return nil, err - } - } else if srcVolID != "" { - // Check if the array is V4 or above - isV4 := s.isV4OrAbove(ctx, symID, pmaxClient) - if isV4 { - // V4 and above, use clone with "establish_terminate" option - err = s.LinkSRDFCloneVolume(ctx, reqID, symID, srcVol, vol, localProtectionGroupID, localRDFGrpNo, "false", pmaxClient) - if err != nil { - return nil, status.Errorf(codes.Internal, "Failed to create SRDF volume from volume (%s)", err.Error()) - } - } else { - // V3 or below, use legacy SnapVx clone - tmpSnapID := fmt.Sprintf("%s%s-%d", TempSnap, s.getClusterPrefix(), time.Now().Nanosecond()) - err = s.LinkSRDFVolToVolume(ctx, reqID, symID, srcVol, vol, tmpSnapID, localProtectionGroupID, localRDFGrpNo, "false", false, pmaxClient) - if err != nil { - return nil, status.Errorf(codes.Internal, "Failed to create SRDF volume from volume (%s)", err.Error()) - } - } - } - } else { // replication is not enabled - if srcSnapID != "" { - err = s.UnlinkTargets(ctx, symID, SrcDevID, pmaxClient) - if err != nil { - return nil, status.Errorf(codes.Internal, "Failed unlink existing target from snapshot (%s)", err.Error()) - } - err = s.LinkVolumeToSnapshot(ctx, symID, srcVol.VolumeID, vol.VolumeID, snapID, reqID, false, pmaxClient) - if err != nil { - return nil, status.Errorf(codes.Internal, "Failed to create volume from snapshot (%s)", err.Error()) - } - } else if srcVolID != "" { - // Check if the array is V4 or above - isV4 := s.isV4OrAbove(ctx, symID, pmaxClient) - if isV4 { - // V4 and above, use clone with "establish_terminate" option - replicaRequest := types.ReplicationRequest{ - ReplicationPair: []types.ReplicationPair{ - { - SourceVolumeName: srcVol.VolumeID, - TargetVolumeName: vol.VolumeID, - }, - }, - Establish: true, - EstablishTerminate: true, - } - err = pmaxClient.CloneVolumeFromVolume(ctx, symID, replicaRequest) - if err != nil { - return nil, status.Errorf(codes.Internal, "Failed to create volume from volume (%s)", err.Error()) - } - } else { - // V3 or below, use legacy SnapVx clone - tmpSnapID := fmt.Sprintf("%s%s-%d", TempSnap, s.getClusterPrefix(), time.Now().Nanosecond()) - err = s.LinkVolumeToVolume(ctx, symID, srcVol, vol.VolumeID, tmpSnapID, reqID, false, pmaxClient) - if err != nil { - return nil, status.Errorf(codes.Internal, "Failed to create volume from volume (%s)", err.Error()) - } - } - } - } - } - - log.WithFields(fields).Info("Idempotent volume detected, returning success") - vol.VolumeID = fmt.Sprintf("%s-%s-%s", vol.VolumeIdentifier, symmetrixID, vol.VolumeID) - volResp := s.getCSIVolume(vol) - // Set the volume context - attributes := map[string]string{ - ServiceLevelParam: serviceLevel, - StoragePoolParam: storagePoolID, - path.Join(s.opts.ReplicationContextPrefix, SymmetrixIDParam): symmetrixID, - CapacityGB: fmt.Sprintf("%.2f", vol.CapacityGB), - ContentSource: volContent, - StorageGroup: storageGroupName, - // Format the time output - "CreationTime": time.Now().Format("20060102150405"), - } - - if symmIDFoundInAZ { - s.addZoneLabelsToVolumeAttributes(attributes, symmetrixID) - } - - if replicationEnabled == "true" { - addReplicationParamsToVolumeAttributes(attributes, s.opts.ReplicationContextPrefix, remoteSymID, repMode, remoteVolumeID, localRDFGrpNo, remoteRDFGrpNo) - } - volResp.VolumeContext = attributes - csiResp := &csi.CreateVolumeResponse{ - Volume: volResp, - } - volResp.ContentSource = contentSource - if accessibility != nil { - volResp.AccessibleTopology = accessibility.Preferred - } - return csiResp, nil - } - } - } - if alreadyExists { - log.Error("A volume with the same name " + volumeName + "exists but has a different size than requested. Use a different name.") - return nil, status.Errorf(codes.AlreadyExists, "A volume with the same name %s exists but has a different size than requested. Use a different name.", volumeName) - } - - // CSI specific metada for authorization - headerMetadata := addMetaData(params) - - // Let's create the volume - if !isLocalVolumePresent { - vol, err = pmaxClient.CreateVolumeInStorageGroupS(ctx, symmetrixID, storageGroupName, volumeIdentifier, requiredCylinders, nil, headerMetadata) - if err != nil { - log.Error(fmt.Sprintf("Could not create volume: %s: %s", volumeName, err.Error())) - return nil, status.Errorf(codes.Internal, "Could not create volume: %s: %s", volumeName, err.Error()) - } - } - - if replicationEnabled == "true" { - log.Debugf("RDF: Found Rdf enabled") - // remote storage group name is kept same as local storage group name - // Check if volume is already added in SG, else add it - protectedSGID := s.GetProtectedStorageGroupID(vol.StorageGroupIDList, localRDFGrpNo+"-"+repMode) - if protectedSGID == "" { - // Volume is not present in Protected Storage Group, Add - err = s.addVolumesToProtectedStorageGroup(ctx, reqID, symmetrixID, localProtectionGroupID, remoteSymID, remoteProtectionGroupID, false, vol.VolumeID, pmaxClient) - if err != nil { - return nil, err - } - } - if isSGUnprotected { - // If the required SG is still unprotected, protect the local SG with RDF info - // If valid RDF group is supplied this will create a remote SG, a RDF pair and add the vol in respective SG created - // Remote storage group name is kept same as local storage group name - err := s.ProtectStorageGroup(ctx, symmetrixID, remoteSymID, localProtectionGroupID, remoteProtectionGroupID, "", localRDFGrpNo, repMode, vol.VolumeID, reqID, false, pmaxClient) - if err != nil { - log.Errorf("Proceeding to remove volume from protected storage group as rollback") - // Remove volume from protected storage group as a rollback - // The device could be just a TDEV and can make RDF unmanageable due to slow u4p response - _, er := pmaxClient.RemoveVolumesFromStorageGroup(ctx, symmetrixID, localProtectionGroupID, true, vol.VolumeID) - if er != nil { - log.Errorf("Error removing volume %s from protected SG %s with error: %s", vol.VolumeID, localProtectionGroupID, er.Error()) - } - return nil, err - } - } - } - - // If volume content source is specified, initiate no_copy to newly created volume - if contentSource != nil { - if srcVolID != "" { - // Check if the array is V4 or above - isV4 := s.isV4OrAbove(ctx, symID, pmaxClient) - if replicationEnabled == "true" { - if isV4 { - // V4 and above, use clone with "establish_terminate" option - err = s.LinkSRDFCloneVolume(ctx, reqID, symID, srcVol, vol, localProtectionGroupID, localRDFGrpNo, "false", pmaxClient) - if err != nil { - return nil, status.Errorf(codes.Internal, "Failed to create SRDF volume from volume (%s)", err.Error()) - } - } else { - // V3 or below, use legacy SnapVx clone - tmpSnapID := fmt.Sprintf("%s%s-%d", TempSnap, s.getClusterPrefix(), time.Now().Nanosecond()) - err = s.LinkSRDFVolToVolume(ctx, reqID, symID, srcVol, vol, tmpSnapID, localProtectionGroupID, localRDFGrpNo, "false", false, pmaxClient) - if err != nil { - return nil, status.Errorf(codes.Internal, "Failed to create SRDF volume from volume (%s)", err.Error()) - } - } - } else { - if isV4 { - // V4 and above, use clone with "establish_terminate" option - replicaRequest := types.ReplicationRequest{ - ReplicationPair: []types.ReplicationPair{ - { - SourceVolumeName: srcVol.VolumeID, - TargetVolumeName: vol.VolumeID, - }, - }, - Establish: true, - EstablishTerminate: true, - } - err = pmaxClient.CloneVolumeFromVolume(ctx, symID, replicaRequest) - if err != nil { - return nil, status.Errorf(codes.Internal, "Failed to create volume from volume (%s)", err.Error()) - } - } else { - // V3 or below, use legacy SnapVx clone - tmpSnapID := fmt.Sprintf("%s%s-%d", TempSnap, s.getClusterPrefix(), time.Now().Nanosecond()) - err = s.LinkVolumeToVolume(ctx, symID, srcVol, vol.VolumeID, tmpSnapID, reqID, false, pmaxClient) - if err != nil { - return nil, status.Errorf(codes.Internal, "Failed to create volume from volume (%s)", err.Error()) - } - } - } - } else if srcSnapID != "" { - if replicationEnabled == "true" { - err = s.LinkSRDFVolToSnapshot(ctx, reqID, symID, srcVol.VolumeID, snapID, localProtectionGroupID, localRDFGrpNo, vol, bias, false, pmaxClient) - if err != nil { - return nil, err - } - } else { - // Unlink all previous targets from this snapshot if the link is in defined state - err = s.UnlinkTargets(ctx, symID, SrcDevID, pmaxClient) - if err != nil { - return nil, status.Errorf(codes.Internal, "Failed unlink existing target from snapshot (%s)", err.Error()) - } - err = s.LinkVolumeToSnapshot(ctx, symID, srcVol.VolumeID, vol.VolumeID, snapID, reqID, false, pmaxClient) - if err != nil { - return nil, status.Errorf(codes.Internal, "Failed to create volume from snapshot (%s)", err.Error()) - } - } - } - } - - // Formulate the return response - volID := vol.VolumeID - vol.VolumeID = fmt.Sprintf("%s-%s-%s", vol.VolumeIdentifier, symmetrixID, vol.VolumeID) - volResp := s.getCSIVolume(vol) - volResp.ContentSource = contentSource - // Set the volume context - attributes := map[string]string{ - ServiceLevelParam: serviceLevel, - StoragePoolParam: storagePoolID, - path.Join(s.opts.ReplicationContextPrefix, SymmetrixIDParam): symmetrixID, - CapacityGB: fmt.Sprintf("%.2f", vol.CapacityGB), - ContentSource: volContent, - StorageGroup: storageGroupName, - // Format the time output - "CreationTime": time.Now().Format("20060102150405"), - } - if replicationEnabled == "true" { - remoteVolumeID, _, err := s.GetRemoteVolumeID(ctx, symmetrixID, localRDFGrpNo, volID, pmaxClient) - if err != nil { - return nil, status.Errorf(codes.Internal, "Failed to fetch rdf pair information for (%s) - Error (%s)", vol.VolumeID, err.Error()) + log.Errorf("10.4 CreateVolume failed for array %s: %v", symmetrixID, err) + return nil, err } - addReplicationParamsToVolumeAttributes(attributes, s.opts.ReplicationContextPrefix, remoteSymID, repMode, remoteVolumeID, localRDFGrpNo, remoteRDFGrpNo) - } - - volResp.VolumeContext = attributes - if accessibility != nil { - volResp.AccessibleTopology = accessibility.Preferred - } - csiResp := &csi.CreateVolumeResponse{ - Volume: volResp, + return resp, nil } - fields[storageGroupName] = storageGroupName - log.WithFields(fields).Infof("Created volume with ID: %s", volResp.VolumeId) - return csiResp, nil + return creator.Create(ctx, req) } func (s *service) createMetroVolume(ctx context.Context, req *csi.CreateVolumeRequest, reqID, storagePoolID, symID, storageGroupName, serviceLevel, thick, remoteSymID, localRDFGrpNo, remoteRDFGrpNo, remoteServiceLevel, remoteSRPID, namespace, applicationPrefix, bias, hostLimitName, hostMBsec, hostIOsec, hostDynDist string) (*csi.CreateVolumeResponse, error) { @@ -1889,7 +1105,8 @@ func (s *service) validateVolSize(ctx context.Context, cr *csi.CapacityRange, sy if minSizeBytes < 0 || maxSizeBytes < 0 { return 0, status.Errorf( codes.OutOfRange, - "bad capacity: requested volume size bytes %d and limit size bytes: %d must not be negative", minSizeBytes, maxSizeBytes) + "bad capacity: requested volume size %d bytes and limit size %d bytes must not be negative", minSizeBytes, maxSizeBytes, + ) } if minSizeBytes == 0 { @@ -1941,22 +1158,25 @@ func (s *service) validateVolSize(ctx context.Context, cr *csi.CapacityRange, sy if maxSizeBytes > maxAvailBytes { return 0, status.Errorf( codes.OutOfRange, - "bad capacity: requested maximum size (%d bytes) is greater than the maximum available capacity (%d bytes)", maxSizeBytes, maxAvailBytes) + "bad capacity: requested maximum size %d bytes is greater than the maximum available capacity %d bytes", maxSizeBytes, maxAvailBytes, + ) } if minSizeBytes > maxAvailBytes { return 0, status.Errorf( codes.OutOfRange, - "bad capacity: requested minimum size (%d bytes) is greater than the maximum available capacity (%d bytes)", minSizeBytes, maxAvailBytes) + "bad capacity: requested minimum size %d bytes is greater than the maximum available capacity %d bytes", minSizeBytes, maxAvailBytes, + ) } if minSizeBytes < MinVolumeSizeBytes { - log.Warnf("bad capacity: requested size (%d bytes) is less than the minimum volume size (%d bytes) supported by PowerMax..", minSizeBytes, MinVolumeSizeBytes) + log.Warnf("bad capacity: requested size %d bytes is less than the minimum volume size %d bytes supported by PowerMax..", minSizeBytes, MinVolumeSizeBytes) log.Warnf("Proceeding with minimum volume size supported by PowerMax Array ......") minSizeBytes = MinVolumeSizeBytes } if maxSizeBytes < minSizeBytes { return 0, status.Errorf( codes.OutOfRange, - "bad capacity: requested maximum size (%d bytes) is less than the requested minimum size (%d bytes)", maxSizeBytes, minSizeBytes) + "bad capacity: requested maximum size %d bytes is less than the requested minimum size %d bytes", maxSizeBytes, minSizeBytes, + ) } minNumberOfCylinders := int(minSizeBytes / cylinderSizeInBytes) var numOfCylinders int @@ -1969,7 +1189,8 @@ func (s *service) validateVolSize(ctx context.Context, cr *csi.CapacityRange, sy if sizeInBytes > maxSizeBytes { return 0, status.Errorf( codes.OutOfRange, - "bad capacity: size in bytes %d exceeds limit size bytes %d", sizeInBytes, maxSizeBytes) + "bad capacity: size %d bytes exceeds limit size %d bytes", sizeInBytes, maxSizeBytes, + ) } return numOfCylinders, nil } @@ -2202,6 +1423,23 @@ func (s *service) deleteVolume(ctx context.Context, reqID, symID, volName, devID } if vol.VolumeIdentifier != volName { + // Check if the volume was previously renamed with the deletion prefix + // but failed to be queued for actual deletion (e.g., due to a transient error). + // In that case, re-attempt queuing instead of assuming it's already deleted. + expectedDelName := fmt.Sprintf("%s%s", DeletionPrefix, volName) + if len(expectedDelName) > MaxVolIdentifierLength { + expectedDelName = expectedDelName[:MaxVolIdentifierLength] + } + if vol.VolumeIdentifier == expectedDelName { + log.Info(fmt.Sprintf("DeleteVolume: VolumeIdentifier %s indicates volume %s was renamed for deletion but not yet queued. Re-attempting deletion.", + vol.VolumeIdentifier, volName)) + err = s.deletionWorker.QueueDeviceForDeletion(vol.VolumeID, vol.VolumeIdentifier, symID) + if err != nil { + log.Errorf("RequestSoftVolDelete (retry) failed with error - %s", err.Error()) + return status.Errorf(codes.Internal, "Failed marking volume for deletion with error (%s)", err.Error()) + } + return nil + } // This volume is already deleted or marked for deletion, // or volume id is an old stale identifier not matching a volume. // Either way idempotence calls for doing nothing and returning ok. @@ -2267,7 +1505,7 @@ func (s *service) deleteFileSystem(ctx context.Context, reqID, symID, fsName, fs log.WithFields(fields).Info("Executing Delete File System with following fields") fileSystem, err := pmaxClient.GetFileSystemByID(ctx, symID, fsID) - log.Debugf("fileSysetm: %#v, error: %#v", fileSystem, err) + log.Debugf("fileSystem: %#v, error: %#v", fileSystem, err) if err != nil { if strings.Contains(err.Error(), cannotBeFound) { @@ -2318,13 +1556,6 @@ func (s *service) ControllerPublishVolume( reqID = req[0] } } - volumeContext := req.GetVolumeContext() - if volumeContext != nil { - log.Infof("VolumeContext:") - for key, value := range volumeContext { - log.Infof(" [%s]=%s", key, value) - } - } volID := req.GetVolumeId() if volID == "" { @@ -2341,7 +1572,6 @@ func (s *service) ControllerPublishVolume( pmaxClient, err := s.GetPowerMaxClient(symID, remoteSymID) if err != nil { log.Error(err.Error()) - return nil, status.Error(codes.InvalidArgument, err.Error()) } @@ -2356,151 +1586,15 @@ func (s *service) ControllerPublishVolume( return nil, err } - nodeID := req.GetNodeId() - if nodeID == "" { - log.Error("node ID is required") - return nil, status.Error(codes.InvalidArgument, - "node ID is required") - } - - vc := req.GetVolumeCapability() - if vc == nil { - log.Error("volume capability is required") - return nil, status.Error(codes.InvalidArgument, - "volume capability is required") - } - am := vc.GetAccessMode() - if am == nil { - log.Error("access mode is required") - return nil, status.Error(codes.InvalidArgument, - "access mode is required") - } - - if am.Mode == csi.VolumeCapability_AccessMode_UNKNOWN { - log.Error(errUnknownAccessMode) - return nil, status.Error(codes.InvalidArgument, errUnknownAccessMode) - } - isNFS := accTypeIsNFS([]*csi.VolumeCapability{vc}) - if isNFS { - // incoming request for file system volume - return file.CreateNFSExport(ctx, reqID, symID, devID, am, volumeContext, pmaxClient) - } - - if vc.GetMount().GetFsType() == "" { - // can happen when doing static provisioning, check for filesystem existence - log.Debug("fsType empty...checking for file system existence") - _, err := pmaxClient.GetFileSystemByID(ctx, symID, devID) - if err == nil { - // we found fs, proceed to CreateNFSExport - return nil, status.Errorf(codes.Unavailable, "static provisioning on a file system is not supported.") - } - } - - // Fetch the volume details from array - symID, devID, vol, err := s.GetVolumeByID(ctx, volID, pmaxClient) + versionDetails, err := s.getVersionCache().getOrFetchVersionDetails(ctx, symID, pmaxClient) if err != nil { - log.Error("GetVolumeByID Error: " + err.Error()) - return nil, err - } - - // log all parameters used in ControllerPublishVolume call - fields := map[string]interface{}{ - "SymmetrixID": symID, - "VolumeId": volID, - "NodeId": nodeID, - "AccessMode": am.Mode, - "CSIRequestID": reqID, - "IsVsphereVolume": s.opts.IsVsphereEnabled, - } - log.WithFields(fields).Info("Executing ControllerPublishVolume with following fields") - // flag createNFSExport() - isNVMETCP := false - isISCSI := false - // Check if node ID is present in cache - nodeInCache := false - cacheID := symID + ":" + nodeID - tempHostID, ok := nodeCache.Load(cacheID) - if ok { - log.Debugf("REQ ID: %s Loaded nodeID: %s, hostID: %s from node cache", - reqID, nodeID, tempHostID.(string)) - nodeInCache = true - if !strings.Contains(tempHostID.(string), "-FC") { - isISCSI = true - } - if strings.Contains(tempHostID.(string), "-NVMETCP") { - isISCSI = false - isNVMETCP = true - } - } else { - log.Debugf("REQ ID: %s nodeID: %s not present in node cache", reqID, nodeID) - isNVMETCP, err = s.IsNodeNVMe(ctx, symID, nodeID, pmaxClient) - if err != nil { - return nil, status.Error(codes.NotFound, err.Error()) - } - if !isNVMETCP { - isISCSI, err = s.IsNodeISCSI(ctx, symID, nodeID, pmaxClient) - if err != nil { - return nil, status.Error(codes.NotFound, err.Error()) - } - } - } - - hostID, tgtStorageGroupID, tgtMaskingViewID := s.GetNVMETCPHostSGAndMVIDFromNodeID(nodeID) - - if !isNVMETCP { - // Update the values, if NVME is false - hostID, tgtStorageGroupID, tgtMaskingViewID = s.GetHostSGAndMVIDFromNodeID(nodeID, isISCSI) - } - - if !nodeInCache { - // Update the map - val, ok := nodeCache.LoadOrStore(cacheID, hostID) - if !ok { - log.Debugf("REQ ID: %s Added nodeID: %s, hostID: %s to node cache", reqID, nodeID, hostID) - } else { - log.Debugf("REQ ID: %s Some other goroutine added hostID: %s for node: %s to node cache", - reqID, val.(string), nodeID) - if hostID != val.(string) { - log.Warnf("REQ ID: %s Mismatch between calculated value: %s and latest value: %s from node cache", - reqID, val.(string), hostID) - } - } - } - - publishContext := make(map[string]string) - if len(vol.EffectiveWWN) > 0 { - publishContext[PublishContextDeviceWWN] = vol.EffectiveWWN - } else { - return nil, status.Errorf(codes.Internal, "PublishVolume: Volume %s has no effective WWN, Unisphere may not be synchronized with array or synchronization may be in progress", volID) - } - - ctrlPubRes, ctrlPubErr := s.publishVolume(ctx, publishContext, tgtStorageGroupID, hostID, symID, symID, tgtMaskingViewID, devID, reqID, volumeName, am, pmaxClient, true) - if ctrlPubErr != nil { - return nil, ctrlPubErr + log.Error("Error in getversion API " + err.Error()) + return nil, status.Errorf(codes.Internal, "Error in getVersion API: %s", err.Error()) } + unisphereVersion := versionDetails.Version - if remoteSymID != "" && remoteVolumeID != "" { - remoteVol, err := pmaxClient.GetVolumeByID(ctx, remoteSymID, remoteVolumeID) - if strings.Compare(remoteVol.EffectiveWWN, vol.EffectiveWWN) != 0 { - // Refresh the symmetrix - err := pmaxClient.RefreshSymmetrix(ctx, symID) - if err != nil { - if !strings.Contains(err.Error(), "Too Many Requests") { - return nil, status.Errorf(codes.Internal, "PublishVolume: Could not refresh symmetrix: (%s)", err.Error()) - } - return nil, status.Errorf(codes.Internal, "symmetrix sync in progress, waiting for cache to update") - } - // wait till the remote volume has an effective wwn - return nil, status.Errorf(codes.Internal, "PublishVolume: Could not publish remote volume: (%s)", "remote volume does not have effective wwn, waiting for it to SYNC") - } - log.Debugf("remote-vol: %#v, error: %#v", remoteVol, err) - if err != nil { - return nil, status.Errorf(codes.Internal, "PublishVolume: Could not retrieve remote volume: (%s)", err.Error()) - } - publishContext[RemotePublishContextDeviceWWN] = remoteVol.EffectiveWWN - return s.publishVolume(ctx, publishContext, tgtStorageGroupID, hostID, symID, remoteSymID, tgtMaskingViewID, remoteVolumeID, reqID, volumeName, am, pmaxClient, false) - } - return ctrlPubRes, ctrlPubErr + publisher := publisherFor(ctx, s, pmaxClient, s.adminClient104, symID, devID, volID, volumeName, remoteSymID, remoteVolumeID, reqID, unisphereVersion, req.GetVolumeCapability()) + return publisher.Publish(ctx, req) } func (s *service) publishVolume(ctx context.Context, publishContext map[string]string, tgtStorageGroupID, hostID, clientSymID, symID, tgtMaskingViewID, deviceID, reqID, volumeName string, accessMode *csi.VolumeCapability_AccessMode, pmaxClient pmax.Pmax, isLocal bool) (*csi.ControllerPublishVolumeResponse, error) { @@ -2601,7 +1695,31 @@ func (s *service) updatePublishContext(ctx context.Context, publishContext map[s } } if lunid == "" { - return nil, status.Error(codes.Internal, "PublishContext: No matching connections for deviceID") + // For vSphere RDM, connections will not exist at publish time because + // the ESXi host has not yet discovered the LUN. The node plugin's + // RescanAllHba() in NodeStageVolume will trigger LUN discovery and + // AttachRDM() will map it into the VM. Build publish context from the + // masking view's port group instead of from connections. + if s.opts.IsVsphereEnabled { + log.Infof("PublishContext: No connections for vSphere volume %s, building context from masking view port group", deviceID) + mv, mvErr := pmaxClient.GetMaskingViewByID(ctx, symID, tgtMaskingViewID) + if mvErr != nil { + return nil, status.Errorf(codes.Internal, "PublishContext: Failed to get masking view %s: %s", tgtMaskingViewID, mvErr.Error()) + } + pg, pgErr := pmaxClient.GetPortGroupByID(ctx, symID, mv.PortGroupID) + if pgErr != nil { + return nil, status.Errorf(codes.Internal, "PublishContext: Failed to get port group %s: %s", mv.PortGroupID, pgErr.Error()) + } + for _, portKey := range pg.SymmetrixPortKey { + dirPorts = appendIfMissing(dirPorts, fmt.Sprintf("%s:%s", portKey.DirectorID, portKey.PortID)) + } + if len(dirPorts) == 0 { + return nil, status.Errorf(codes.Internal, "PublishContext: Port group %s has no ports configured", mv.PortGroupID) + } + lunid = "0000" + } else { + return nil, status.Error(codes.Internal, "PublishContext: No matching connections for deviceID") + } } portIdentifiers := "" @@ -3580,7 +2698,7 @@ func (s *service) SelectOrCreateFCPGForHost(ctx context.Context, symID string, h return "", fmt.Errorf("failed to find a valid initiator for hostID %s from %s", hostID, symID) } - versionDetails, err := pmaxClient.GetVersionDetails(ctx) + versionDetails, err := s.getVersionCache().getOrFetchVersionDetails(ctx, symID, pmaxClient) if err != nil { return "", fmt.Errorf("error in getversion API: %s", symID) } @@ -3595,7 +2713,7 @@ func (s *service) SelectOrCreateFCPGForHost(ctx context.Context, symID string, h } isV4 := s.isV4OrAbove(ctx, symID, pmaxClient) - if version >= 103 && isV4 { + if version >= APIVersion103 && isV4 { log.Debug("API version is greater than or equal to 103. Using enhanced API") portGroupList, err := pmaxClient.GetPortGroupListByType(ctx, symID, "fibre") if err != nil { @@ -3984,7 +3102,7 @@ func (s *service) ControllerExpandVolume( } if accTypeIsNFS([]*csi.VolumeCapability{req.VolumeCapability}) { - newSizeInMib := (req.CapacityRange.GetRequiredBytes()) / MiBSizeInBytes + newSizeInMib := req.CapacityRange.GetRequiredBytes() / MiBSizeInBytes return file.ExpandFileSystem(ctx, reqID, symID, devID, newSizeInMib, pmaxClient) } @@ -4976,12 +4094,23 @@ func (s *service) resolveParameter(params map[string]string, arrayID, param, def return defaultValue } +// getVersionCache returns the service's version cache, initializing it lazily if needed. +// Callers should not cache the returned pointer across calls. +func (s *service) getVersionCache() *versionCache { + s.versionCacheOnce.Do(func() { + if s.versionCache == nil { + s.versionCache = newVersionCache() + } + }) + return s.versionCache +} + // isV4OrAbove checks if the PowerMax array is V4 or above by examining the microcode version. // Take the first 2 digits of the microcode and convert to a number. // If the number is greater than 59, the array is V4 or above. func (s *service) isV4OrAbove(ctx context.Context, symID string, pmaxClient pmax.Pmax) bool { log := log.WithContext(ctx) - symmetrix, err := pmaxClient.GetSymmetrixByID(ctx, symID) + symmetrix, err := s.getVersionCache().getOrFetchSymmetrix(ctx, symID, pmaxClient) if err != nil { log.Warnf("Failed to get symmetrix info for %s: %s, defaulting to legacy clone", symID, err.Error()) return false diff --git a/service/controller_test.go b/service/controller_test.go index c745e210..4fc7f87f 100644 --- a/service/controller_test.go +++ b/service/controller_test.go @@ -15,6 +15,7 @@ package service import ( + "context" "errors" "fmt" "net/http" @@ -36,9 +37,45 @@ import ( "github.com/golang/mock/gomock" gmock "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" - "golang.org/x/net/context" ) +// DeletionWorker interface for testing purposes +type DeletionWorker interface { + QueueDeviceForDeletion(devID string, volumeIdentifier, symID string) error +} + +// MockDeletionWorker is a mock implementation of DeletionWorker interface +type MockDeletionWorker struct { + ctrl *gomock.Controller + recorder *MockDeletionWorkerMockRecorder +} + +type MockDeletionWorkerMockRecorder struct { + mock *MockDeletionWorker +} + +func NewMockDeletionWorker(ctrl *gomock.Controller) *MockDeletionWorker { + mock := &MockDeletionWorker{ctrl: ctrl} + mock.recorder = &MockDeletionWorkerMockRecorder{mock} + return mock +} + +func (m *MockDeletionWorker) EXPECT() *MockDeletionWorkerMockRecorder { + return m.recorder +} + +func (m *MockDeletionWorker) QueueDeviceForDeletion(devID string, volumeIdentifier, symID string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "QueueDeviceForDeletion", devID, volumeIdentifier, symID) + ret0, _ := ret[0].(error) + return ret0 +} + +func (mr *MockDeletionWorkerMockRecorder) QueueDeviceForDeletion(devID, volumeIdentifier, symID interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "QueueDeviceForDeletion", reflect.TypeOf((*MockDeletionWorker)(nil).QueueDeviceForDeletion), devID, volumeIdentifier, symID) +} + const ( KiB int64 = 1024 KB int64 = 1000 @@ -71,7 +108,6 @@ const ( type serviceFields struct { opts Opts mode string - pmaxTimeoutSeconds int64 adminClient pmax.Pmax deletionWorker *deletionWorker iscsiClient goiscsi.ISCSIinterface @@ -1093,7 +1129,6 @@ func Test_service_createMetroVolume(t *testing.T) { s := &service{ opts: tt.fields.opts, mode: tt.fields.mode, - pmaxTimeoutSeconds: tt.fields.pmaxTimeoutSeconds, adminClient: tt.fields.adminClient, deletionWorker: tt.fields.deletionWorker, iscsiClient: tt.fields.iscsiClient, @@ -1221,7 +1256,6 @@ func Test_service_getStoragePoolCapacities(t *testing.T) { s := &service{ opts: tt.fields.opts, mode: tt.fields.mode, - pmaxTimeoutSeconds: tt.fields.pmaxTimeoutSeconds, adminClient: tt.fields.adminClient, deletionWorker: tt.fields.deletionWorker, iscsiClient: tt.fields.iscsiClient, @@ -1379,7 +1413,6 @@ func Test_service_validateVolSize(t *testing.T) { s := &service{ opts: tt.fields.opts, mode: tt.fields.mode, - pmaxTimeoutSeconds: tt.fields.pmaxTimeoutSeconds, adminClient: tt.fields.adminClient, deletionWorker: tt.fields.deletionWorker, iscsiClient: tt.fields.iscsiClient, @@ -1476,7 +1509,6 @@ func Test_service_controllerProbe(t *testing.T) { s := &service{ opts: tt.fields.opts, mode: tt.fields.mode, - pmaxTimeoutSeconds: tt.fields.pmaxTimeoutSeconds, adminClient: tt.fields.adminClient, deletionWorker: tt.fields.deletionWorker, iscsiClient: tt.fields.iscsiClient, @@ -1573,7 +1605,6 @@ func Test_service_requireProbe(t *testing.T) { s := &service{ opts: tt.fields.opts, mode: tt.fields.mode, - pmaxTimeoutSeconds: tt.fields.pmaxTimeoutSeconds, adminClient: tt.fields.adminClient, deletionWorker: tt.fields.deletionWorker, iscsiClient: tt.fields.iscsiClient, @@ -1669,7 +1700,6 @@ func Test_service_SelectOrCreatePortGroup(t *testing.T) { s := &service{ opts: tt.fields.opts, mode: tt.fields.mode, - pmaxTimeoutSeconds: tt.fields.pmaxTimeoutSeconds, adminClient: tt.fields.adminClient, deletionWorker: tt.fields.deletionWorker, iscsiClient: tt.fields.iscsiClient, @@ -1809,7 +1839,6 @@ func Test_service_CreateRemoteVolume(t *testing.T) { s := &service{ opts: tt.fields.opts, mode: tt.fields.mode, - pmaxTimeoutSeconds: tt.fields.pmaxTimeoutSeconds, adminClient: tt.fields.adminClient, deletionWorker: tt.fields.deletionWorker, iscsiClient: tt.fields.iscsiClient, @@ -1920,7 +1949,6 @@ func Test_service_GetPortIdentifier(t *testing.T) { s := &service{ opts: tt.fields.opts, mode: tt.fields.mode, - pmaxTimeoutSeconds: tt.fields.pmaxTimeoutSeconds, adminClient: tt.fields.adminClient, deletionWorker: tt.fields.deletionWorker, iscsiClient: tt.fields.iscsiClient, @@ -2142,7 +2170,6 @@ func Test_service_CreateSnapshot(t *testing.T) { s := &service{ opts: tt.fields.opts, mode: tt.fields.mode, - pmaxTimeoutSeconds: tt.fields.pmaxTimeoutSeconds, adminClient: tt.fields.adminClient, deletionWorker: tt.fields.deletionWorker, iscsiClient: tt.fields.iscsiClient, @@ -2466,7 +2493,8 @@ func Test_service_verifyProtectionGroupID(t *testing.T) { // building a bad Storage Group ID "-" + localRDFGroupNum + "-" + Async, }, - }, nil) + }, nil, + ) return client }(), @@ -2490,7 +2518,8 @@ func Test_service_verifyProtectionGroupID(t *testing.T) { // building a bad Storage Group ID "-" + localRDFGroupNum + "-" + Async, }, - }, nil) + }, nil, + ) return client }(), @@ -2638,6 +2667,7 @@ func Test_service_ControllerPublishVolume(t *testing.T) { c.EXPECT().WithSymmetrixID(symIDLocal).AnyTimes().Return(c) c.EXPECT().GetHTTPClient().AnyTimes().Return(&http.Client{}) + c.EXPECT().GetVersionDetails(gomock.Any()).AnyTimes().Return(&types.VersionDetails{APIVersion: "103"}, nil) c.EXPECT().GetFileSystemByID(gomock.Any(), symIDLocal, gomock.Any()).Times(1).Return( &types.FileSystem{}, errors.New("failed to fetch file system"), ) @@ -2678,6 +2708,7 @@ func Test_service_ControllerPublishVolume(t *testing.T) { c.EXPECT().WithSymmetrixID(symIDLocal).AnyTimes().Return(c) c.EXPECT().GetHTTPClient().AnyTimes().Return(&http.Client{}) + c.EXPECT().GetVersionDetails(gomock.Any()).AnyTimes().Return(&types.VersionDetails{APIVersion: "103"}, nil) c.EXPECT().GetFileSystemByID(gomock.Any(), symIDLocal, gomock.Any()).Times(1).Return( &types.FileSystem{}, nil, // successfully returning a file system to simulate static provisioning ) @@ -3534,6 +3565,501 @@ func TestGetDynamicSG(t *testing.T) { } } +// setupFCInitiatorMocks sets up the common mock expectations for a Fibre host +// that successfully resolves initiators to SCSI_FC ports. +// Returns dirPort "FA-1D:0" in portListFromHost. +func setupFCInitiatorMocks(client *mocks.MockPmaxClient, symID string) { + client.EXPECT().GetPortListByProtocol(gomock.Any(), symID, "SCSI_FC").Return(&types.PortList{ + SymmetrixPortKey: []types.PortKey{ + {DirectorID: "FA-1D", PortID: "0"}, + }, + }, nil) + client.EXPECT().GetInitiatorList(gomock.Any(), symID, "5000000000000001", false, false).Return(&types.InitiatorList{ + InitiatorIDs: []string{"FA-1D:0:5000000000000001"}, + }, nil) +} + +func Test_service_SelectOrCreateFCPGForHost(t *testing.T) { + defaultHost := &types.Host{ + HostID: "host1", + HostType: "Fibre", + Initiators: []string{"5000000000000001"}, + } + + tests := []struct { + name string + symID string + host *types.Host + setup func(client *mocks.MockPmaxClient) + expectedPGID string + expectedError bool + errorMsg string + }{ + { + name: "nil host returns error", + symID: "000120000001", + host: nil, + setup: func(_ *mocks.MockPmaxClient) { + }, + expectedError: true, + errorMsg: "SelectOrCreateFCPGForHost: host can't be nil", + }, + { + name: "non-Fibre host type returns error for no valid initiators", + symID: "000120000001", + host: &types.Host{ + HostID: "host1", + HostType: "iSCSI", + Initiators: []string{}, + }, + setup: func(_ *mocks.MockPmaxClient) { + }, + expectedError: true, + errorMsg: "failed to find a valid initiator", + }, + { + name: "Fibre host with initiator on non-SCSI_FC port returns error", + symID: "000120000001", + host: &types.Host{ + HostID: "host1", + HostType: "Fibre", + Initiators: []string{"5000000000000001"}, + }, + setup: func(client *mocks.MockPmaxClient) { + client.EXPECT().GetPortListByProtocol(gomock.Any(), "000120000001", "SCSI_FC").Return(&types.PortList{ + SymmetrixPortKey: []types.PortKey{ + {DirectorID: "FA-1D", PortID: "0"}, + }, + }, nil) + client.EXPECT().GetInitiatorList(gomock.Any(), "000120000001", "5000000000000001", false, false).Return(&types.InitiatorList{ + InitiatorIDs: []string{"FA-2D:0:5000000000000001"}, + }, nil) + }, + expectedError: true, + errorMsg: "failed to find a valid initiator", + }, + { + name: "GetPortListByProtocol error returns error", + symID: "000120000001", + host: defaultHost, + setup: func(client *mocks.MockPmaxClient) { + client.EXPECT().GetPortListByProtocol(gomock.Any(), "000120000001", "SCSI_FC").Return(nil, errors.New("port list error")) + }, + expectedError: true, + errorMsg: "Failed to fetch SCSI_FC port", + }, + { + name: "GetInitiatorList error skips initiator and returns no valid initiator error", + symID: "000120000001", + host: defaultHost, + setup: func(client *mocks.MockPmaxClient) { + client.EXPECT().GetPortListByProtocol(gomock.Any(), "000120000001", "SCSI_FC").Return(&types.PortList{ + SymmetrixPortKey: []types.PortKey{ + {DirectorID: "FA-1D", PortID: "0"}, + }, + }, nil) + client.EXPECT().GetInitiatorList(gomock.Any(), "000120000001", "5000000000000001", false, false).Return(nil, errors.New("initiator error")) + }, + expectedError: true, + errorMsg: "failed to find a valid initiator", + }, + { + name: "GetVersionDetails error returns error", + symID: "000120000001", + host: defaultHost, + setup: func(client *mocks.MockPmaxClient) { + setupFCInitiatorMocks(client, "000120000001") + client.EXPECT().GetVersionDetails(gomock.Any()).Return(nil, errors.New("version error")) + }, + expectedError: true, + errorMsg: "error in getversion API", + }, + { + name: "non-numeric API version returns parsing error", + symID: "000120000001", + host: defaultHost, + setup: func(client *mocks.MockPmaxClient) { + setupFCInitiatorMocks(client, "000120000001") + client.EXPECT().GetVersionDetails(gomock.Any()).Return(&types.VersionDetails{ + APIVersion: "abc", + }, nil) + }, + expectedError: true, + errorMsg: "error in parsing Version", + }, + { + name: "enhanced API GetPortGroupListByType error returns error", + symID: "000120000001", + host: defaultHost, + setup: func(client *mocks.MockPmaxClient) { + setupFCInitiatorMocks(client, "000120000001") + // satisfy 103 version check + client.EXPECT().GetVersionDetails(gomock.Any()).Return(&types.VersionDetails{ + APIVersion: "103", + }, nil) + // satisfy v4 version check with anything above "59xx.xxx.x" + client.EXPECT().GetSymmetrixByID(gomock.Any(), "000120000001").Return(&types.Symmetrix{ + SymmetrixID: "000120000001", + Microcode: "6079.325.0", + }, nil) + client.EXPECT().GetPortGroupListByType(gomock.Any(), "000120000001", "fibre").Return(nil, errors.New("api error")) + }, + expectedError: true, + errorMsg: "failed to fetch Fibre channel port groups for array(enhanced API)", + }, + { + name: "enhanced API invalid base64-encoded port ID returns error", + symID: "000120000001", + host: defaultHost, + setup: func(client *mocks.MockPmaxClient) { + setupFCInitiatorMocks(client, "000120000001") + client.EXPECT().GetVersionDetails(gomock.Any()).Return(&types.VersionDetails{ + APIVersion: "103", + }, nil) + client.EXPECT().GetSymmetrixByID(gomock.Any(), "000120000001").Return(&types.Symmetrix{ + SymmetrixID: "000120000001", + Microcode: "6079.325.0", + }, nil) + client.EXPECT().GetPortGroupListByType(gomock.Any(), "000120000001", "fibre").Return(&types.PortGroupListResult{ + Results: []types.PortGroupListv1{ + { + ID: "pg1", + Protocol: FcIscsiID, + Ports: []types.PortValues{ + { + // bad base64 encoding triggers error + PortID: "!!!invalid-base64!!!", + Type: "Fibre", + Director: types.DirectorID{ID: "FA-1D"}, + }, + }, + }, + }, + }, nil) + }, + expectedError: true, + errorMsg: "Failed to fetch Fibre channel port ID", + }, + { + name: "enhanced API finds matching port group", + symID: "000120000001", + host: defaultHost, + setup: func(client *mocks.MockPmaxClient) { + setupFCInitiatorMocks(client, "000120000001") + client.EXPECT().GetVersionDetails(gomock.Any()).Return(&types.VersionDetails{ + APIVersion: "103", + }, nil) + client.EXPECT().GetSymmetrixByID(gomock.Any(), "000120000001").Return(&types.Symmetrix{ + SymmetrixID: "000120000001", + Microcode: "6079.325.0", + }, nil) + // base64.RawStdEncoding.Encode("FA-1D|0") => "RkEtMUR8MA" + client.EXPECT().GetPortGroupListByType(gomock.Any(), "000120000001", "fibre").Return(&types.PortGroupListResult{ + Results: []types.PortGroupListv1{ + { + ID: "pg-enhanced-match", + Protocol: FcIscsiID, + Ports: []types.PortValues{ + { + PortID: "RkEtMUR8MA", + Type: "Fibre", + Director: types.DirectorID{ID: "FA-1D"}, + }, + }, + }, + }, + }, nil) + }, + expectedPGID: "pg-enhanced-match", + expectedError: false, + }, + { + name: "enhanced API no match creates port group", + symID: "000120000001", + host: defaultHost, + setup: func(client *mocks.MockPmaxClient) { + setupFCInitiatorMocks(client, "000120000001") + client.EXPECT().GetVersionDetails(gomock.Any()).Return(&types.VersionDetails{ + APIVersion: "103", + }, nil) + client.EXPECT().GetSymmetrixByID(gomock.Any(), "000120000001").Return(&types.Symmetrix{ + SymmetrixID: "000120000001", + Microcode: "6079.325.0", + }, nil) + // base64.RawStdEncoding.Encode("FA-2D|0") => "RkEtMkR8MA" (different director) + client.EXPECT().GetPortGroupListByType(gomock.Any(), "000120000001", "fibre").Return(&types.PortGroupListResult{ + Results: []types.PortGroupListv1{ + { + ID: "pg-other", + Protocol: FcIscsiID, + Ports: []types.PortValues{ + { + PortID: "RkEtMkR8MA", + Type: "Fibre", + Director: types.DirectorID{ID: "FA-2D"}, + }, + }, + }, + }, + }, nil) + client.EXPECT().CreatePortGroup(gomock.Any(), "000120000001", "csi-ABC-FA-1D-0-PG", gomock.Any(), "SCSI_FC").Return(&types.PortGroup{}, nil) + }, + expectedPGID: "csi-ABC-FA-1D-0-PG", + expectedError: false, + }, + { + name: "legacy API GetPortGroupList error returns error", + symID: "000120000001", + host: defaultHost, + setup: func(client *mocks.MockPmaxClient) { + setupFCInitiatorMocks(client, "000120000001") + client.EXPECT().GetVersionDetails(gomock.Any()).Return(&types.VersionDetails{ + APIVersion: "102", + }, nil) + client.EXPECT().GetSymmetrixByID(gomock.Any(), "000120000001").Return(&types.Symmetrix{ + SymmetrixID: "000120000001", + Microcode: "5978.441.0", + }, nil) + client.EXPECT().GetPortGroupList(gomock.Any(), "000120000001", "fibre").Return(nil, errors.New("pg list error")) + }, + expectedError: true, + errorMsg: "Failed to fetch Fibre channel port groups for array:", + }, + { + name: "legacy API finds matching port group", + symID: "000120000001", + host: defaultHost, + setup: func(client *mocks.MockPmaxClient) { + setupFCInitiatorMocks(client, "000120000001") + client.EXPECT().GetVersionDetails(gomock.Any()).Return(&types.VersionDetails{ + APIVersion: "102", + }, nil) + client.EXPECT().GetSymmetrixByID(gomock.Any(), "000120000001").Return(&types.Symmetrix{ + SymmetrixID: "000120000001", + Microcode: "5978.441.0", + }, nil) + client.EXPECT().GetPortGroupList(gomock.Any(), "000120000001", "fibre").Return(&types.PortGroupList{ + PortGroupIDs: []string{"csi-ABC-pg1"}, + }, nil) + client.EXPECT().GetPortGroupByID(gomock.Any(), "000120000001", "csi-ABC-pg1").Return(&types.PortGroup{ + PortGroupID: "csi-ABC-pg1", + PortGroupType: "Fibre", + SymmetrixPortKey: []types.PortKey{ + {DirectorID: "FA-1D", PortID: "0"}, + }, + }, nil) + }, + expectedPGID: "csi-ABC-pg1", + expectedError: false, + }, + { + name: "legacy API filters port groups by csi prefix", + symID: "000120000001", + host: defaultHost, + setup: func(client *mocks.MockPmaxClient) { + setupFCInitiatorMocks(client, "000120000001") + client.EXPECT().GetVersionDetails(gomock.Any()).Return(&types.VersionDetails{ + APIVersion: "102", + }, nil) + client.EXPECT().GetSymmetrixByID(gomock.Any(), "000120000001").Return(&types.Symmetrix{ + SymmetrixID: "000120000001", + Microcode: "5978.441.0", + }, nil) + // Only "csi-ABC-pg1" matches prefix "csi-ABC"; "other-pg" does not + client.EXPECT().GetPortGroupList(gomock.Any(), "000120000001", "fibre").Return(&types.PortGroupList{ + PortGroupIDs: []string{"other-pg", "csi-ABC-pg1"}, + }, nil) + client.EXPECT().GetPortGroupByID(gomock.Any(), "000120000001", "csi-ABC-pg1").Return(&types.PortGroup{ + PortGroupID: "csi-ABC-pg1", + PortGroupType: "SCSI_FC", + SymmetrixPortKey: []types.PortKey{ + {DirectorID: "FA-1D", PortID: "0"}, + }, + }, nil) + }, + expectedPGID: "csi-ABC-pg1", + expectedError: false, + }, + { + name: "legacy API GetPortGroupByID error continues to next", + symID: "000120000001", + host: defaultHost, + setup: func(client *mocks.MockPmaxClient) { + setupFCInitiatorMocks(client, "000120000001") + client.EXPECT().GetVersionDetails(gomock.Any()).Return(&types.VersionDetails{ + APIVersion: "102", + }, nil) + client.EXPECT().GetSymmetrixByID(gomock.Any(), "000120000001").Return(&types.Symmetrix{ + SymmetrixID: "000120000001", + Microcode: "5978.441.0", + }, nil) + client.EXPECT().GetPortGroupList(gomock.Any(), "000120000001", "fibre").Return(&types.PortGroupList{ + PortGroupIDs: []string{"csi-ABC-pg1", "csi-ABC-pg2"}, + }, nil) + // First PG errors, second PG matches + client.EXPECT().GetPortGroupByID(gomock.Any(), "000120000001", "csi-ABC-pg1").Return(nil, errors.New("pg error")) + client.EXPECT().GetPortGroupByID(gomock.Any(), "000120000001", "csi-ABC-pg2").Return(&types.PortGroup{ + PortGroupID: "csi-ABC-pg2", + PortGroupType: "Fibre", + SymmetrixPortKey: []types.PortKey{ + {DirectorID: "FA-1D", PortID: "0"}, + }, + }, nil) + }, + expectedPGID: "csi-ABC-pg2", + expectedError: false, + }, + { + name: "legacy API no match creates port group", + symID: "000120000001", + host: defaultHost, + setup: func(client *mocks.MockPmaxClient) { + setupFCInitiatorMocks(client, "000120000001") + client.EXPECT().GetVersionDetails(gomock.Any()).Return(&types.VersionDetails{ + APIVersion: "102", + }, nil) + client.EXPECT().GetSymmetrixByID(gomock.Any(), "000120000001").Return(&types.Symmetrix{ + SymmetrixID: "000120000001", + Microcode: "5978.441.0", + }, nil) + client.EXPECT().GetPortGroupList(gomock.Any(), "000120000001", "fibre").Return(&types.PortGroupList{ + PortGroupIDs: []string{"other-pg"}, + }, nil) + client.EXPECT().CreatePortGroup(gomock.Any(), "000120000001", "csi-ABC-FA-1D-0-PG", gomock.Any(), "SCSI_FC").Return(&types.PortGroup{}, nil) + }, + expectedPGID: "csi-ABC-FA-1D-0-PG", + expectedError: false, + }, + { + name: "new API with legacy microcode creates port group", + symID: "000120000001", + host: defaultHost, + setup: func(client *mocks.MockPmaxClient) { + setupFCInitiatorMocks(client, "000120000001") + client.EXPECT().GetVersionDetails(gomock.Any()).Return(&types.VersionDetails{ + APIVersion: "103", + }, nil) + client.EXPECT().GetSymmetrixByID(gomock.Any(), "000120000001").Return(&types.Symmetrix{ + SymmetrixID: "000120000001", + Microcode: "5978.441.0", + }, nil) + client.EXPECT().GetPortGroupList(gomock.Any(), "000120000001", "fibre").Return(&types.PortGroupList{ + PortGroupIDs: []string{"other-pg"}, + }, nil) + client.EXPECT().CreatePortGroup(gomock.Any(), "000120000001", "csi-ABC-FA-1D-0-PG", gomock.Any(), "SCSI_FC").Return(&types.PortGroup{}, nil) + }, + expectedPGID: "csi-ABC-FA-1D-0-PG", + expectedError: false, + }, + { + name: "CreatePortGroup error returns error", + symID: "000120000001", + host: defaultHost, + setup: func(client *mocks.MockPmaxClient) { + setupFCInitiatorMocks(client, "000120000001") + client.EXPECT().GetVersionDetails(gomock.Any()).Return(&types.VersionDetails{ + APIVersion: "102", + }, nil) + client.EXPECT().GetSymmetrixByID(gomock.Any(), "000120000001").Return(&types.Symmetrix{ + SymmetrixID: "000120000001", + Microcode: "5978.441.0", + }, nil) + client.EXPECT().GetPortGroupList(gomock.Any(), "000120000001", "fibre").Return(&types.PortGroupList{ + PortGroupIDs: []string{"other-pg"}, + }, nil) + client.EXPECT().CreatePortGroup(gomock.Any(), "000120000001", "csi-ABC-FA-1D-0-PG", gomock.Any(), "SCSI_FC").Return(nil, errors.New("create error")) + }, + expectedError: true, + errorMsg: "Failed to create PortGroup", + }, + { + name: "legacy API port group with non-Fibre type is skipped", + symID: "000120000001", + host: defaultHost, + setup: func(client *mocks.MockPmaxClient) { + setupFCInitiatorMocks(client, "000120000001") + client.EXPECT().GetVersionDetails(gomock.Any()).Return(&types.VersionDetails{ + APIVersion: "102", + }, nil) + client.EXPECT().GetSymmetrixByID(gomock.Any(), "000120000001").Return(&types.Symmetrix{ + SymmetrixID: "000120000001", + Microcode: "5978.441.0", + }, nil) + client.EXPECT().GetPortGroupList(gomock.Any(), "000120000001", "fibre").Return(&types.PortGroupList{ + PortGroupIDs: []string{"csi-ABC-pg1"}, + }, nil) + // PortGroupType is iSCSI, not Fibre/SCSI_FC, so it should be skipped + client.EXPECT().GetPortGroupByID(gomock.Any(), "000120000001", "csi-ABC-pg1").Return(&types.PortGroup{ + PortGroupID: "csi-ABC-pg1", + PortGroupType: "iSCSI", + SymmetrixPortKey: []types.PortKey{ + {DirectorID: "FA-1D", PortID: "0"}, + }, + }, nil) + // No match → creates PG + client.EXPECT().CreatePortGroup(gomock.Any(), "000120000001", "csi-ABC-FA-1D-0-PG", gomock.Any(), "SCSI_FC").Return(&types.PortGroup{}, nil) + }, + expectedPGID: "csi-ABC-FA-1D-0-PG", + expectedError: false, + }, + { + name: "legacy API port group with mismatched ports creates new PG", + symID: "000120000001", + host: defaultHost, + setup: func(client *mocks.MockPmaxClient) { + setupFCInitiatorMocks(client, "000120000001") + client.EXPECT().GetVersionDetails(gomock.Any()).Return(&types.VersionDetails{ + APIVersion: "102", + }, nil) + client.EXPECT().GetSymmetrixByID(gomock.Any(), "000120000001").Return(&types.Symmetrix{ + SymmetrixID: "000120000001", + Microcode: "5978.441.0", + }, nil) + client.EXPECT().GetPortGroupList(gomock.Any(), "000120000001", "fibre").Return(&types.PortGroupList{ + PortGroupIDs: []string{"csi-ABC-pg1"}, + }, nil) + // Different ports than what the host has + client.EXPECT().GetPortGroupByID(gomock.Any(), "000120000001", "csi-ABC-pg1").Return(&types.PortGroup{ + PortGroupID: "csi-ABC-pg1", + PortGroupType: "Fibre", + SymmetrixPortKey: []types.PortKey{ + {DirectorID: "FA-2D", PortID: "1"}, + }, + }, nil) + client.EXPECT().CreatePortGroup(gomock.Any(), "000120000001", "csi-ABC-FA-1D-0-PG", gomock.Any(), "SCSI_FC").Return(&types.PortGroup{}, nil) + }, + expectedPGID: "csi-ABC-FA-1D-0-PG", + expectedError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + client := mocks.NewMockPmaxClient(ctrl) + client.EXPECT().GetHTTPClient().AnyTimes().Return(&http.Client{}) + tt.setup(client) + + svc := &service{ + opts: Opts{ + ClusterPrefix: "ABC", + }, + } + pgID, err := svc.SelectOrCreateFCPGForHost(context.Background(), tt.symID, tt.host, client) + if tt.expectedError { + assert.Error(t, err) + if tt.errorMsg != "" { + assert.Contains(t, err.Error(), tt.errorMsg) + } + } else { + assert.NoError(t, err) + assert.Equal(t, tt.expectedPGID, pgID) + } + }) + } +} + func Test_service_isV4OrAbove(t *testing.T) { tests := []struct { name string @@ -3633,6 +4159,7 @@ func Test_service_isV4OrAbove(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() client := mocks.NewMockPmaxClient(ctrl) + client.EXPECT().GetHTTPClient().AnyTimes().Return(&http.Client{}) tt.setup(client) svc := &service{} @@ -3641,3 +4168,343 @@ func Test_service_isV4OrAbove(t *testing.T) { }) } } + +func TestDeleteVolumeWithDeletionPrefix(t *testing.T) { + // Test to verify that volumes with _DEL prefix are processed for deletion + // rather than being skipped due to VolumeIdentifier mismatch + tests := []struct { + name string + volumeIdentifier string + volName string + expectContinue bool + }{ + { + name: "Volume with _DEL prefix containing original name should continue", + volumeIdentifier: "_DEL_csi-test-cluster-my-volume", + volName: "csi-test-cluster-my-volume", + expectContinue: true, + }, + { + name: "Volume without _DEL prefix mismatch should return nil", + volumeIdentifier: "csi-test-cluster-different-volume", + volName: "csi-test-cluster-my-volume", + expectContinue: false, + }, + { + name: "Volume with _DEL prefix but different name should return nil", + volumeIdentifier: "_DEL_csi-test-cluster-different-volume", + volName: "csi-test-cluster-my-volume", + expectContinue: false, + }, + { + name: "Volume with exact match should not enter this logic", + volumeIdentifier: "csi-test-cluster-my-volume", + volName: "csi-test-cluster-my-volume", + expectContinue: false, // This case won't enter the != check + }, + { + name: "Volume with _DEL prefix but empty original name", + volumeIdentifier: "_DEL_", + volName: "", + expectContinue: true, + }, + { + name: "Volume with _DEL prefix and partial name match", + volumeIdentifier: "_DEL_csi-test-cluster", + volName: "csi-test-cluster", + expectContinue: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Test the logic that was added to deleteVolume function + // Only test the != case, as exact match bypasses this logic + if tt.volumeIdentifier != tt.volName { + hasDeletionPrefix := strings.HasPrefix(tt.volumeIdentifier, DeletionPrefix) + containsOriginalName := strings.Contains(tt.volumeIdentifier, tt.volName) + + shouldContinue := hasDeletionPrefix && containsOriginalName + + assert.Equal(t, tt.expectContinue, shouldContinue, + "Expected continue=%v for VolumeIdentifier=%s, volName=%s", + tt.expectContinue, tt.volumeIdentifier, tt.volName) + } else { + // For exact matches, this logic shouldn't be reached + assert.False(t, tt.expectContinue, + "Exact match case should not enter the != logic") + } + }) + } +} + +func TestDeleteVolumeRetryQueuing(t *testing.T) { + // Test for the new enhancement where volumes with deletion prefix that failed + // to be queued get re-attempted for queuing instead of being assumed deleted + tests := []struct { + name string + volumeIdentifier string + volName string + expectRetryCall bool + expectedDelName string + description string + }{ + { + name: "Volume with exact deletion prefix should retry queuing", + volumeIdentifier: "_DELcsi-test-cluster-my-volume", + volName: "csi-test-cluster-my-volume", + expectRetryCall: true, + expectedDelName: "_DELcsi-test-cluster-my-volume", + description: "Volume was renamed for deletion but not yet queued", + }, + { + name: "Volume with deletion prefix but different name should not retry", + volumeIdentifier: "_DELcsi-test-cluster-different-volume", + volName: "csi-test-cluster-my-volume", + expectRetryCall: false, + expectedDelName: "_DELcsi-test-cluster-my-volume", + description: "Different volume name, should be assumed deleted", + }, + { + name: "Volume without deletion prefix should not retry", + volumeIdentifier: "csi-test-cluster-my-volume", + volName: "csi-test-cluster-my-volume", + expectRetryCall: false, + expectedDelName: "_DELcsi-test-cluster-my-volume", + description: "Normal case, exact match bypasses this logic", + }, + { + name: "Volume with truncated deletion prefix should retry", + volumeIdentifier: "_DELvery-long-volume-name-that-exceeds-maximum-identifier-length", + volName: "very-long-volume-name-that-exceeds-maximum-identifier-length-and-should-be-truncated", + expectRetryCall: true, + expectedDelName: "_DELvery-long-volume-name-that-exceeds-maximum-identifier-length", // Truncated to MaxVolIdentifierLength + description: "Test truncation logic for long identifiers", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + // Create mocks + mockDeletionWorker := NewMockDeletionWorker(mockCtrl) + + // Test the logic from the new enhancement + if tt.volumeIdentifier != tt.volName { + // Calculate expected deletion name (same logic as in the actual code) + expectedDelName := fmt.Sprintf("%s%s", DeletionPrefix, tt.volName) + if len(expectedDelName) > MaxVolIdentifierLength { + expectedDelName = expectedDelName[:MaxVolIdentifierLength] + } + + // Check if the volume identifier matches the expected deletion name + shouldRetry := tt.volumeIdentifier == expectedDelName + + if shouldRetry && tt.expectRetryCall { + // Mock the QueueDeviceForDeletion call + mockDeletionWorker.EXPECT().QueueDeviceForDeletion(gomock.Any(), tt.volumeIdentifier, gomock.Any()).Return(nil) + + // Simulate the retry logic + err := mockDeletionWorker.QueueDeviceForDeletion("test-volume-id", tt.volumeIdentifier, "test-symid") + assert.NoError(t, err, "QueueDeviceForDeletion should not return error") + } + + assert.Equal(t, tt.expectRetryCall, shouldRetry, + "Expected retry call=%v for VolumeIdentifier=%s, volName=%s", + tt.expectRetryCall, tt.volumeIdentifier, tt.volName) + + assert.Equal(t, tt.expectedDelName, expectedDelName, + "Expected deletion name mismatch") + } + }) + } +} + +func TestDeleteVolumeRetryQueuingError(t *testing.T) { + // Test error handling when retry queuing fails + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + mockDeletionWorker := NewMockDeletionWorker(mockCtrl) + + // Mock QueueDeviceForDeletion to return an error + expectedError := errors.New("failed to queue device for deletion") + mockDeletionWorker.EXPECT().QueueDeviceForDeletion(gomock.Any(), gomock.Any(), gomock.Any()).Return(expectedError) + + // Test the error scenario + err := mockDeletionWorker.QueueDeviceForDeletion("test-volume-id", "_DEL_test-volume", "test-symid") + assert.Error(t, err, "QueueDeviceForDeletion should return error") + assert.Contains(t, err.Error(), "failed to queue device for deletion", "Error message should match") +} + +func Test_service_updatePublishContext(t *testing.T) { + symID := "000120000001" + mvID := "csi-mv--worker-1" + devID := "011AB" + portGroupID := "csi-vsphere-VC-PG" + + // Minimize retry delay for tests + origDelay := getMVConnectionsDelay + getMVConnectionsDelay = 1 * time.Millisecond + defer func() { getMVConnectionsDelay = origDelay }() + + tests := []struct { + name string + isVsphere bool + connections []*types.MaskingViewConnection + setupMock func(client *mocks.MockPmaxClient) + expectErr bool + errContains string + expectLUN string + expectDirPorts int + }{ + { + name: "vSphere: no connections, builds context from port group", + isVsphere: true, + connections: []*types.MaskingViewConnection{ + // connection for a different volume so our devID gets lunid="" + {VolumeID: "OTHER", HostLUNAddress: "0001", DirectorPort: "SE-1E:4"}, + }, + setupMock: func(client *mocks.MockPmaxClient) { + client.EXPECT().GetMaskingViewConnections(gomock.Any(), symID, mvID, devID). + Return([]*types.MaskingViewConnection{}, nil).AnyTimes() + client.EXPECT().GetMaskingViewByID(gomock.Any(), symID, mvID). + Return(&types.MaskingView{ + MaskingViewID: mvID, + PortGroupID: portGroupID, + }, nil).Times(1) + client.EXPECT().GetPortGroupByID(gomock.Any(), symID, portGroupID). + Return(&types.PortGroup{ + PortGroupID: portGroupID, + SymmetrixPortKey: []types.PortKey{ + {DirectorID: "OR-1C", PortID: "4"}, + {DirectorID: "OR-2C", PortID: "4"}, + }, + }, nil).Times(1) + client.EXPECT().GetPort(gomock.Any(), symID, "OR-1C", "4"). + Return(&types.Port{ + SymmetrixPort: types.SymmetrixPortType{ + Identifier: "50000973f0064001", + }, + }, nil).Times(1) + client.EXPECT().GetPort(gomock.Any(), symID, "OR-2C", "4"). + Return(&types.Port{ + SymmetrixPort: types.SymmetrixPortType{ + Identifier: "50000973f0064002", + }, + }, nil).Times(1) + }, + expectErr: false, + expectLUN: "0000", + }, + { + name: "non-vSphere: no connections returns error", + isVsphere: false, + connections: []*types.MaskingViewConnection{}, + setupMock: func(client *mocks.MockPmaxClient) { + client.EXPECT().GetMaskingViewConnections(gomock.Any(), symID, mvID, devID). + Return([]*types.MaskingViewConnection{}, nil).AnyTimes() + }, + expectErr: true, + errContains: "No matching connections for deviceID", + }, + { + name: "vSphere: GetMaskingViewByID fails", + isVsphere: true, + connections: []*types.MaskingViewConnection{ + {VolumeID: "OTHER", HostLUNAddress: "0001", DirectorPort: "SE-1E:4"}, + }, + setupMock: func(client *mocks.MockPmaxClient) { + client.EXPECT().GetMaskingViewConnections(gomock.Any(), symID, mvID, devID). + Return([]*types.MaskingViewConnection{}, nil).AnyTimes() + client.EXPECT().GetMaskingViewByID(gomock.Any(), symID, mvID). + Return(nil, errors.New("masking view not found")).Times(1) + }, + expectErr: true, + errContains: "Failed to get masking view", + }, + { + name: "vSphere: GetPortGroupByID fails", + isVsphere: true, + connections: []*types.MaskingViewConnection{ + {VolumeID: "OTHER", HostLUNAddress: "0001", DirectorPort: "SE-1E:4"}, + }, + setupMock: func(client *mocks.MockPmaxClient) { + client.EXPECT().GetMaskingViewConnections(gomock.Any(), symID, mvID, devID). + Return([]*types.MaskingViewConnection{}, nil).AnyTimes() + client.EXPECT().GetMaskingViewByID(gomock.Any(), symID, mvID). + Return(&types.MaskingView{ + MaskingViewID: mvID, + PortGroupID: portGroupID, + }, nil).Times(1) + client.EXPECT().GetPortGroupByID(gomock.Any(), symID, portGroupID). + Return(nil, errors.New("port group not found")).Times(1) + }, + expectErr: true, + errContains: "Failed to get port group", + }, + { + name: "vSphere: empty port group returns error", + isVsphere: true, + connections: []*types.MaskingViewConnection{ + {VolumeID: "OTHER", HostLUNAddress: "0001", DirectorPort: "SE-1E:4"}, + }, + setupMock: func(client *mocks.MockPmaxClient) { + client.EXPECT().GetMaskingViewConnections(gomock.Any(), symID, mvID, devID). + Return([]*types.MaskingViewConnection{}, nil).AnyTimes() + client.EXPECT().GetMaskingViewByID(gomock.Any(), symID, mvID). + Return(&types.MaskingView{ + MaskingViewID: mvID, + PortGroupID: portGroupID, + }, nil).Times(1) + client.EXPECT().GetPortGroupByID(gomock.Any(), symID, portGroupID). + Return(&types.PortGroup{ + PortGroupID: portGroupID, + SymmetrixPortKey: []types.PortKey{}, + }, nil).Times(1) + }, + expectErr: true, + errContains: "has no ports configured", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + LockRequestHandler() + + mockClient := mocks.NewMockPmaxClient(ctrl) + tt.setupMock(mockClient) + + svc := &service{ + opts: Opts{ + IsVsphereEnabled: tt.isVsphere, + }, + } + getPmaxCache(symID) + + publishContext := make(map[string]string) + resp, err := svc.updatePublishContext( + context.Background(), publishContext, symID, mvID, devID, "req-1", + tt.connections, mockClient, true, + ) + + if tt.expectErr { + assert.Error(t, err) + if tt.errContains != "" { + assert.Contains(t, err.Error(), tt.errContains) + } + assert.Nil(t, resp) + } else { + assert.NoError(t, err) + assert.NotNil(t, resp) + assert.Equal(t, tt.expectLUN, resp.PublishContext[PublishContextLUNAddress]) + assert.NotEmpty(t, resp.PublishContext[PortIdentifiers+"_1"]) + } + }) + } +} diff --git a/service/csi_ctrl_to_node_connectivity_test.go b/service/csi_ctrl_to_node_connectivity_test.go index 5d343298..1617a7fb 100644 --- a/service/csi_ctrl_to_node_connectivity_test.go +++ b/service/csi_ctrl_to_node_connectivity_test.go @@ -15,13 +15,13 @@ package service import ( + "context" "net/http" "net/http/httptest" "testing" "time" "github.com/stretchr/testify/assert" - "golang.org/x/net/context" ) // TestQueryArrayStatus tests the query array status diff --git a/service/deletion_worker.go b/service/deletion_worker.go index b878ecb8..2d62f79f 100644 --- a/service/deletion_worker.go +++ b/service/deletion_worker.go @@ -50,7 +50,10 @@ const ( FinalError = "Final error: Max error count reached, device will be removed from Deletion Queue" ) -var waitTillSyncInProgTime = 20 * time.Second +var ( + waitTillSyncInProgTime = 20 * time.Second + APIPropagationDelay = 2 * time.Second +) // symDeviceID - holds a hexadecimal device id in string as well as the corresponding integer value type symDeviceID struct { @@ -97,6 +100,7 @@ type deletionQueue struct { DeleteTracks bool ClusterPrefix string lock sync.Mutex + versionCache *versionCache } // deletionWorker - represents the deletion worker @@ -108,6 +112,15 @@ type deletionWorker struct { DeletionRequestChan chan deletionRequest DeletionQueueChan chan csiDevice State string + stopChan chan struct{} + versionCache *versionCache +} + +// Stop signals the deletion worker goroutines to exit +func (worker *deletionWorker) Stop() { + if worker.stopChan != nil { + close(worker.stopChan) + } } func (req *deletionRequest) isValid(clusterPrefix string) error { @@ -480,12 +493,26 @@ func (queue *deletionQueue) removeVolumesFromStorageGroup(pmaxClient pmax.Pmax) log.Debugf("GetRDFInfoFromSGID failed for (%s) on symID (%s). Proceeding for RemoveVolumesFromStorageGroup", sgID, queue.SymID) // This is the default SG in which all the volumes are replicated _, err = pmaxClient.RemoveVolumesFromStorageGroup(context.Background(), queue.SymID, sgID, true, volumeIDs...) + // If batch removal failed because a volume is no longer in the SG, + // exclude the stale volume(s) and retry the batch (ECS01A-920). + if err != nil && strings.Contains(err.Error(), "does not contain") { + retryIDs := make([]string, 0, len(volumeIDs)) + for _, vid := range volumeIDs { + if !strings.Contains(err.Error(), vid) { + retryIDs = append(retryIDs, vid) + } + } + if len(retryIDs) > 0 && len(retryIDs) < len(volumeIDs) { + log.Warnf("SG %s does not contain some volume(s). Retrying batch removal with %d remaining volumes.", sgID, len(retryIDs)) + _, err = pmaxClient.RemoveVolumesFromStorageGroup(context.Background(), queue.SymID, sgID, true, retryIDs...) + } + } } else { // replicated volumes // RemoveVolumesFromProtectedStorageGroup should be done only from r1 type - psg, err := pmaxClient.GetStorageGroupRDFInfo(context.Background(), queue.SymID, sgID, rdfNo) - if err != nil { - if strings.Contains(err.Error(), "No SRDF records found") { + psg, sgRDFErr := pmaxClient.GetStorageGroupRDFInfo(context.Background(), queue.SymID, sgID, rdfNo) + if sgRDFErr != nil { + if strings.Contains(sgRDFErr.Error(), "No SRDF records found") { // it is empty protected storage group return true } @@ -497,8 +524,8 @@ func (queue *deletionQueue) removeVolumesFromStorageGroup(pmaxClient pmax.Pmax) return true } log.Debugf("LocalSG: (%s), Mode: (%s), RDF No: (%s), Namespace: (%s)", sgID, mode, rdfNo, ns) - rdfInfo, err := pmaxClient.GetRDFGroupByID(context.Background(), queue.SymID, rdfNo) - if err != nil { + rdfInfo, rdfErr := pmaxClient.GetRDFGroupByID(context.Background(), queue.SymID, rdfNo) + if rdfErr != nil { log.Errorf("GetRDFGroup failed for (%s) on symID (%s)", sgID, queue.SymID) return false } @@ -521,6 +548,14 @@ func (queue *deletionQueue) removeVolumesFromStorageGroup(pmaxClient pmax.Pmax) _, err = pmaxClient.RemoveVolumesFromProtectedStorageGroup(context.Background(), queue.SymID, sgID, rdfInfo.RemoteSymmetrix, remoteSGID, true, volumeIDs...) } + // Allow Unisphere API to propagate SG membership changes before + // refreshing volume caches. Without this delay, cache refresh may + // return stale data causing devices to remain stuck in disAssociateSG + // state and poison subsequent batches (ECS01A-920). + if err == nil { + time.Sleep(APIPropagationDelay) + } + for _, volumeID := range volumeIDs { device := getDevice(volumeID, queue.DeviceList) if device != nil { @@ -568,7 +603,11 @@ func (queue *deletionQueue) deleteVolumes(pmaxClient pmax.Pmax) bool { return false } - versionDetails, err := pmaxClient.GetVersionDetails(ctx) + vc := queue.versionCache + if vc == nil { + vc = newVersionCache() + } + versionDetails, err := vc.getOrFetchVersionDetails(ctx, queue.SymID, pmaxClient) if err != nil { log.Errorf("error getting api version of array %s, Error: %s", queue.SymID, err.Error()) } else if versionDetails.APIVersion != "" { @@ -579,7 +618,7 @@ func (queue *deletionQueue) deleteVolumes(pmaxClient pmax.Pmax) bool { } // Usebulk Volume get if api is greater than or 101 - if version >= 101 { + if version >= APIVersion101 { volDeletePrefix := DeletionPrefix + CSIPrefix + "-" + queue.ClusterPrefix log.Infof("Deletion Prefix: %s", volDeletePrefix) volumev1, err := pmaxClient.GetVolumesByIdentifierMatch(ctx, queue.SymID, volDeletePrefix) @@ -599,7 +638,7 @@ func (queue *deletionQueue) deleteVolumes(pmaxClient pmax.Pmax) bool { var volumeIdentifier string // Check once more if volume is part of any storage groups - if enhancedVolume, ok := volumeMap[device.SymDeviceID.DeviceID]; ok && version >= 101 { + if enhancedVolume, ok := volumeMap[device.SymDeviceID.DeviceID]; ok && version >= APIVersion101 { sgCount = len(enhancedVolume.StorageGroups) volumeIdentifier = enhancedVolume.Identifier log.Debugf("Enhanced volumeIdentifier: %s and SG Count: %d", volumeIdentifier, sgCount) @@ -714,6 +753,8 @@ func (worker *deletionWorker) pruneDeletionQueues() { func (worker *deletionWorker) updateDeletionQueues(duration time.Duration) { for afterCh := time.After(duration); ; { select { + case <-worker.stopChan: + return case device := <-worker.DeletionQueueChan: queue, ok := worker.DeletionQueues[device.SymID] if ok { @@ -755,60 +796,68 @@ func (worker *deletionWorker) QueueDeviceForDeletion(devID string, volumeIdentif func (worker *deletionWorker) deletionRequestHandler() { log.Info("Starting deletion request handler goroutine") - for req := range worker.DeletionRequestChan { - log.Infof("Received deletion request for Device ID: %s, Sym ID: %s", req.DeviceID, req.SymID) - if !isStringInSlice(req.SymID, worker.SymmetrixIDs) { - req.errChan <- fmt.Errorf("unable to process device deletion request as sym id is not managed by deletion worker") - continue - } - pmaxClient, err := symmetrix.GetPowerMaxClient(req.SymID) - if err != nil { - log.Error(err.Error()) - req.errChan <- fmt.Errorf("unable to process device deletion request as sym id is not managed by deletion worker") - continue - } - vol, err := pmaxClient.GetVolumeByID(context.Background(), req.SymID, req.DeviceID) - if err != nil { - req.errChan <- err - continue - } - err = req.isValid(worker.ClusterPrefix) - if err != nil { - req.errChan <- err - continue - } - deviceID, err := getDeviceID(req.DeviceID) - if err != nil { - req.errChan <- err - continue - } - currentTime := time.Now() - symVolume := symVolumeCache{ - volume: vol, - lastUpdate: currentTime, - } + for { + select { + case <-worker.stopChan: + return + case req, ok := <-worker.DeletionRequestChan: + if !ok { + return + } + log.Infof("Received deletion request for Device ID: %s, Sym ID: %s", req.DeviceID, req.SymID) + if !isStringInSlice(req.SymID, worker.SymmetrixIDs) { + req.errChan <- fmt.Errorf("unable to process device deletion request as sym id is not managed by deletion worker") + continue + } + pmaxClient, err := symmetrix.GetPowerMaxClient(req.SymID) + if err != nil { + log.Error(err.Error()) + req.errChan <- fmt.Errorf("unable to process device deletion request as sym id is not managed by deletion worker") + continue + } + vol, err := pmaxClient.GetVolumeByID(context.Background(), req.SymID, req.DeviceID) + if err != nil { + req.errChan <- err + continue + } + err = req.isValid(worker.ClusterPrefix) + if err != nil { + req.errChan <- err + continue + } + deviceID, err := getDeviceID(req.DeviceID) + if err != nil { + req.errChan <- err + continue + } + currentTime := time.Now() + symVolume := symVolumeCache{ + volume: vol, + lastUpdate: currentTime, + } - initialState := deletionStateDisAssociateSG - if len(vol.StorageGroupIDList) == 0 { - if vol.SnapSource || vol.SnapTarget { - initialState = deletionStateCleanupSnaps - } else { - initialState = deletionStateDeleteVol + initialState := deletionStateDisAssociateSG + if len(vol.StorageGroupIDList) == 0 { + if vol.SnapSource || vol.SnapTarget { + initialState = deletionStateCleanupSnaps + } else { + initialState = deletionStateDeleteVol + } } + device := csiDevice{ + SymDeviceID: deviceID, + SymID: req.SymID, + SymVolumeCache: symVolume, + VolumeIdentifier: req.VolumeHandle, + Status: deletionRequestStatus{ + AdditionTime: currentTime, + LastUpdate: currentTime, + State: initialState, + }, + } + worker.DeletionQueueChan <- device + req.errChan <- err } - device := csiDevice{ - SymDeviceID: deviceID, - SymID: req.SymID, - SymVolumeCache: symVolume, - VolumeIdentifier: req.VolumeHandle, - Status: deletionRequestStatus{ - AdditionTime: currentTime, - LastUpdate: currentTime, - State: initialState, - }, - } - worker.DeletionQueueChan <- device - req.errChan <- err } } @@ -818,6 +867,11 @@ func (worker *deletionWorker) deletionWorker() { symIndex := 0 updated := false for { + select { + case <-worker.stopChan: + return + default: + } worker.nextStep() switch worker.State { case cleanupSnapshotStep: @@ -883,12 +937,14 @@ func (worker *deletionWorker) populateDeletionQueue() { continue } - versionDetails, err := pmaxClient.GetVersionDetails(ctx) + vc := worker.versionCache + if vc == nil { + vc = newVersionCache() + } + versionDetails, err := vc.getOrFetchVersionDetails(ctx, symID, pmaxClient) if err != nil { log.Errorf("error getting api version of array %s, Error: %s", symID, err.Error()) - } - - if versionDetails.APIVersion != "" { + } else if versionDetails.APIVersion != "" { version, err = strconv.Atoi(versionDetails.APIVersion) if err != nil { log.Errorf("error in parsing Version: %s", err.Error()) @@ -898,7 +954,7 @@ func (worker *deletionWorker) populateDeletionQueue() { log.Infof("Deletion Prefix: %s", volDeletePrefix) // Usebulk Volume get if api is greater than or 101 - if version >= 101 { + if version >= APIVersion101 { volDeletePrefix := DeletionPrefix + CSIPrefix + "-" + worker.ClusterPrefix volumev1, err := pmaxClient.GetVolumesByIdentifierMatch(ctx, symID, volDeletePrefix) if err != nil { @@ -965,11 +1021,14 @@ func (s *service) NewDeletionWorker(clusterPrefix string, symIDs []string) { delWorker.SymmetrixIDs = symIDs delWorker.DeletionRequestChan = make(chan deletionRequest, DeletionQueueLength) delWorker.DeletionQueues = make(map[string]*deletionQueue, 0) + delWorker.stopChan = make(chan struct{}) + delWorker.versionCache = s.getVersionCache() for _, symID := range symIDs { delWorker.DeletionQueues[symID] = &deletionQueue{ DeviceList: make([]*csiDevice, 0), SymID: symID, ClusterPrefix: clusterPrefix, + versionCache: s.getVersionCache(), } } log.Infof("Configuring deletion worker with Cluster Prefix: %s, Sym IDs: %v", diff --git a/service/deletion_worker_test.go b/service/deletion_worker_test.go index 50a155ea..29764de7 100644 --- a/service/deletion_worker_test.go +++ b/service/deletion_worker_test.go @@ -17,6 +17,7 @@ package service import ( "errors" "fmt" + "net/http" "sync" "testing" "time" @@ -597,6 +598,7 @@ func TestDeleteVolumes(t *testing.T) { }, pmaxClient: func() pmax.Pmax { pmaxClient := mocks.NewMockPmaxClient(gomock.NewController(t)) + pmaxClient.EXPECT().GetHTTPClient().AnyTimes().Return(&http.Client{}) pmaxClient.EXPECT().GetVersionDetails(gomock.Any()).AnyTimes().Return(&types.VersionDetails{APIVersion: "103"}, nil) pmaxClient.EXPECT().GetVolumesByIdentifierMatch(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(&types.Volumev1{ Volumes: []types.VolumeEnhanced{ @@ -638,6 +640,7 @@ func TestDeleteVolumes(t *testing.T) { }, pmaxClient: func() pmax.Pmax { pmaxClient := mocks.NewMockPmaxClient(gomock.NewController(t)) + pmaxClient.EXPECT().GetHTTPClient().AnyTimes().Return(&http.Client{}) pmaxClient.EXPECT().GetVersionDetails(gomock.Any()).AnyTimes().Return(&types.VersionDetails{APIVersion: "100"}, nil) pmaxClient.EXPECT().GetVolumeByID(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(nil, errors.New("error")) return pmaxClient @@ -668,6 +671,7 @@ func TestDeleteVolumes(t *testing.T) { }, pmaxClient: func() pmax.Pmax { pmaxClient := mocks.NewMockPmaxClient(gomock.NewController(t)) + pmaxClient.EXPECT().GetHTTPClient().AnyTimes().Return(&http.Client{}) pmaxClient.EXPECT().GetVersionDetails(gomock.Any()).AnyTimes().Return(&types.VersionDetails{APIVersion: "103"}, nil) pmaxClient.EXPECT().GetVolumesByIdentifierMatch(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(&types.Volumev1{ Volumes: []types.VolumeEnhanced{ @@ -711,6 +715,7 @@ func TestDeleteVolumes(t *testing.T) { }, pmaxClient: func() pmax.Pmax { pmaxClient := mocks.NewMockPmaxClient(gomock.NewController(t)) + pmaxClient.EXPECT().GetHTTPClient().AnyTimes().Return(&http.Client{}) pmaxClient.EXPECT().GetVersionDetails(gomock.Any()).AnyTimes().Return(&types.VersionDetails{APIVersion: "103"}, nil) pmaxClient.EXPECT().GetVolumesByIdentifierMatch(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(&types.Volumev1{ Volumes: []types.VolumeEnhanced{ @@ -1124,6 +1129,96 @@ func TestRemoveVolumesFromStorageGroup(t *testing.T) { } } +// TestRemoveVolumesFromSG_CascadePrevention regression test for ECS01A-920: +// verifies that when batch removal fails with "does not contain", the code +// excludes the stale volume and retries the batch so healthy volumes still advance. +func TestRemoveVolumesFromSG_CascadePrevention(t *testing.T) { + ctrl := gomock.NewController(t) + pmaxClient := mocks.NewMockPmaxClient(ctrl) + + // Setup: two devices in disAssociateSG state, both in SG "sg-opt" + queue := &deletionQueue{ + SymID: "sym1", + DeviceList: []*csiDevice{ + { + SymID: "sym1", + SymDeviceID: symDeviceID{DeviceID: "0012C", IntVal: 0x12C}, + VolumeIdentifier: "_DEL_csi-vol-0012C", + SymVolumeCache: symVolumeCache{}, + Status: deletionRequestStatus{State: deletionStateDisAssociateSG}, + }, + { + SymID: "sym1", + SymDeviceID: symDeviceID{DeviceID: "0012B", IntVal: 0x12B}, + VolumeIdentifier: "_DEL_csi-vol-0012B", + SymVolumeCache: symVolumeCache{}, + Status: deletionRequestStatus{State: deletionStateDisAssociateSG}, + }, + }, + } + + // Both devices initially in sg-opt + pmaxClient.EXPECT().GetVolumeByID(gomock.Any(), "sym1", "0012C"). + Return(&types.Volume{ + VolumeID: "0012C", + VolumeIdentifier: "_DEL_csi-vol-0012C", + StorageGroupIDList: []string{"sg-opt"}, + }, nil).Times(1) + pmaxClient.EXPECT().GetVolumeByID(gomock.Any(), "sym1", "0012B"). + Return(&types.Volume{ + VolumeID: "0012B", + VolumeIdentifier: "_DEL_csi-vol-0012B", + StorageGroupIDList: []string{"sg-opt"}, + }, nil).Times(1) + + // SG has no masking views + pmaxClient.EXPECT().GetStorageGroup(gomock.Any(), "sym1", "sg-opt"). + AnyTimes().Return(&types.StorageGroup{NumOfMaskingViews: 0}, nil) + + // Batch removal fails with "does not contain" (simulating the cascade bug) + pmaxClient.EXPECT().RemoveVolumesFromStorageGroup( + gomock.Any(), "sym1", "sg-opt", true, "0012C", "0012B"). + Return(nil, errors.New("The Storage Group sg-opt does not contain volume 0012C")) + + // Retry: batch with stale volume 0012C excluded, only 0012B remains + pmaxClient.EXPECT().RemoveVolumesFromStorageGroup( + gomock.Any(), "sym1", "sg-opt", true, "0012B"). + Return(&types.StorageGroup{}, nil) + + // Post-removal cache refresh: both now show empty SG list + pmaxClient.EXPECT().GetVolumeByID(gomock.Any(), "sym1", "0012C"). + Return(&types.Volume{ + VolumeID: "0012C", + VolumeIdentifier: "_DEL_csi-vol-0012C", + StorageGroupIDList: []string{}, + }, nil).Times(1) + pmaxClient.EXPECT().GetVolumeByID(gomock.Any(), "sym1", "0012B"). + Return(&types.Volume{ + VolumeID: "0012B", + VolumeIdentifier: "_DEL_csi-vol-0012B", + StorageGroupIDList: []string{}, + }, nil).Times(1) + + // Override delays for test speed + oldDelay := APIPropagationDelay + oldSyncTime := waitTillSyncInProgTime + APIPropagationDelay = 1 * time.Millisecond + waitTillSyncInProgTime = 1 * time.Millisecond + defer func() { + APIPropagationDelay = oldDelay + waitTillSyncInProgTime = oldSyncTime + }() + + result := queue.removeVolumesFromStorageGroup(pmaxClient) + assert.True(t, result) + + // Both devices should have advanced past disAssociateSG + assert.Equal(t, deletionStateDeleteVol, queue.DeviceList[0].Status.State, + "Device 0012C should advance to deleteVolume (already removed from SG)") + assert.Equal(t, deletionStateDeleteVol, queue.DeviceList[1].Status.State, + "Device 0012B should advance to deleteVolume after retry") +} + func TestDeletionRequestHandler(t *testing.T) { tests := []struct { name string diff --git a/service/envvars.go b/service/envvars.go index 45e46442..81e0a322 100644 --- a/service/envvars.go +++ b/service/envvars.go @@ -180,4 +180,21 @@ const ( // EnvSGVolumeLimit is an env variable which indicates the configured storage group volume limit EnvSGVolumeLimit = "X_CSI_STORAGE_GROUP_VOLUME_LIMIT" + + // EnvFsCheckEnabled enables file system check before mount + EnvFsCheckEnabled = "X_CSI_FS_CHECK_ENABLED" + + // EnvFsCheckMode controls the file system check operation mode + EnvFsCheckMode = "X_CSI_FS_CHECK_MODE" + // EnvSpaceReclamationEnabled enables/disables space reclamation + EnvSpaceReclamationEnabled = "X_CSI_SPACE_RECLAMATION_ENABLED" + + // EnvSpaceReclamationSchedule is the cron schedule for space reclamation + EnvSpaceReclamationSchedule = "X_CSI_SPACE_RECLAMATION_SCHEDULE" + + // EnvSpaceReclamationMaxConcurrent is the max concurrent reclamation operations + EnvSpaceReclamationMaxConcurrent = "X_CSI_SPACE_RECLAMATION_MAX_CONCURRENT" + + // EnvSpaceReclamationTimeout is the timeout for each reclamation operation + EnvSpaceReclamationTimeout = "X_CSI_SPACE_RECLAMATION_TIMEOUT" ) diff --git a/service/features/clone.feature b/service/features/clone.feature new file mode 100644 index 00000000..a1197420 --- /dev/null +++ b/service/features/clone.feature @@ -0,0 +1,79 @@ +Feature: PowerMax CSI Interface + As a consumer of the CSI interface + I want to test snapshot interfaces + So that they are known to work +@v2.17.0 + Scenario: 10.4 Create a volume clone with same size + Given a PowerMax service + And a 104 array + And I call CreateVolume "volume1" + And a valid CreateVolumeResponse is returned + And a valid volume with size of 54614 CYL + When I call Create Volume from Volume + Then a valid CreateVolumeResponse is returned + +@v2.17.0 + Scenario: 10.4 Idempotent volume clone + Given a PowerMax service + And a 104 array + And I call CreateVolume "volume1" + And a valid CreateVolumeResponse is returned + And a valid volume with size of 54614 CYL + When I call Create Volume from Volume + Then a valid CreateVolumeResponse is returned + When I call Create Volume from Volume + Then a valid CreateVolumeResponse is returned + +@v2.17.0 + Scenario: 10.4 Create a volume clone without specifying a source + Given a PowerMax service + And a 104 array + And I induce error "NoVolumeSource" + And I call Create Volume from Volume + Then the error contains "VolumeContentSource is missing volume and snapshot source" + +@v2.17.0 + Scenario: 10.4 Create a volume clone with non-existent source + Given a PowerMax service + And a 104 array + And I induce error "NonExistentVolume" + And I call Create Volume from Volume + Then the error contains "Volume content source couldn't be found in the array" + +@v2.17.0 + Scenario: 10.4 Create a volume clone from invalid volume + Given a PowerMax service + And a 104 array + And I induce error "InvalidVolumeID" + And I call Create Volume from Volume + Then the error contains "not in supported format" + +@v2.17.0 + Scenario: 10.4 Create a volume clone with smaller capacity + Given a PowerMax service + And a 104 array + And I call CreateVolume "volume1" + And a valid CreateVolumeResponse is returned + And I induce error "WrongCapacity" + And I call Create Volume from Volume + Then the error contains "Requested capacity is smaller than the source" + +@v2.17.0 + Scenario: 10.4 Create a volume clone with larger capacity succeeds + Given a PowerMax service + And a 104 array + And I call CreateVolume "volume1" + And a valid CreateVolumeResponse is returned + And a larger capacity than the source + And I call Create Volume from Volume + Then a valid CreateVolumeResponse is returned + +@v2.17.0 + Scenario: 10.4 Create a volume clone with CreateVolume error + Given a PowerMax service + And a 104 array + And I call CreateVolume "volume1" + And a valid CreateVolumeResponse is returned + And I induce error "CreateVolumeError" + And I call Create Volume from Volume + Then the error contains "Failed to create volume" \ No newline at end of file diff --git a/service/features/controller_publish_unpublish.feature b/service/features/controller_publish_unpublish.feature index accd7845..fcf4909b 100644 --- a/service/features/controller_publish_unpublish.feature +++ b/service/features/controller_publish_unpublish.feature @@ -23,9 +23,9 @@ Feature: PowerMax CSI interface @v1.0.0 Scenario Outline: Publish volume with single writer, enhanced API Given a PowerMax service - And I call CreateVolumeEnhanced "volume1" + And I call CreateVolume "volume1" When I request a PortGroup - And a valid CreateVolumeEnhancedResponse is returned + And a valid CreateVolumeResponse is returned And I have a Node "node1" with MaskingView And I call PublishVolume with to "node1" Then a valid PublishVolumeResponse is returned @@ -65,9 +65,10 @@ Feature: PowerMax CSI interface And I call PublishVolume with to "node1" And the error contains Examples: - | "single-writer" | "GetPortError" | "Failed to fetch SCSI_FC port for array" | - | "single-node-single-writer" | "GetPortError" | "Failed to fetch SCSI_FC port for array" | - | "single-node-multi-writer" | "GetPortError" | "Failed to fetch SCSI_FC port for array" | + | access | induced | errormsg | + | "single-writer" | "GetPortError" | "Failed to fetch SCSI_FC port for array" | + | "single-node-single-writer" | "GetPortError" | "Failed to fetch SCSI_FC port for array" | + | "single-node-multi-writer" | "GetPortError" | "Failed to fetch SCSI_FC port for array" | @controllerPublish @v1.1.0 @@ -97,9 +98,10 @@ Feature: PowerMax CSI interface And I call PublishVolume with to "node1" Then the error contains Examples: - | "single-writer" | "GetPortError" | "Failed to fetch SCSI_FC port for array" | - | "single-node-single-writer" | "GetPortError" | "Failed to fetch SCSI_FC port for array" | - | "single-node-multi-writer" | "GetPortError" | "Failed to fetch SCSI_FC port for array" | + | access | induced | errormsg | + | "single-writer" | "GetPortError" | "Failed to fetch SCSI_FC port for array" | + | "single-node-single-writer" | "GetPortError" | "Failed to fetch SCSI_FC port for array" | + | "single-node-multi-writer" | "GetPortError" | "Failed to fetch SCSI_FC port for array" | @controllerPublish @v1.1.0 @@ -129,9 +131,10 @@ Feature: PowerMax CSI interface And I call PublishVolume with to "node1" Then the error contains Examples: - | "single-writer" | "GetPortError" | "Failed to fetch SCSI_FC port for array" | - | "single-node-single-writer" | "GetPortError" | "Failed to fetch SCSI_FC port for array" | - | "single-node-multi-writer" | "GetPortError" | "Failed to fetch SCSI_FC port for array" | + | access | induced | errormsg | + | "single-writer" | "GetPortError" | "Failed to fetch SCSI_FC port for array" | + | "single-node-single-writer" | "GetPortError" | "Failed to fetch SCSI_FC port for array" | + | "single-node-multi-writer" | "GetPortError" | "Failed to fetch SCSI_FC port for array" | @controllerPublish @v1.0.0 @@ -577,7 +580,7 @@ Feature: PowerMax CSI interface @v1.0.0 Scenario: Unpublish volume with invalid volume id Given a PowerMax service - And an invalid volume + And I induce error "InvalidVolumeID" And I call UnpublishVolume from "node1" Then the error contains "Invalid volume id" diff --git a/service/features/csi_extension.feature b/service/features/csi_extension.feature index d62561cd..4a17c34b 100644 --- a/service/features/csi_extension.feature +++ b/service/features/csi_extension.feature @@ -68,7 +68,7 @@ Feature: PowerMax CSI interface | "/array-status/symmetrixID3" | "notConnected" | "none" | "none" | | "/array-status/symmetrixID4" | "invalid" | "unexpected end of JSON input" | "none" | | "/array-status/symmetrixID5" | "new" | "unexpected response from the server" | "QueryArrayStatusUnexpectedResponse" | - | "" | "none" | "connection refused" | "none" | + | "" | "none" | "connection refused@@context deadline exceeded" | "none" | @resiliency @v2.11.0 diff --git a/service/features/node_publish_unpublish.feature b/service/features/node_publish_unpublish.feature index abb87e8a..83da33f8 100644 --- a/service/features/node_publish_unpublish.feature +++ b/service/features/node_publish_unpublish.feature @@ -567,14 +567,14 @@ Feature: PowerMax CSI interface And lastUnmounted should be Examples: - | mnta | mntb | induced | lastUnmounted | errormsg | - | "none" | "none" | "none" | "true" | "none" | - | "test/mnt1" | "none" | "none" | "true" | "none" | - | "test/mnt1" | "none" | "GOFSMockGetMountsError" | "false" | "getMounts induced error" | - | "test/mnt1" | "none" | "GOFSMockUnmountError" | "false" | "unmount induced error" | - | "test/mnt1" | "test/mnt2" | "GOFSMockGetMountsError" | "false" | "getMounts induced error" | - | "test/mnt1" | "test/mnt2" | "none" | "false" | "none" | - + | mnta | mntb | induced | lastUnmounted | errormsg | + | "none" | "none" | "none" | "true" | "none" | + | "test/mnt1" | "none" | "none" | "true" | "none" | + | "test/mnt1" | "test/pod/mount" | "none" | "false" | "none" | + | "test/mnt1" | "none" | "GOFSMockGetMountsError" | "false" | "getMounts induced error" | + | "test/mnt1" | "none" | "GOFSMockUnmountError" | "false" | "unmount induced error" | + | "test/mnt1" | "test/mnt2" | "GOFSMockGetMountsError" | "false" | "getMounts induced error" | + | "test/mnt1" | "test/mnt2" | "none" | "false" | "none" | @v1.1.0 Scenario Outline: Call verifyAndUpdateInitiatorsInADiffHost in various scenarios without modifying host name diff --git a/service/features/replication.feature b/service/features/replication.feature index feebad9a..f30cd741 100644 --- a/service/features/replication.feature +++ b/service/features/replication.feature @@ -280,14 +280,13 @@ Feature: PowerMax CSI Interface Then the error contains Examples: - | induced | errorMsg | - | none | | - | noVolumeSource | missing source volume ID | - | nonExistentVolume | source volume does not exist | - | invalidVolumeID | invalid volume ID provided | - | wrongCapacity | capacity mismatch error | - | wrongStoragePool | invalid storage pool specified | - + | induced | errormsg | + | "none" | "none" | + | "NoVolumeSource" | "missing volume and snapshot source" | + | "InvalidVolumeID" | "volume identifier not in supported format" | + | "NonExistentVolume" | "content source volume couldn't be found" | + | "WrongCapacity" | "capacity is smaller than the source" | + | "WrongStoragePool" | "bad storage pool not found" | @srdf @v2.9.0 @@ -348,13 +347,13 @@ Feature: PowerMax CSI Interface When I call RDF enabled CreateVolume "volume2" in namespace "test", mode "METRO" and RDFGNo 14 from volume Then the error contains Examples: - | induced | errorMsg | - | none | | - | noVolumeSource | missing source volume ID | - | nonExistentVolume | source volume does not exist | - | invalidVolumeID | invalid volume ID provided | - | wrongCapacity | capacity mismatch error | - | wrongStoragePool | invalid storage pool specified | + | induced | errormsg | + | "none" | "none" | + | "NoVolumeSource" | "missing volume and snapshot source" | + | "InvalidVolumeID" | "volume identifier not in supported format" | + | "NonExistentVolume" | "content source volume couldn't be found" | + | "WrongCapacity" | "capacity is smaller than the source" | + | "WrongStoragePool" | "bad storage pool not found" | @srdf @v2.9.0 diff --git a/service/features/service.feature b/service/features/service.feature index f28f29b5..16acd93e 100644 --- a/service/features/service.feature +++ b/service/features/service.feature @@ -419,10 +419,10 @@ Feature: PowerMax CSI interface | "testhost" |"NoArray" | "none" | "No array specified" | 0 | | "testhost" |"NoNodeName" | "none" | "No nodeName specified" | 0 | | "testhost" |"NoIQNs" | "none" | "No port WWNs specified" | 0 | - | "testhost" |"GetHostError" | "CreateHostError" | "Unable to create Host" | 0 | + | "testhost" |"GetHostError" | "CreateHostError" | "failed to create host" | 0 | | "testhost" |"none" | "none" | "none" | 1 | | "CSI-Test-Node-2" |"GetInitiatorError" | "none" | "Error retrieving Initiator(s)" | 0 | - | "CSI-Test-Node-2" |"UpdateHostError" | "CreateHostError" | "Unable to" | 0 | + | "CSI-Test-Node-2" |"UpdateHostError" | "CreateHostError" | "failed to create host" | 0 | | "CSI-Test-Node-2" |"UpdateHostError" | "ResetAfterFirstError"| "none" | 1 | | "CSI-Test-Node-2" |"GetHostError" | "none" | "none" | 1 | @@ -585,6 +585,29 @@ Feature: PowerMax CSI interface And I call CreateVolume "volume1" Then the error contains "Block Volume Capability is not supported" +@v2.17.0 + Scenario: 10.4 Create a basic volume + Given a PowerMax service + And a 104 array + And I call CreateVolume "volume1" + Then a valid CreateVolumeResponse is returned + +@v2.17.0 + Scenario: 10.4 Create a block volume and block not enabled + Given a PowerMax service + And a 104 array + And block volumes are not enabled + And I call CreateVolume "volume1" + Then the error contains "Block Volume Capability is not supported" + +@v2.17.0 + Scenario: 10.4 Create a volume with invalid SLO + Given a PowerMax service + And a 104 array + And I induce error "InvalidServiceLevel" + And I call CreateVolume "volume1" + Then the error contains "An invalid Service Level parameter was specified" + @v1.1.0 Scenario Outline: Test GetPortIdentifier function Given a PowerMax service diff --git a/service/features/snapshot.feature b/service/features/snapshot.feature index 70494266..efe40e2b 100644 --- a/service/features/snapshot.feature +++ b/service/features/snapshot.feature @@ -66,7 +66,7 @@ Feature: PowerMax CSI Interface Scenario: Create snapshot with no probe Given a PowerMax service - And an invalid volume + And I induce error "InvalidVolumeID" When I invalidate the Probe cache And I call CreateSnapshot With "snap1" Then the error contains "Controller Service has not been probed" @@ -79,13 +79,13 @@ Feature: PowerMax CSI Interface @v1.2.0 Scenario: Create snapshot with an invalid volume Given a PowerMax service - And an invalid volume + And I induce error "InvalidVolumeID" And I call CreateSnapshot With "snapshot1" Then the error contains "Could not parse CSI VolumeId" @v1.2.0 Scenario: Create snapshot on a non-existent volume Given a PowerMax service - And a non-existent volume + And I induce error "NonExistentVolume" And I call CreateSnapshot With "snapshot1" Then the error contains "Could not find source volume on the array" @v1.4.0 @@ -135,7 +135,7 @@ Feature: PowerMax CSI Interface @v1.2.0 Scenario: Existence of a snapshot on a non-existent volume Given a PowerMax service - And a non-existent volume + And I induce error "NonExistentVolume" When I call IsVolumeInSnapSession on "" Then the error contains "Could not find volume" @v1.2.0 @@ -349,19 +349,19 @@ Feature: PowerMax CSI Interface @v1.2.0 Scenario: Create a volume without specifying a source Given a PowerMax service - And no volume source + And I induce error "NoVolumeSource" And I call Create Volume from Volume Then the error contains "VolumeContentSource is missing volume and snapshot source" @v1.2.0 Scenario: Create a volume with non-existent volume as a source Given a PowerMax service - And a non-existent volume + And I induce error "NonExistentVolume" And I call Create Volume from Volume Then the error contains "Volume content source volume couldn't be found" @v1.2.0 Scenario: Create a volume from invalid volume Given a PowerMax service - And an invalid volume + And I induce error "InvalidVolumeID" And I call Create Volume from Volume Then the error contains "Source volume identifier not in supported format" @v1.2.0 @@ -683,4 +683,76 @@ Feature: PowerMax CSI Interface And I induce error "MaxSnapSessionError" When I call Create Volume from Snapshot Then the error contains "Failed to create volume from snapshot" - +@v2.17.0 + Scenario: 10.4 Create a volume from a snapshot (success) + Given a PowerMax service + And a 104 array + And I call CreateVolume "volume1" + And a valid CreateVolumeResponse is returned + And I call CreateSnapshot "snapshot1" on "volume1" + And a valid CreateSnapshotResponse is returned + When I call Create Volume from Snapshot + Then a valid CreateVolumeResponse is returned +@v2.17.0 + Scenario: 10.4 Create a volume from a snapshot is idempotent + Given a PowerMax service + And a 104 array + And I call CreateVolume "volume1" + And a valid CreateVolumeResponse is returned + And I call CreateSnapshot "snapshot1" on "volume1" + And a valid CreateSnapshotResponse is returned + When I call Create Volume from Snapshot + Then a valid CreateVolumeResponse is returned + When I call Create Volume from Snapshot + Then a valid CreateVolumeResponse is returned +@v2.17.0 + Scenario: 10.4 Create a volume from an invalid snapshot ID + Given a PowerMax service + And a 104 array + And an invalid snapshot + When I call Create Volume from Snapshot + Then the error contains "Snapshot identifier not in supported format" +@v2.17.0 + Scenario: 10.4 Create a volume from a snapshot but receive create error + Given a PowerMax service + And a 104 array + And I call CreateVolume "volume1" + And a valid CreateVolumeResponse is returned + And I call CreateSnapshot "snapshot1" on "volume1" + And a valid CreateSnapshotResponse is returned + And I induce error "CreateVolumeError" + When I call Create Volume from Snapshot + Then the error contains "induced error" +@v2.17.0 + Scenario: 10.4 Create a volume from a snapshot with unlicensed array + Given a PowerMax service + And a 104 array + And I call CreateVolume "volume1" + And a valid CreateVolumeResponse is returned + And I call CreateSnapshot "snapshot1" on "volume1" + And a valid CreateSnapshotResponse is returned + And I induce error "SnapshotNotLicensed" + When I call Create Volume from Snapshot + Then the error contains "doesn't have Snapshot license" +@v2.17.0 + Scenario: 10.4 Create a volume from a snapshot with larger capacity succeeds + Given a PowerMax service + And a 104 array + And I call CreateVolume "volume1" + And a valid CreateVolumeResponse is returned + And I call CreateSnapshot "snapshot1" on "volume1" + And a valid CreateSnapshotResponse is returned + And a larger capacity than the source + When I call Create Volume from Snapshot + Then a valid CreateVolumeResponse is returned +@v2.17.0 + Scenario: 10.4 Create a volume from a snapshot with smaller capacity returns error + Given a PowerMax service + And a 104 array + And I call CreateVolume "volume1" + And a valid CreateVolumeResponse is returned + And I call CreateSnapshot "snapshot1" on "volume1" + And a valid CreateSnapshotResponse is returned + And I induce error "WrongCapacity" + When I call Create Volume from Snapshot + Then the error contains "smaller than the source" diff --git a/service/features/snapshot.feature.disabled b/service/features/snapshot.feature.disabled index cd8bf590..6343c764 100644 --- a/service/features/snapshot.feature.disabled +++ b/service/features/snapshot.feature.disabled @@ -33,7 +33,7 @@ Feature: PowerMax CSI interface Scenario: Call snapshot create with invalid volume Given a PowerMax service - And an invalid volume + And I induce error "InvalidVolumeID" When I call Probe And I call CreateSnapshot With "snap1" Then the error contains "volume not found" @@ -47,7 +47,7 @@ Feature: PowerMax CSI interface Scenario: Call snapshot with no probe Given a PowerMax service - And an invalid volume + And I induce error "InvalidVolumeID" When I invalidate the Probe cache And I call CreateSnapshot With "snap1" Then the error contains "Controller Service has not been probed" @@ -89,7 +89,7 @@ Feature: PowerMax CSI interface Scenario: Delete a snapshot with invalid volume Given a PowerMax service - And an invalid volume + And I induce error "InvalidVolumeID" When I call Probe And I call DeleteSnapshot Then the error contains "volume not found" @@ -154,7 +154,7 @@ Feature: PowerMax CSI interface Scenario: Create a volume from a snapshot with wrong capacity Given a PowerMax service And a valid snapshot - And the wrong capacity + And I induce error "WrongCapacity" When I call Probe And I call Create Volume from Snapshot Then the error contains "incompatible size" @@ -162,7 +162,7 @@ Feature: PowerMax CSI interface Scenario: Create a volume from a snapshot with wrong storage pool Given a PowerMax service And a valid snapshot - And the wrong storage pool + And I induce error "WrongStoragePool" When I call Probe And I call Create Volume from Snapshot Then the error contains "different than the requested storage pool" diff --git a/service/fscheck.go b/service/fscheck.go new file mode 100644 index 00000000..996df0d2 --- /dev/null +++ b/service/fscheck.go @@ -0,0 +1,397 @@ +/* + Copyright © 2025 Dell Inc. or its subsidiaries. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package service + +import ( + "context" + "fmt" + "regexp" + "strings" + "sync" + + csi "github.com/container-storage-interface/spec/lib/go/csi" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/dell/csi-powermax/v2/k8sutils" + "github.com/dell/gofsutil" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" + typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/record" +) + +// PVC label keys for per-volume FS check overrides +const ( + PVCLabelFSCheckEnabled = "csi.dell.com/fs_check_enabled" + PVCLabelFSCheckMode = "csi.dell.com/fs_check_mode" +) + +// getFSCheckerFunc is a variable to allow mocking in tests. +var getFSCheckerFunc = gofsutil.GetFSChecker + +// getDevMountsFunc is a variable to allow mocking in tests. +var getDevMountsFunc = getDevMounts + +// newEventRecorderFunc is a variable to allow mocking in tests. +var newEventRecorderFunc = newEventRecorder + +// fsCheckConfig holds FS check settings and dependencies for a single volume publish +type fsCheckConfig struct { + enabled bool // global setting from X_CSI_FS_CHECK_ENABLED + mode string // global setting: "checkOnly" or "checkAndRepair" + k8sUtils k8sutils.UtilsInterface // for lazy PVC lookup; nil-safe +} + +// fsCheckPVCObserver bridges gofsutil FSCheckObserver events to K8s PVC events and logs +type fsCheckPVCObserver struct { + pvcName string + pvcNamespace string + devicePath string + fsType string + volumeID string + sawTimeout bool + events []string + eventRecorder record.EventRecorder // nil-safe; skips event posting if nil +} + +// OnEvent implements gofsutil.FSCheckObserver +func (o *fsCheckPVCObserver) OnEvent(message string) { + o.events = append(o.events, message) + + if message == gofsutil.FSCheckTimedOutEvent || message == gofsutil.FSRepairTimedOutEvent { + o.sawTimeout = true + } + + log.Infof("FS check event: %s on %s (%s)", message, o.devicePath, o.fsType) + + // Post K8s event on PVC + if o.eventRecorder == nil || o.pvcName == "" || o.pvcNamespace == "" { + return + } + + eventType, reason := mapFSCheckEventToK8s(message) + pvcRef := &corev1.ObjectReference{ + Kind: "PersistentVolumeClaim", + Name: o.pvcName, + Namespace: o.pvcNamespace, + } + eventMessage := fmt.Sprintf("Volume %s device %s (%s): %s", o.volumeID, o.devicePath, o.fsType, message) + o.eventRecorder.Event(pvcRef, eventType, reason, eventMessage) +} + +// Ensure fsCheckPVCObserver implements gofsutil.FSCheckObserver at compile time +var _ gofsutil.FSCheckObserver = (*fsCheckPVCObserver)(nil) + +var ( + cachedEventRecorder record.EventRecorder + eventRecorderOnce sync.Once +) + +// newEventRecorder creates a Kubernetes EventRecorder. Replaceable for testing. +func newEventRecorder() (record.EventRecorder, error) { + config, err := rest.InClusterConfig() + if err != nil { + return nil, fmt.Errorf("failed to get in-cluster config: %w", err) + } + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, fmt.Errorf("failed to create Kubernetes client: %w", err) + } + + eventBroadcaster := record.NewBroadcaster() + eventBroadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: clientset.CoreV1().Events("")}) + + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + return nil, fmt.Errorf("failed to add scheme: %w", err) + } + + return eventBroadcaster.NewRecorder(scheme, corev1.EventSource{Component: "csi-powermax-node"}), nil +} + +func initEventRecorder() record.EventRecorder { + eventRecorderOnce.Do(func() { + recorder, err := newEventRecorderFunc() + if err != nil { + log.Warnf("Failed to initialize FS check event recorder: %v - PVC events will not be posted", err) + return + } + cachedEventRecorder = recorder + }) + return cachedEventRecorder +} + +// isAccessModeReadOnlyOrMulti returns true if the access mode is read-only or multi-node +func isAccessModeReadOnlyOrMulti(accMode *csi.VolumeCapability_AccessMode) bool { + if accMode == nil { + return false + } + switch accMode.GetMode() { + case csi.VolumeCapability_AccessMode_SINGLE_NODE_READER_ONLY, + csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY, + csi.VolumeCapability_AccessMode_MULTI_NODE_SINGLE_WRITER, + csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER: + return true + } + return false +} + +var pvNameFromPathRegex = regexp.MustCompile(`/.*/pods/[^/]+/volumes/kubernetes\.io~csi/([^/]+)/mount`) + +// parsePVNameFromTargetPath extracts the PV name from the target path. +// Expected format: /pods//volumes/kubernetes.io~csi//mount +func parsePVNameFromTargetPath(targetPath string) string { + matches := pvNameFromPathRegex.FindStringSubmatch(targetPath) + if len(matches) > 1 { + return matches[1] + } + return "" +} + +// isSupportedFSType returns true if the filesystem type supports FS check +func isSupportedFSType(fsType string) bool { + switch fsType { + case "ext4", "ext3", "ext2", "xfs": + return true + } + return false +} + +// performFSCheck runs FS check/repair if conditions are met. +// PVC label lookup is deferred until after cheap preconditions pass (FR-3). +// Returns nil to proceed with mount, or a gRPC status error to abort. +func performFSCheck( + ctx context.Context, + sysDevice *Device, + fsCfg *fsCheckConfig, + accMode *csi.VolumeCapability_AccessMode, + volumeID string, + targetPath string, +) error { + // 1. Nil config means feature is completely unavailable + if fsCfg == nil { + log.Info("Skipping FS check: feature disabled") + return nil + } + + // Note: We do NOT early-return for !fsCfg.enabled here because a PVC label + // (csi.dell.com/fs_check_enabled=true) can override the global disabled setting (AC-6). + // The global enabled flag is passed to resolvePVCOverrides at step 7, which determines + // the final effective enabled state after consulting PVC labels. + + // 2. Skip for read-only or multi-node access modes (cheap) + if isAccessModeReadOnlyOrMulti(accMode) { + log.Info("Skipping FS check: read-only or multi-node access mode") + return nil + } + + // 3. Check existing filesystem type (cheap) + existingFs, err := gofsutil.GetDiskFormat(ctx, sysDevice.FullPath) + if err != nil { + log.Warnf("Could not determine disk format for %s: %s. Skipping FS check.", sysDevice.FullPath, err) + return nil + } + + // 4. Skip if newly formatted + if existingFs == "" { + log.Info("Skipping FS check: newly formatted volume") + return nil + } + + // 5. Skip if unsupported FS type (ext2, ext3, ext4, xfs supported) + if !isSupportedFSType(existingFs) { + log.Warnf("Skipping FS check: unsupported filesystem type %q", existingFs) + return nil + } + + // 6. Skip if device already has mounts on this node (FR-2) + devMnts, err := getDevMountsFunc(sysDevice) + if err != nil { + log.Warnf("Could not check mount status for %s: %s. Skipping FS check.", sysDevice.FullPath, err) + return nil + } + if len(devMnts) > 0 { + log.Infof("Skipping FS check: volume already mounted on this node at %s", devMnts[0].Path) + return nil + } + + // 7. Lazy PVC label lookup (expensive - only runs after all cheap checks pass) + enabled, mode, pvcName, pvcNamespace := resolvePVCOverrides(ctx, fsCfg, volumeID, targetPath) + if !enabled { + log.Info("Skipping FS check: feature disabled (global or PVC override)") + return nil + } + + // 8. Initialize event recorder (singleton) + eventRecorder := initEventRecorder() + + // 9. Create observer + observer := &fsCheckPVCObserver{ + pvcName: pvcName, + pvcNamespace: pvcNamespace, + devicePath: sysDevice.FullPath, + fsType: existingFs, + volumeID: volumeID, + eventRecorder: eventRecorder, + } + + // 10. Get FS checker + checker, err := getFSCheckerFunc(sysDevice.FullPath, existingFs, observer) + if err != nil { + return fmt.Errorf("failed to create FS checker for %s (%s): %w", sysDevice.FullPath, existingFs, err) + } + + // 11. Run check + doRepair := mode == "checkAndRepair" + log.Infof("Starting file system check on %s (%s), repair=%v", sysDevice.FullPath, existingFs, doRepair) + + if err := checker.Check(ctx, doRepair); err != nil { + if observer.sawTimeout { + msg := fmt.Sprintf("File system check timed out on device %s (%s). The operation will be retried.", sysDevice.FullPath, existingFs) + log.Error(msg) + return status.Error(codes.Aborted, msg) + } + + msg := fmt.Sprintf("File system check failed on device %s (volume ID: %s, fs: %s): %s. Manual intervention required. Do not attempt to mount this volume until the file system has been repaired.", + sysDevice.FullPath, volumeID, existingFs, err.Error()) + log.Error(msg) + + // Post extra PVC Warning event with actionable message (FR-4) + if eventRecorder != nil && pvcName != "" && pvcNamespace != "" { + pvcRef := &corev1.ObjectReference{ + Kind: "PersistentVolumeClaim", + Name: pvcName, + Namespace: pvcNamespace, + } + eventRecorder.Event(pvcRef, string(corev1.EventTypeWarning), "FSCheckFailed", + fmt.Sprintf("File system on device %s (fs: %s) cannot be mounted safely. Manual intervention required.", sysDevice.FullPath, existingFs)) + } + + return status.Error(codes.Internal, msg) + } + + log.Infof("FS check completed successfully on %s (%s)", sysDevice.FullPath, existingFs) + return nil +} + +// resolvePVCOverrides performs the lazy PVC label lookup and returns the effective settings. +// On any failure, it gracefully falls back to global settings (AC-17). +func resolvePVCOverrides( + ctx context.Context, + fsCfg *fsCheckConfig, + volumeID string, + targetPath string, +) (enabled bool, mode string, pvcName string, pvcNamespace string) { + enabled = fsCfg.enabled + mode = fsCfg.mode + + // Parse PV name from target path + pvName := parsePVNameFromTargetPath(targetPath) + if pvName == "" { + log.Debugf("Could not extract PV name from target path %q; using global FS check config", targetPath) + return + } + + // Check k8sUtils availability + if fsCfg.k8sUtils == nil { + log.Debug("k8sUtils not available; using global FS check config") + return + } + + // Look up the PVC via the PV + pvc, err := fsCfg.k8sUtils.GetPVCForVolume(ctx, pvName, volumeID) + if err != nil { + log.Warnf("Could not look up PVC for PV %s (volume %s): %s. Using global FS check config.", pvName, volumeID, err) + return + } + if pvc == nil { + log.Debugf("PVC not found for PV %s; using global FS check config", pvName) + return + } + + // Populate PVC info for event posting + pvcName = pvc.Name + pvcNamespace = pvc.Namespace + + // Apply PVC label overrides + if pvc.Labels != nil { + if val, ok := pvc.Labels[PVCLabelFSCheckEnabled]; ok { + switch strings.ToLower(val) { + case "true": + enabled = true + case "false": + enabled = false + default: + log.Warnf("Invalid value %q for PVC label %s on %s/%s; ignoring", + val, PVCLabelFSCheckEnabled, pvc.Namespace, pvc.Name) + } + } + + if enabled { + if val, ok := pvc.Labels[PVCLabelFSCheckMode]; ok { + switch strings.ToLower(val) { + case "checkonly": + mode = "checkOnly" + case "checkandrepair": + mode = "checkAndRepair" + default: + log.Warnf("Invalid value %q for PVC label %s on %s/%s; ignoring", + val, PVCLabelFSCheckMode, pvc.Namespace, pvc.Name) + } + } + } + } + + log.Infof("Resolved FS check config for volume %s: enabled=%v, mode=%s, pvc=%s/%s", + volumeID, enabled, mode, pvcNamespace, pvcName) + return +} + +// mapFSCheckEventToK8s maps a gofsutil observer event to K8s event type and reason +func mapFSCheckEventToK8s(message string) (eventType, reason string) { + switch message { + case gofsutil.StartedFSCheckEvent: + return string(corev1.EventTypeNormal), "FSCheckStarted" + case gofsutil.FoundNoErrorsEvent: + return string(corev1.EventTypeNormal), "FSCheckSucceeded" + case gofsutil.FinishedFSRepairEvent: + return string(corev1.EventTypeNormal), "FSCheckRepaired" + case gofsutil.FoundErrorsEvent: + return string(corev1.EventTypeWarning), "FSCheckFailed" + case gofsutil.StartFSRepairEvent: + return string(corev1.EventTypeNormal), "FSRepairStarted" + case gofsutil.FoundDirtyLogEvent: + return string(corev1.EventTypeNormal), "FSCheckDirtyLog" + case gofsutil.StartLogReplayEvent: + return string(corev1.EventTypeNormal), "FSLogReplayStarted" + case gofsutil.LogReplayDoneEvent: + return string(corev1.EventTypeNormal), "FSLogReplayDone" + case gofsutil.FSCheckTimedOutEvent: + return string(corev1.EventTypeWarning), "FSCheckTimedOut" + case gofsutil.FSRepairFailedEvent: + return string(corev1.EventTypeWarning), "FSRepairFailed" + case gofsutil.FSRepairTimedOutEvent: + return string(corev1.EventTypeWarning), "FSRepairTimedOut" + case gofsutil.LogReplayFailedEvent: + return string(corev1.EventTypeWarning), "FSLogReplayFailed" + case gofsutil.FSCheckFailedEvent: + return string(corev1.EventTypeWarning), "FSCheckFailed" + default: + return string(corev1.EventTypeNormal), "FSCheckEvent" + } +} diff --git a/service/fscheck_test.go b/service/fscheck_test.go new file mode 100644 index 00000000..410a808f --- /dev/null +++ b/service/fscheck_test.go @@ -0,0 +1,924 @@ +/* + Copyright © 2025 Dell Inc. or its subsidiaries. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package service + +import ( + "context" + "errors" + "fmt" + "sync" + "testing" + + csi "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/dell/csi-powermax/v2/k8smock" + "github.com/dell/gofsutil" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/record" +) + +// ============================================================================ +// Helpers +// ============================================================================ + +func setupMockFS() { + gofsutil.UseMockFS() + gofsutil.GOFSMock.InduceGetDiskFormatError = false + gofsutil.GOFSMock.InduceGetDiskFormatType = "" +} + +func mockPVC(name, namespace string, labels map[string]string) *corev1.PersistentVolumeClaim { + return &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: labels, + }, + } +} + +const testTargetPath = "/var/lib/kubelet/pods/uid-123/volumes/kubernetes.io~csi/test-pv/mount" + +// Helper that returns a config with a mock k8sUtils attached +func newTestConfig(t *testing.T, enabled bool, mode string) (*fsCheckConfig, *k8smock.MockUtilsInterface, *gomock.Controller) { + ctrl := gomock.NewController(t) + mockUtils := k8smock.NewMockUtilsInterface(ctrl) + cfg := &fsCheckConfig{ + enabled: enabled, + mode: mode, + k8sUtils: mockUtils, + } + return cfg, mockUtils, ctrl +} + +// noMountsFunc is a mock getDevMountsFunc that returns no mounts +func noMountsFunc(_ *Device) ([]gofsutil.Info, error) { + return []gofsutil.Info{}, nil +} + +// mockEventRecorderFunc replaces newEventRecorderFunc for tests +func mockEventRecorderFunc() (record.EventRecorder, error) { + return record.NewFakeRecorder(100), nil +} + +func setupTestMocks() func() { + origGetDevMounts := getDevMountsFunc + origNewEventRecorder := newEventRecorderFunc + origCachedEventRecorder := cachedEventRecorder + + getDevMountsFunc = noMountsFunc + newEventRecorderFunc = mockEventRecorderFunc + // Reset event recorder singleton for tests + eventRecorderOnce = *new(sync.Once) + cachedEventRecorder = nil + + return func() { + getDevMountsFunc = origGetDevMounts + newEventRecorderFunc = origNewEventRecorder + eventRecorderOnce = *new(sync.Once) + cachedEventRecorder = origCachedEventRecorder + } +} + +// ============================================================================ +// Section 1: parsePVNameFromTargetPath tests +// ============================================================================ + +// Test ID: U-026 +func TestParsePVNameFromTargetPath(t *testing.T) { + tests := []struct { + name string + targetPath string + want string + }{ + { + name: "standard kubelet path", + targetPath: "/var/lib/kubelet/pods/abc-123/volumes/kubernetes.io~csi/pvc-test-vol/mount", + want: "pvc-test-vol", + }, + { + name: "custom kubelet root", + targetPath: "/custom/kubelet/pods/uid-456/volumes/kubernetes.io~csi/my-pv-name/mount", + want: "my-pv-name", + }, + { + name: "path with hyphens in PV name", + targetPath: "/var/lib/kubelet/pods/pod-uid/volumes/kubernetes.io~csi/pvc-abc-def-123/mount", + want: "pvc-abc-def-123", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := parsePVNameFromTargetPath(tt.targetPath) + assert.Equal(t, tt.want, got, "parsePVNameFromTargetPath(%q)", tt.targetPath) + }) + } +} + +// Test ID: U-027 +func TestParsePVNameFromTargetPathMalformed(t *testing.T) { + tests := []struct { + name string + targetPath string + }{ + { + name: "empty path", + targetPath: "", + }, + { + name: "no kubernetes.io~csi segment", + targetPath: "/var/lib/kubelet/pods/abc/volumes/other/pv-name/mount", + }, + { + name: "path ends at kubernetes.io~csi", + targetPath: "/var/lib/kubelet/pods/abc/volumes/kubernetes.io~csi", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := parsePVNameFromTargetPath(tt.targetPath) + assert.Empty(t, got, "parsePVNameFromTargetPath(%q) should return empty for malformed path", tt.targetPath) + }) + } +} + +// ============================================================================ +// Section 2: isAccessModeReadOnlyOrMulti tests +// ============================================================================ + +// Test ID: U-013 +func TestFSCheckSkipReadOnly(t *testing.T) { + mode := &csi.VolumeCapability_AccessMode{ + Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_READER_ONLY, + } + assert.True(t, isAccessModeReadOnlyOrMulti(mode), + "SINGLE_NODE_READER_ONLY should be detected as read-only") +} + +// Test ID: U-014 +func TestFSCheckSkipROX(t *testing.T) { + mode := &csi.VolumeCapability_AccessMode{ + Mode: csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY, + } + assert.True(t, isAccessModeReadOnlyOrMulti(mode), + "MULTI_NODE_READER_ONLY should be detected as read-only/multi") +} + +// Test ID: U-015 +func TestFSCheckSkipRWX(t *testing.T) { + mode := &csi.VolumeCapability_AccessMode{ + Mode: csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER, + } + assert.True(t, isAccessModeReadOnlyOrMulti(mode), + "MULTI_NODE_MULTI_WRITER should be detected as multi-node") +} + +// Test ID: U-013 (continued) - RWO should NOT be read-only +func TestFSCheckRWONotReadOnly(t *testing.T) { + mode := &csi.VolumeCapability_AccessMode{ + Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, + } + assert.False(t, isAccessModeReadOnlyOrMulti(mode), + "SINGLE_NODE_WRITER should not be detected as read-only/multi") +} + +// Test ID: U-013 (nil safety) +func TestFSCheckNilAccessMode(t *testing.T) { + assert.False(t, isAccessModeReadOnlyOrMulti(nil), + "nil access mode should return false") +} + +// ============================================================================ +// Section 3: isSupportedFSType tests (NEW) +// ============================================================================ + +func TestIsSupportedFSType(t *testing.T) { + assert.True(t, isSupportedFSType("ext4")) + assert.True(t, isSupportedFSType("ext3")) + assert.True(t, isSupportedFSType("ext2")) + assert.True(t, isSupportedFSType("xfs")) + assert.False(t, isSupportedFSType("ntfs")) + assert.False(t, isSupportedFSType("btrfs")) + assert.False(t, isSupportedFSType("nfs")) + assert.False(t, isSupportedFSType("")) +} + +// ============================================================================ +// Section 4: performFSCheck skip condition tests +// ============================================================================ + +// Test ID: U-006 - disabled +func TestFSCheckSkipDisabled(t *testing.T) { + setupMockFS() + ctx := context.Background() + sysDevice := &Device{FullPath: "/dev/sda", RealDev: "/dev/sda"} + cfg := &fsCheckConfig{enabled: false, mode: "checkOnly"} + accMode := &csi.VolumeCapability_AccessMode{Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER} + err := performFSCheck(ctx, sysDevice, cfg, accMode, "vol-002", testTargetPath) + assert.NoError(t, err) +} + +// Test ID: U-006b - nil config +func TestFSCheckSkipNilConfig(t *testing.T) { + setupMockFS() + ctx := context.Background() + sysDevice := &Device{FullPath: "/dev/sda", RealDev: "/dev/sda"} + accMode := &csi.VolumeCapability_AccessMode{Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER} + err := performFSCheck(ctx, sysDevice, nil, accMode, "vol-nil", testTargetPath) + assert.NoError(t, err) +} + +// Skip for read-only +func TestFSCheckSkipReadOnlyAccessMode(t *testing.T) { + setupMockFS() + gofsutil.GOFSMock.InduceGetDiskFormatType = "ext4" + ctx := context.Background() + sysDevice := &Device{FullPath: "/dev/sda", RealDev: "/dev/sda"} + cfg := &fsCheckConfig{enabled: true, mode: "checkOnly"} + accMode := &csi.VolumeCapability_AccessMode{Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_READER_ONLY} + err := performFSCheck(ctx, sysDevice, cfg, accMode, "vol-ro", testTargetPath) + assert.NoError(t, err) +} + +// Skip for multi-node +func TestFSCheckSkipMultiNodeAccessMode(t *testing.T) { + setupMockFS() + gofsutil.GOFSMock.InduceGetDiskFormatType = "ext4" + ctx := context.Background() + sysDevice := &Device{FullPath: "/dev/sda", RealDev: "/dev/sda"} + cfg := &fsCheckConfig{enabled: true, mode: "checkOnly"} + accMode := &csi.VolumeCapability_AccessMode{Mode: csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER} + err := performFSCheck(ctx, sysDevice, cfg, accMode, "vol-multi", testTargetPath) + assert.NoError(t, err) +} + +// Skip newly formatted +func TestFSCheckSkipNewlyFormatted(t *testing.T) { + setupMockFS() + gofsutil.GOFSMock.InduceGetDiskFormatType = "" + ctx := context.Background() + sysDevice := &Device{FullPath: "/dev/sda", RealDev: "/dev/sda"} + cfg := &fsCheckConfig{enabled: true, mode: "checkOnly"} + accMode := &csi.VolumeCapability_AccessMode{Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER} + err := performFSCheck(ctx, sysDevice, cfg, accMode, "vol-new", testTargetPath) + assert.NoError(t, err) +} + +// Skip unsupported FS +func TestFSCheckSkipUnsupportedFS(t *testing.T) { + setupMockFS() + gofsutil.GOFSMock.InduceGetDiskFormatType = "ntfs" + ctx := context.Background() + sysDevice := &Device{FullPath: "/dev/sda", RealDev: "/dev/sda"} + cfg := &fsCheckConfig{enabled: true, mode: "checkOnly"} + accMode := &csi.VolumeCapability_AccessMode{Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER} + err := performFSCheck(ctx, sysDevice, cfg, accMode, "vol-ntfs", testTargetPath) + assert.NoError(t, err) +} + +// GetDiskFormat error +func TestFSCheckGetDiskFormatError(t *testing.T) { + setupMockFS() + gofsutil.GOFSMock.InduceGetDiskFormatError = true + ctx := context.Background() + sysDevice := &Device{FullPath: "/dev/sda", RealDev: "/dev/sda"} + cfg := &fsCheckConfig{enabled: true, mode: "checkOnly"} + accMode := &csi.VolumeCapability_AccessMode{Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER} + err := performFSCheck(ctx, sysDevice, cfg, accMode, "vol-err", testTargetPath) + assert.NoError(t, err) +} + +// ============================================================================ +// Section 5: Already-mounted check (NEW - per FR-2) +// ============================================================================ + +// Test ID: AC-14 - skip when already mounted +func TestFSCheckSkipAlreadyMounted(t *testing.T) { + cleanup := setupTestMocks() + defer cleanup() + + // Override getDevMountsFunc to return a mount + getDevMountsFunc = func(_ *Device) ([]gofsutil.Info, error) { + return []gofsutil.Info{{Device: "/dev/sda", Path: "/mnt/target"}}, nil + } + + setupMockFS() + gofsutil.GOFSMock.InduceGetDiskFormatType = "ext4" + ctx := context.Background() + sysDevice := &Device{FullPath: "/dev/sda", RealDev: "/dev/sda"} + cfg := &fsCheckConfig{enabled: true, mode: "checkOnly"} + accMode := &csi.VolumeCapability_AccessMode{Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER} + err := performFSCheck(ctx, sysDevice, cfg, accMode, "vol-mounted", testTargetPath) + assert.NoError(t, err, "should skip FS check for already mounted volume") +} + +// Skip gracefully on mount check error +func TestFSCheckSkipOnMountCheckError(t *testing.T) { + cleanup := setupTestMocks() + defer cleanup() + + getDevMountsFunc = func(_ *Device) ([]gofsutil.Info, error) { + return nil, errors.New("mount check failed") + } + + setupMockFS() + gofsutil.GOFSMock.InduceGetDiskFormatType = "ext4" + ctx := context.Background() + sysDevice := &Device{FullPath: "/dev/sda", RealDev: "/dev/sda"} + cfg := &fsCheckConfig{enabled: true, mode: "checkOnly"} + accMode := &csi.VolumeCapability_AccessMode{Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER} + err := performFSCheck(ctx, sysDevice, cfg, accMode, "vol-mnterr", testTargetPath) + assert.NoError(t, err, "should skip gracefully on mount check error") +} + +// ============================================================================ +// Section 6: Lazy resolution / PVC label tests (resolvePVCOverrides) +// ============================================================================ + +// AC-17: PVC lookup fails -> falls back to global +func TestResolvePVCOverrides_LookupFails(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockUtils := k8smock.NewMockUtilsInterface(ctrl) + mockUtils.EXPECT().GetPVCForVolume(gomock.Any(), "test-pv", "vol-123"). + Return(nil, fmt.Errorf("PV not found")) + + cfg := &fsCheckConfig{enabled: true, mode: "checkAndRepair", k8sUtils: mockUtils} + enabled, mode, pvcName, pvcNS := resolvePVCOverrides(context.Background(), cfg, "vol-123", testTargetPath) + assert.True(t, enabled, "should fall back to global enabled") + assert.Equal(t, "checkAndRepair", mode, "should fall back to global mode") + assert.Empty(t, pvcName) + assert.Empty(t, pvcNS) +} + +// No PV name in path +func TestResolvePVCOverrides_NoPVNameInPath(t *testing.T) { + cfg := &fsCheckConfig{enabled: true, mode: "checkOnly"} + enabled, mode, pvcName, pvcNS := resolvePVCOverrides(context.Background(), cfg, "vol-123", "/some/random/path") + assert.True(t, enabled) + assert.Equal(t, "checkOnly", mode) + assert.Empty(t, pvcName) + assert.Empty(t, pvcNS) +} + +// Nil k8sUtils +func TestResolvePVCOverrides_NilK8sUtils(t *testing.T) { + cfg := &fsCheckConfig{enabled: true, mode: "checkAndRepair", k8sUtils: nil} + enabled, mode, _, _ := resolvePVCOverrides(context.Background(), cfg, "vol-123", testTargetPath) + assert.True(t, enabled) + assert.Equal(t, "checkAndRepair", mode) +} + +// Nil PVC returned +func TestResolvePVCOverrides_NilPVC(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockUtils := k8smock.NewMockUtilsInterface(ctrl) + mockUtils.EXPECT().GetPVCForVolume(gomock.Any(), "test-pv", "vol-123").Return(nil, nil) + + cfg := &fsCheckConfig{enabled: true, mode: "checkOnly", k8sUtils: mockUtils} + enabled, mode, _, _ := resolvePVCOverrides(context.Background(), cfg, "vol-123", testTargetPath) + assert.True(t, enabled) + assert.Equal(t, "checkOnly", mode) +} + +// AC-5: PVC label disables +func TestResolvePVCOverrides_PVCDisables(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockUtils := k8smock.NewMockUtilsInterface(ctrl) + mockUtils.EXPECT().GetPVCForVolume(gomock.Any(), "test-pv", "vol-123"). + Return(mockPVC("my-pvc", "default", map[string]string{PVCLabelFSCheckEnabled: "false"}), nil) + + cfg := &fsCheckConfig{enabled: true, mode: "checkAndRepair", k8sUtils: mockUtils} + enabled, mode, pvcName, pvcNS := resolvePVCOverrides(context.Background(), cfg, "vol-123", testTargetPath) + assert.False(t, enabled, "PVC label should disable") + assert.Equal(t, "checkAndRepair", mode, "mode should not be affected when disabled") + assert.Equal(t, "my-pvc", pvcName) + assert.Equal(t, "default", pvcNS) +} + +// AC-6: PVC label enables even when global disabled +func TestResolvePVCOverrides_PVCEnables(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockUtils := k8smock.NewMockUtilsInterface(ctrl) + mockUtils.EXPECT().GetPVCForVolume(gomock.Any(), "test-pv", "vol-123"). + Return(mockPVC("my-pvc", "default", map[string]string{PVCLabelFSCheckEnabled: "true"}), nil) + + cfg := &fsCheckConfig{enabled: false, mode: "checkOnly", k8sUtils: mockUtils} + enabled, _, _, _ := resolvePVCOverrides(context.Background(), cfg, "vol-123", testTargetPath) + assert.True(t, enabled, "PVC label should enable even when global disabled") +} + +// AC-7: PVC overrides mode +func TestResolvePVCOverrides_PVCOverridesMode(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockUtils := k8smock.NewMockUtilsInterface(ctrl) + mockUtils.EXPECT().GetPVCForVolume(gomock.Any(), "test-pv", "vol-123"). + Return(mockPVC("my-pvc", "default", map[string]string{PVCLabelFSCheckMode: "checkAndRepair"}), nil) + + cfg := &fsCheckConfig{enabled: true, mode: "checkOnly", k8sUtils: mockUtils} + _, mode, _, _ := resolvePVCOverrides(context.Background(), cfg, "vol-123", testTargetPath) + assert.Equal(t, "checkAndRepair", mode) +} + +// AC-18: Invalid labels fall back to global +func TestResolvePVCOverrides_InvalidLabels(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockUtils := k8smock.NewMockUtilsInterface(ctrl) + mockUtils.EXPECT().GetPVCForVolume(gomock.Any(), "test-pv", "vol-123"). + Return(mockPVC("my-pvc", "default", map[string]string{ + PVCLabelFSCheckEnabled: "invalid", + PVCLabelFSCheckMode: "badvalue", + }), nil) + + cfg := &fsCheckConfig{enabled: true, mode: "checkOnly", k8sUtils: mockUtils} + enabled, mode, _, _ := resolvePVCOverrides(context.Background(), cfg, "vol-123", testTargetPath) + assert.True(t, enabled, "invalid label should not change enabled") + assert.Equal(t, "checkOnly", mode, "invalid label should not change mode") +} + +// ============================================================================ +// Section 7: Lazy resolution - verify k8sUtils NOT called for cheap skips +// ============================================================================ + +// When fscheck is globally disabled but PVC lookup is reachable, k8sUtils IS called +// because a PVC label may override the global setting (AC-6). +func TestPerformFSCheck_GlobalDisabledStillChecksPVCLabels(t *testing.T) { + cleanup := setupTestMocks() + defer cleanup() + + setupMockFS() + gofsutil.GOFSMock.InduceGetDiskFormatType = "ext4" + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockUtils := k8smock.NewMockUtilsInterface(ctrl) + // PVC lookup WILL be called -- PVC has no override labels, so global disabled wins + mockUtils.EXPECT().GetPVCForVolume(gomock.Any(), "test-pv", "vol-lazy"). + Return(mockPVC("my-pvc", "default", nil), nil) + + cfg := &fsCheckConfig{enabled: false, mode: "checkOnly", k8sUtils: mockUtils} + sysDevice := &Device{FullPath: "/dev/sda", RealDev: "/dev/sda"} + accMode := &csi.VolumeCapability_AccessMode{Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER} + err := performFSCheck(context.Background(), sysDevice, cfg, accMode, "vol-lazy", testTargetPath) + assert.NoError(t, err, "should skip FS check when globally disabled and no PVC override") +} + +// When access mode is read-only, k8sUtils should NOT be called +func TestPerformFSCheck_ReadOnlySkipsPVCLookup(t *testing.T) { + cleanup := setupTestMocks() + defer cleanup() + + setupMockFS() + gofsutil.GOFSMock.InduceGetDiskFormatType = "ext4" + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockUtils := k8smock.NewMockUtilsInterface(ctrl) + // NO EXPECT -- must not be called + + cfg := &fsCheckConfig{enabled: true, mode: "checkOnly", k8sUtils: mockUtils} + sysDevice := &Device{FullPath: "/dev/sda", RealDev: "/dev/sda"} + accMode := &csi.VolumeCapability_AccessMode{Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_READER_ONLY} + err := performFSCheck(context.Background(), sysDevice, cfg, accMode, "vol-lazy2", testTargetPath) + assert.NoError(t, err) +} + +// When FS type is unsupported, k8sUtils should NOT be called +func TestPerformFSCheck_UnsupportedFSSkipsPVCLookup(t *testing.T) { + cleanup := setupTestMocks() + defer cleanup() + + setupMockFS() + gofsutil.GOFSMock.InduceGetDiskFormatType = "ntfs" + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockUtils := k8smock.NewMockUtilsInterface(ctrl) + // NO EXPECT -- must not be called + + cfg := &fsCheckConfig{enabled: true, mode: "checkOnly", k8sUtils: mockUtils} + sysDevice := &Device{FullPath: "/dev/sda", RealDev: "/dev/sda"} + accMode := &csi.VolumeCapability_AccessMode{Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER} + err := performFSCheck(context.Background(), sysDevice, cfg, accMode, "vol-lazy3", testTargetPath) + assert.NoError(t, err) +} + +// ============================================================================ +// Section 7b: AC-6 end-to-end - PVC label overrides global disabled through performFSCheck +// ============================================================================ + +// AC-6: Global disabled + PVC label enables = fsck MUST run (end-to-end through performFSCheck) +func TestPerformFSCheck_AC6_PVCOverridesGlobalDisabled(t *testing.T) { + cleanup := setupTestMocks() + defer cleanup() + + setupMockFS() + gofsutil.GOFSMock.InduceGetDiskFormatType = "ext4" + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockUtils := k8smock.NewMockUtilsInterface(ctrl) + // PVC has fs_check_enabled=true, overriding global disabled + mockUtils.EXPECT().GetPVCForVolume(gomock.Any(), "test-pv", "vol-ac6"). + Return(mockPVC("my-pvc", "default", map[string]string{ + PVCLabelFSCheckEnabled: "true", + }), nil) + + cfg := &fsCheckConfig{enabled: false, mode: "checkOnly", k8sUtils: mockUtils} + sysDevice := &Device{FullPath: "/dev/sda", RealDev: "/dev/sda"} + accMode := &csi.VolumeCapability_AccessMode{Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER} + + // performFSCheck should NOT return nil (skip) -- it should proceed to run the checker. + // In CI without real e2fsck, GetFSChecker will fail, but that proves we got PAST the + // skip checks and into the actual fsck execution path. + err := performFSCheck(context.Background(), sysDevice, cfg, accMode, "vol-ac6", testTargetPath) + // We expect an error because GetFSChecker will fail in test environment (no real device), + // but the key assertion is that we did NOT get nil (which would mean "skipped"). + if err == nil { + // If somehow it succeeded (unlikely in test env), that's also fine -- fsck ran. + return + } + // The error should be from GetFSChecker or Check, NOT a "skipping" nil return + assert.NotContains(t, err.Error(), "Skipping", + "AC-6 violation: PVC label enabled=true should override global disabled, but fsck was skipped") +} + +// ============================================================================ +// Section 8: ext4 check execution and ext2/ext3 support (NEW) +// ============================================================================ + +// Test ID: U-001 - ext4 check +func TestFSCheckExt4WithMockFS(t *testing.T) { + cleanup := setupTestMocks() + defer cleanup() + + setupMockFS() + gofsutil.GOFSMock.InduceGetDiskFormatType = "ext4" + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockUtils := k8smock.NewMockUtilsInterface(ctrl) + mockUtils.EXPECT().GetPVCForVolume(gomock.Any(), "test-pv", "vol-001"). + Return(mockPVC("my-pvc", "default", nil), nil) + + cfg := &fsCheckConfig{enabled: true, mode: "checkOnly", k8sUtils: mockUtils} + sysDevice := &Device{FullPath: "/dev/sda", RealDev: "/dev/sda"} + accMode := &csi.VolumeCapability_AccessMode{Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER} + err := performFSCheck(context.Background(), sysDevice, cfg, accMode, "vol-001", testTargetPath) + // In CI, GetFSChecker may fail (no e2fsck). Code path is exercised either way. + if err != nil { + st, ok := status.FromError(err) + if ok { + assert.NotEqual(t, codes.InvalidArgument, st.Code()) + } + } +} + +// ext3 check (NEW) +func TestFSCheckExt3NotSkipped(t *testing.T) { + cleanup := setupTestMocks() + defer cleanup() + + setupMockFS() + gofsutil.GOFSMock.InduceGetDiskFormatType = "ext3" + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockUtils := k8smock.NewMockUtilsInterface(ctrl) + mockUtils.EXPECT().GetPVCForVolume(gomock.Any(), "test-pv", "vol-ext3"). + Return(mockPVC("my-pvc", "default", nil), nil) + + cfg := &fsCheckConfig{enabled: true, mode: "checkOnly", k8sUtils: mockUtils} + sysDevice := &Device{FullPath: "/dev/sda", RealDev: "/dev/sda"} + accMode := &csi.VolumeCapability_AccessMode{Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER} + err := performFSCheck(context.Background(), sysDevice, cfg, accMode, "vol-ext3", testTargetPath) + // Reaching GetFSChecker means ext3 was NOT skipped -- that's the test + if err != nil { + assert.NotContains(t, err.Error(), "unsupported", "ext3 should not be skipped as unsupported") + } +} + +// ext2 check (NEW) +func TestFSCheckExt2NotSkipped(t *testing.T) { + cleanup := setupTestMocks() + defer cleanup() + + setupMockFS() + gofsutil.GOFSMock.InduceGetDiskFormatType = "ext2" + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockUtils := k8smock.NewMockUtilsInterface(ctrl) + mockUtils.EXPECT().GetPVCForVolume(gomock.Any(), "test-pv", "vol-ext2"). + Return(mockPVC("my-pvc", "default", nil), nil) + + cfg := &fsCheckConfig{enabled: true, mode: "checkOnly", k8sUtils: mockUtils} + sysDevice := &Device{FullPath: "/dev/sda", RealDev: "/dev/sda"} + accMode := &csi.VolumeCapability_AccessMode{Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER} + err := performFSCheck(context.Background(), sysDevice, cfg, accMode, "vol-ext2", testTargetPath) + if err != nil { + assert.NotContains(t, err.Error(), "unsupported", "ext2 should not be skipped as unsupported") + } +} + +// Test ID: U-012 - context timeout +func TestFSCheckContextTimeout(t *testing.T) { + cleanup := setupTestMocks() + defer cleanup() + + setupMockFS() + gofsutil.GOFSMock.InduceGetDiskFormatType = "ext4" + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockUtils := k8smock.NewMockUtilsInterface(ctrl) + mockUtils.EXPECT().GetPVCForVolume(gomock.Any(), "test-pv", "vol-003"). + Return(mockPVC("my-pvc", "default", nil), nil) + + cfg := &fsCheckConfig{enabled: true, mode: "checkOnly", k8sUtils: mockUtils} + sysDevice := &Device{FullPath: "/dev/sda", RealDev: "/dev/sda"} + accMode := &csi.VolumeCapability_AccessMode{Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER} + err := performFSCheck(ctx, sysDevice, cfg, accMode, "vol-003", testTargetPath) + if err != nil { + st, ok := status.FromError(err) + if ok { + assert.Equal(t, codes.Aborted, st.Code()) + } + } +} + +// ============================================================================ +// Section 9: Observer tests +// ============================================================================ + +// Test ID: U-020 +func TestFSCheckObserverEvents(t *testing.T) { + observer := &fsCheckPVCObserver{ + pvcName: "test-pvc", + pvcNamespace: "default", + devicePath: "/dev/sda", + fsType: "ext4", + volumeID: "vol-001", + } + observer.OnEvent(gofsutil.StartedFSCheckEvent) + observer.OnEvent(gofsutil.FoundNoErrorsEvent) + assert.Contains(t, observer.events, gofsutil.StartedFSCheckEvent) + assert.Contains(t, observer.events, gofsutil.FoundNoErrorsEvent) + assert.False(t, observer.sawTimeout) +} + +// Test ID: U-021 +func TestFSCheckObserverTimeout(t *testing.T) { + observer := &fsCheckPVCObserver{ + pvcName: "test-pvc", + pvcNamespace: "default", + devicePath: "/dev/sda", + fsType: "ext4", + volumeID: "vol-001", + } + observer.OnEvent(gofsutil.StartedFSCheckEvent) + observer.OnEvent(gofsutil.FSCheckTimedOutEvent) + assert.True(t, observer.sawTimeout) +} + +// Observer with nil EventRecorder - no panic +func TestFSCheckObserverNilRecorder(t *testing.T) { + observer := &fsCheckPVCObserver{ + pvcName: "test-pvc", + pvcNamespace: "default", + devicePath: "/dev/sda", + fsType: "ext4", + volumeID: "vol-001", + eventRecorder: nil, + } + observer.OnEvent(gofsutil.StartedFSCheckEvent) + assert.Len(t, observer.events, 1) +} + +// Observer with empty PVC info - no panic +func TestFSCheckObserverEmptyPVC(t *testing.T) { + observer := &fsCheckPVCObserver{ + pvcName: "", + pvcNamespace: "", + devicePath: "/dev/sda", + fsType: "ext4", + volumeID: "vol-001", + } + observer.OnEvent(gofsutil.StartedFSCheckEvent) + assert.Len(t, observer.events, 1) +} + +// Observer with FakeRecorder posts events +func TestFSCheckObserverWithRecorder(t *testing.T) { + fakeRecorder := record.NewFakeRecorder(100) + observer := &fsCheckPVCObserver{ + pvcName: "test-pvc", + pvcNamespace: "default", + devicePath: "/dev/sda", + fsType: "ext4", + volumeID: "vol-001", + eventRecorder: fakeRecorder, + } + observer.OnEvent(gofsutil.StartedFSCheckEvent) + observer.OnEvent(gofsutil.FoundNoErrorsEvent) + + // FakeRecorder buffers events in its channel + assert.Len(t, observer.events, 2) + // Drain channel to verify events were posted + event1 := <-fakeRecorder.Events + assert.Contains(t, event1, "FSCheckStarted") + event2 := <-fakeRecorder.Events + assert.Contains(t, event2, "FSCheckSucceeded") +} + +// ============================================================================ +// Section 10: Event mapping tests +// ============================================================================ + +// Test ID: U-020 (event mapping) +func TestMapFSCheckEventToK8s(t *testing.T) { + tests := []struct { + event string + wantType string + wantReason string + }{ + {gofsutil.StartedFSCheckEvent, "Normal", "FSCheckStarted"}, + {gofsutil.FoundNoErrorsEvent, "Normal", "FSCheckSucceeded"}, + {gofsutil.FinishedFSRepairEvent, "Normal", "FSCheckRepaired"}, + {gofsutil.FoundErrorsEvent, "Warning", "FSCheckFailed"}, + {gofsutil.FSCheckTimedOutEvent, "Warning", "FSCheckTimedOut"}, + {gofsutil.FSRepairFailedEvent, "Warning", "FSRepairFailed"}, + {gofsutil.FSCheckFailedEvent, "Warning", "FSCheckFailed"}, + {gofsutil.FoundDirtyLogEvent, "Normal", "FSCheckDirtyLog"}, + {gofsutil.StartFSRepairEvent, "Normal", "FSRepairStarted"}, + {gofsutil.StartLogReplayEvent, "Normal", "FSLogReplayStarted"}, + {gofsutil.LogReplayDoneEvent, "Normal", "FSLogReplayDone"}, + {gofsutil.LogReplayFailedEvent, "Warning", "FSLogReplayFailed"}, + {gofsutil.FSRepairTimedOutEvent, "Warning", "FSRepairTimedOut"}, + } + for _, tt := range tests { + t.Run(tt.event, func(t *testing.T) { + gotType, gotReason := mapFSCheckEventToK8s(tt.event) + assert.Equal(t, tt.wantType, gotType) + assert.Equal(t, tt.wantReason, gotReason) + }) + } +} + +// Default/unknown event +func TestMapFSCheckEventToK8sDefault(t *testing.T) { + gotType, gotReason := mapFSCheckEventToK8s("some-unknown-event") + assert.Equal(t, "Normal", gotType) + assert.Equal(t, "FSCheckEvent", gotReason) +} + +// ============================================================================ +// Section 11: Precedence matrix (AC-7) +// ============================================================================ + +// Test ID: U-034 - Global vs PVC-level precedence +func TestFSCheckGlobalVsPVCPrecedence(t *testing.T) { + tests := []struct { + name string + globalOn bool + globalMode string + pvcLabels map[string]string + wantEnabled bool + wantMode string + }{ + { + name: "global disabled, no PVC labels", globalOn: false, globalMode: "checkOnly", + pvcLabels: nil, wantEnabled: false, wantMode: "checkOnly", + }, + { + name: "global disabled, PVC enables", globalOn: false, globalMode: "checkOnly", + pvcLabels: map[string]string{PVCLabelFSCheckEnabled: "true"}, + wantEnabled: true, wantMode: "checkOnly", + }, + { + name: "global enabled, PVC disables", globalOn: true, globalMode: "checkAndRepair", + pvcLabels: map[string]string{PVCLabelFSCheckEnabled: "false"}, + wantEnabled: false, wantMode: "checkAndRepair", + }, + { + name: "global checkOnly, PVC overrides to checkAndRepair", globalOn: true, globalMode: "checkOnly", + pvcLabels: map[string]string{PVCLabelFSCheckMode: "checkAndRepair"}, + wantEnabled: true, wantMode: "checkAndRepair", + }, + { + name: "invalid PVC label falls back to global", globalOn: true, globalMode: "checkOnly", + pvcLabels: map[string]string{PVCLabelFSCheckEnabled: "invalid", PVCLabelFSCheckMode: "badvalue"}, + wantEnabled: true, wantMode: "checkOnly", + }, + { + name: "PVC enables and sets checkAndRepair", globalOn: false, globalMode: "checkOnly", + pvcLabels: map[string]string{PVCLabelFSCheckEnabled: "true", PVCLabelFSCheckMode: "checkAndRepair"}, + wantEnabled: true, wantMode: "checkAndRepair", + }, + { + name: "case-insensitive: PVC label True enables", globalOn: false, globalMode: "checkOnly", + pvcLabels: map[string]string{PVCLabelFSCheckEnabled: "True"}, + wantEnabled: true, wantMode: "checkOnly", + }, + { + name: "case-insensitive: PVC label FALSE disables", globalOn: true, globalMode: "checkAndRepair", + pvcLabels: map[string]string{PVCLabelFSCheckEnabled: "FALSE"}, + wantEnabled: false, wantMode: "checkAndRepair", + }, + { + name: "case-insensitive: PVC label CheckAndRepair mode", globalOn: true, globalMode: "checkOnly", + pvcLabels: map[string]string{PVCLabelFSCheckMode: "CheckAndRepair"}, + wantEnabled: true, wantMode: "checkAndRepair", + }, + { + name: "mode label ignored when disabled via PVC label", globalOn: true, globalMode: "checkOnly", + pvcLabels: map[string]string{PVCLabelFSCheckEnabled: "false", PVCLabelFSCheckMode: "checkAndRepair"}, + wantEnabled: false, wantMode: "checkOnly", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockUtils := k8smock.NewMockUtilsInterface(ctrl) + mockUtils.EXPECT().GetPVCForVolume(gomock.Any(), "test-pv", "vol-matrix"). + Return(mockPVC("my-pvc", "default", tt.pvcLabels), nil) + + cfg := &fsCheckConfig{enabled: tt.globalOn, mode: tt.globalMode, k8sUtils: mockUtils} + enabled, mode, _, _ := resolvePVCOverrides(context.Background(), cfg, "vol-matrix", testTargetPath) + assert.Equal(t, tt.wantEnabled, enabled, "enabled mismatch") + assert.Equal(t, tt.wantMode, mode, "mode mismatch") + }) + } +} + +// ============================================================================ +// Section 12: EventRecorder singleton tests (NEW) +// ============================================================================ + +func TestInitEventRecorder_Error(t *testing.T) { + origFn := newEventRecorderFunc + origCached := cachedEventRecorder + defer func() { + newEventRecorderFunc = origFn + eventRecorderOnce = *new(sync.Once) + cachedEventRecorder = origCached + }() + + newEventRecorderFunc = func() (record.EventRecorder, error) { + return nil, errors.New("k8s unavailable") + } + eventRecorderOnce = *new(sync.Once) + cachedEventRecorder = nil + + recorder := initEventRecorder() + assert.Nil(t, recorder) +} + +func TestInitEventRecorder_Success(t *testing.T) { + origFn := newEventRecorderFunc + origCached := cachedEventRecorder + defer func() { + newEventRecorderFunc = origFn + eventRecorderOnce = *new(sync.Once) + cachedEventRecorder = origCached + }() + + newEventRecorderFunc = func() (record.EventRecorder, error) { + return record.NewFakeRecorder(10), nil + } + eventRecorderOnce = *new(sync.Once) + cachedEventRecorder = nil + + recorder := initEventRecorder() + assert.NotNil(t, recorder) + + // Second call should return cached + recorder2 := initEventRecorder() + assert.Equal(t, recorder, recorder2) +} diff --git a/service/group_controller.go b/service/group_controller.go new file mode 100644 index 00000000..5fd33cb3 --- /dev/null +++ b/service/group_controller.go @@ -0,0 +1,756 @@ +/* + Copyright © 2026 Dell Inc. or its subsidiaries. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package service + +import ( + "context" + "fmt" + "strconv" + "strings" + "sync" + + pmax "github.com/dell/gopowermax/v2" + types "github.com/dell/gopowermax/v2/types/v100" + csi "github.com/container-storage-interface/spec/lib/go/csi" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// vgsSnapshotPendingState prevents concurrent group snapshot operations on the same SG. +var vgsSnapshotPendingState = pendingState{ + maxPending: 50, + pendingMutex: &sync.Mutex{}, +} + +// GroupControllerGetCapabilities returns the capabilities of the group controller service. +func (s *service) GroupControllerGetCapabilities( + _ context.Context, + _ *csi.GroupControllerGetCapabilitiesRequest, +) (*csi.GroupControllerGetCapabilitiesResponse, error) { + return &csi.GroupControllerGetCapabilitiesResponse{ + Capabilities: []*csi.GroupControllerServiceCapability{ + { + Type: &csi.GroupControllerServiceCapability_Rpc{ + Rpc: &csi.GroupControllerServiceCapability_RPC{ + Type: csi.GroupControllerServiceCapability_RPC_CREATE_DELETE_GET_VOLUME_GROUP_SNAPSHOT, + }, + }, + }, + }, + }, nil +} + +// CreateVolumeGroupSnapshot creates an atomic, crash-consistent group snapshot +// of multiple volumes on a PowerMax array using StorageGroup-level snapshot APIs. +func (s *service) CreateVolumeGroupSnapshot( + ctx context.Context, + req *csi.CreateVolumeGroupSnapshotRequest, +) (*csi.CreateVolumeGroupSnapshotResponse, error) { + log := log.WithContext(ctx) + + // Validate name + reqName := req.GetName() + if reqName == "" { + return nil, status.Error(codes.InvalidArgument, "required: Name") + } + + // Build the snapshot name with cluster prefix + snapName := buildGroupSnapName(s.getClusterPrefix(), reqName) + if len(snapName) >= MaxSnapIdentifierLength { + return nil, status.Errorf(codes.InvalidArgument, + "snapshot name %q is %d characters, must be less than %d", + snapName, len(snapName), MaxSnapIdentifierLength) + } + + // Validate source volumes + sourceVolumeIDs := req.GetSourceVolumeIds() + if len(sourceVolumeIDs) == 0 { + return nil, status.Error(codes.InvalidArgument, "required: SourceVolumeIds") + } + + vols, commonSymID, err := s.parseAndValidateGroupSourceVolumes(sourceVolumeIDs) + if err != nil { + return nil, err + } + + pmaxClient, err := s.getPmaxClient(commonSymID) + if err != nil { + return nil, err + } + if err := s.validateSnapshotLicense(ctx, commonSymID, pmaxClient); err != nil { + return nil, err + } + + volDetails, devIDToCSIVolID, sgName, err := s.getGroupVolumeDetails(ctx, pmaxClient, commonSymID, vols) + if err != nil { + return nil, err + } + + groupSnapshotID := buildGroupSnapshotID(commonSymID, sgName, snapName) + + // Idempotency: if this snapshot already exists on all requested volumes, + // return the existing group snapshot instead of creating a duplicate. + existingResp, err := s.checkGroupSnapshotIdempotency( + ctx, pmaxClient, commonSymID, snapName, groupSnapshotID, sgName, volDetails, devIDToCSIVolID) + if err != nil { + return nil, err + } + if existingResp != nil { + log.Infof("CreateVolumeGroupSnapshot idempotent: groupSnapshotId=%s already exists, returning existing snapshot", + groupSnapshotID) + return existingResp, nil + } + + // Pending state check per SG + stateID := volumeIDType(fmt.Sprintf("%s-%s", commonSymID, sgName)) + if err := stateID.checkAndUpdatePendingState(&vgsSnapshotPendingState); err != nil { + return nil, err + } + defer stateID.clearPending(&vgsSnapshotPendingState) + + // Create snapshots for multiple volumes in a single CreateSnapshot API call + // This creates a consistency group snapshot of all volumes + creationTime := timestamppb.Now() + memberSnapshots, err := s.CreateMultiVolumeSnapshot(ctx, commonSymID, snapName, sgName, volDetails, devIDToCSIVolID, pmaxClient, creationTime) + if err != nil { + return nil, status.Errorf(codes.Internal, + "failed to create multi-volume snapshot: %s", err.Error()) + } + + log.Infof("Created VolumeGroupSnapshot: groupSnapshotId=%s, members=%d, readyToUse=true", + groupSnapshotID, len(memberSnapshots)) + + return &csi.CreateVolumeGroupSnapshotResponse{ + GroupSnapshot: &csi.VolumeGroupSnapshot{ + GroupSnapshotId: groupSnapshotID, + Snapshots: memberSnapshots, + CreationTime: creationTime, + ReadyToUse: true, // Multi-volume snapshots are ready after creation + }, + }, nil +} + +// DeleteVolumeGroupSnapshot deletes a group snapshot and optionally cleans up temporary resources. +func (s *service) DeleteVolumeGroupSnapshot( + ctx context.Context, + req *csi.DeleteVolumeGroupSnapshotRequest, +) (*csi.DeleteVolumeGroupSnapshotResponse, error) { + log := log.WithContext(ctx) + + groupSnapshotID := req.GetGroupSnapshotId() + if groupSnapshotID == "" { + return nil, status.Error(codes.InvalidArgument, "required: GroupSnapshotId") + } + + symID, sgName, snapName, err := parseGroupSnapshotID(groupSnapshotID) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, + "invalid group snapshot ID format: %s", err.Error()) + } + + pmaxClient, err := s.GetPowerMaxClient(symID) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, + "failed to get PowerMax client for array %s: %s", symID, err.Error()) + } + + // Get the StorageGroup to determine which volumes are part of the group snapshot + _, err = pmaxClient.GetStorageGroup(ctx, symID, sgName) + if err != nil { + // If SG not found, Snapshots associated with it would already be deleted. + if types.IsNotFoundError(err) { + log.Infof("StorageGroup %s not found, assuming group snapshot %s is already deleted", sgName, groupSnapshotID) + return &csi.DeleteVolumeGroupSnapshotResponse{}, nil + } + + return nil, status.Errorf(codes.Internal, + "failed to get StorageGroup: %s", err.Error()) + } + + // Get the storage group snapshot information to find source volumes + sgSnapIDs, err := pmaxClient.GetStorageGroupSnapshotSnapIDs(ctx, symID, sgName, snapName) + if err != nil { + if types.IsNotFoundError(err) { + // Snapshot doesn't exist + log.Infof("Storage group snapshot %s not found, assuming it's already deleted", snapName) + return &csi.DeleteVolumeGroupSnapshotResponse{}, nil + } + + // For non-NotFound errors, fail fast - these could be server errors, permissions, etc. + return nil, status.Errorf(codes.Internal, + "failed to get storage group snapshot IDs: %s", err.Error()) + } + + // If no snapshot IDs exist, snapshot doesn't exist + if len(sgSnapIDs.SnapIDs) == 0 { + log.Infof("No snapshot IDs found for %s, snapshot does not exist", snapName) + return &csi.DeleteVolumeGroupSnapshotResponse{}, nil + } + + // Get the storage group snapshot details using the first snap ID + snapID := strconv.FormatInt(sgSnapIDs.SnapIDs[0], 10) + sgSnapshot, err := pmaxClient.GetStorageGroupSnapshotSnap(ctx, symID, sgName, snapName, snapID) + if err != nil { + if types.IsNotFoundError(err) { + // Snapshot details don't exist, assume snapshot doesn't exist + log.Infof("Storage group snapshot details %s not found, assuming snapshot doesn't exist", snapName) + return &csi.DeleteVolumeGroupSnapshotResponse{}, nil + } + // For non-NotFound errors, fail fast - these could be server errors, permissions, etc. + return nil, status.Errorf(codes.Internal, + "failed to get storage group snapshot details: %s", err.Error()) + } + + // Build source volumes list from the storage group snapshot + sourceVolumes := []types.VolumeList{} + if sgSnapshot != nil && sgSnapshot.SourceVolume != nil { + for _, srcVol := range sgSnapshot.SourceVolume { + sourceVolumes = append(sourceVolumes, types.VolumeList{Name: srcVol.Name}) + } + } + + // CSI spec requires SP to check snapshot_ids and report mismatch if detectable. + // Validate that every requested snapshot ID matches a volume we found in the SG. + if snapshotIDs := req.GetSnapshotIds(); len(snapshotIDs) > 0 { + expectedPrefix := fmt.Sprintf("%s-%s-", snapName, symID) + foundDevIDs := make(map[string]bool, len(sourceVolumes)) + for _, sv := range sourceVolumes { + foundDevIDs[sv.Name] = true + } + for _, sid := range snapshotIDs { + if !strings.HasPrefix(sid, expectedPrefix) { + return nil, status.Errorf(codes.InvalidArgument, + "snapshot ID %q does not match group snapshot (expected prefix %q)", sid, expectedPrefix) + } + devID := strings.TrimPrefix(sid, expectedPrefix) + if !foundDevIDs[devID] { + log.Warnf("snapshot ID %s references device %s which was not found with snapshot %s", sid, devID, snapName) + } + } + } + + // Use generation 0 for multi-volume snapshots + generation := int64(0) + + err = pmaxClient.DeleteSnapshotS(ctx, symID, snapName, sourceVolumes, generation) + if err != nil { + // Treat not-found as success (idempotent) + if types.IsNotFoundError(err) { + log.Infof("Group snapshot %s already deleted (not found), returning success", groupSnapshotID) + } else { + return nil, status.Errorf(codes.Internal, + "failed to delete multi-volume snapshot: %s", err.Error()) + } + } + + log.Infof("Deleted VolumeGroupSnapshot: groupSnapshotId=%s", groupSnapshotID) + return &csi.DeleteVolumeGroupSnapshotResponse{}, nil +} + +// GetVolumeGroupSnapshot queries the current state of a group snapshot. +func (s *service) GetVolumeGroupSnapshot( + ctx context.Context, + req *csi.GetVolumeGroupSnapshotRequest, +) (*csi.GetVolumeGroupSnapshotResponse, error) { + log := log.WithContext(ctx) + + groupSnapshotID := req.GetGroupSnapshotId() + if groupSnapshotID == "" { + return nil, status.Error(codes.InvalidArgument, "required: GroupSnapshotId") + } + + symID, sgName, snapName, err := parseGroupSnapshotID(groupSnapshotID) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, + "invalid group snapshot ID format: %s", err.Error()) + } + + pmaxClient, err := s.GetPowerMaxClient(symID) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, + "failed to get PowerMax client for array %s: %s", symID, err.Error()) + } + + // Validate the StorageGroup exists + _, err = pmaxClient.GetStorageGroup(ctx, symID, sgName) + if err != nil { + if types.IsNotFoundError(err) { + return nil, status.Errorf(codes.NotFound, + "StorageGroup %s not found on array %s", sgName, symID) + } + return nil, status.Errorf(codes.Internal, + "failed to get StorageGroup %s on array %s: %s", sgName, symID, err.Error()) + } + + // Get volume list from the StorageGroup to know which volumes should be in the snapshot + volIDList, err := pmaxClient.GetVolumeIDListInStorageGroup(ctx, symID, sgName) + if err != nil { + if types.IsNotFoundError(err) { + return nil, status.Errorf(codes.NotFound, + "StorageGroup %s not found on array %s", sgName, symID) + } + return nil, status.Errorf(codes.Internal, + "failed to get volume list from StorageGroup %s on array %s: %s", sgName, symID, err.Error()) + } + + // Get the storage group snapshot information using the same approach as DeleteVolumeGroupSnapshot + sgSnapshot, err := s.getStorageGroupSnapshotDetails(ctx, pmaxClient, symID, sgName, snapName) + if err != nil { + return nil, err + } + + // Create a map of snapshot source volumes for quick lookup + snapshotVolMap := make(map[string]types.SourceVolume, len(sgSnapshot.SourceVolume)) + for _, srcVol := range sgSnapshot.SourceVolume { + snapshotVolMap[srcVol.Name] = srcVol + } + + memberSnapshots := make([]*csi.Snapshot, 0, len(volIDList)) + groupReady := true + creationTime := timestamppb.Now() + + // Process all volumes in the StorageGroup and check if they have the snapshot + for _, devID := range volIDList { + srcVol, hasSnapshot := snapshotVolMap[devID] + if !hasSnapshot { + groupReady = false + continue + } + + // Convert capacity from GB to bytes + sizeBytes := int64(srcVol.CapacityGb * 1024 * 1024 * 1024) + + memberSnapshots = append(memberSnapshots, &csi.Snapshot{ + SnapshotId: buildMemberSnapshotID(snapName, symID, devID), + // For volume group snapshots we only have the device ID at this stage. + // Using the device ID keeps parity with CreateMultiVolumeSnapshot. + SourceVolumeId: devID, + SizeBytes: sizeBytes, + CreationTime: creationTime, + ReadyToUse: true, + }) + } + + if len(memberSnapshots) == 0 { + return nil, status.Errorf(codes.NotFound, + "group snapshot %s not found on array %s", snapName, symID) + } + + log.Infof("GetVolumeGroupSnapshot: groupSnapshotId=%s, members=%d, readyToUse=%t", + groupSnapshotID, len(memberSnapshots), groupReady) + + return &csi.GetVolumeGroupSnapshotResponse{ + GroupSnapshot: &csi.VolumeGroupSnapshot{ + GroupSnapshotId: groupSnapshotID, + Snapshots: memberSnapshots, + CreationTime: creationTime, + ReadyToUse: groupReady, + }, + }, nil +} + +// CreateMultiVolumeSnapshot creates snapshots for multiple volumes in a single API call +// This creates a consistency/storage group snapshot of all volumes using the CreateSnapshot API +func (s *service) CreateMultiVolumeSnapshot(ctx context.Context, symID, snapName, sgName string, volDetails []*types.Volume, devIDToCSIVolID map[string]string, pmaxClient pmax.Pmax, creationTime *timestamppb.Timestamp) ([]*csi.Snapshot, error) { + log := log.WithContext(ctx) + + // Build the source list with all volumes + sourceList := make([]types.VolumeList, 0, len(volDetails)) + for _, vol := range volDetails { + sourceList = append(sourceList, types.VolumeList{Name: vol.VolumeID}) + } + + // Create snapshot for all volumes in a single call + err := pmaxClient.CreateSnapshot(ctx, symID, snapName, sourceList, 0) + if err != nil { + return nil, fmt.Errorf("CreateGroupSnapshot failed with error: %s", err.Error()) + } + + // Get snapshot details to retrieve volume capacities + sgSnapshot, err := s.getStorageGroupSnapshotDetails(ctx, pmaxClient, symID, sgName, snapName) + if err != nil { + return nil, err + } + + // Build volume capacity map from snapshot response + capacityMap := make(map[string]float64) + for _, srcVol := range sgSnapshot.SourceVolume { + capacityMap[srcVol.Name] = srcVol.CapacityGb + } + + // Build member snapshot responses using capacities from snapshot response + memberSnapshots := make([]*csi.Snapshot, 0, len(volDetails)) + for _, vol := range volDetails { + volSnapID := buildMemberSnapshotID(snapName, symID, vol.VolumeID) + srcVolID := devIDToCSIVolID[vol.VolumeID] + if srcVolID == "" { + srcVolID = vol.VolumeID + } + + // Use capacity from snapshot response instead of volDetails + capacityGb, exists := capacityMap[vol.VolumeID] + if !exists { + return nil, fmt.Errorf("volume %s not found in snapshot response", vol.VolumeID) + } + sizeBytes := int64(capacityGb * 1024 * 1024 * 1024) + + memberSnapshots = append(memberSnapshots, &csi.Snapshot{ + SnapshotId: volSnapID, + SourceVolumeId: srcVolID, + SizeBytes: sizeBytes, + CreationTime: creationTime, + ReadyToUse: true, + }) + } + + log.Infof("Created group (multi-volume) snapshot: snapName=%s, volumes=%d", snapName, len(volDetails)) + return memberSnapshots, nil +} + +// --- Helper functions --- + +type groupVolInfo struct { + csiVolID string + volName string + symID string + devID string +} + +func (s *service) parseAndValidateGroupSourceVolumes(sourceVolumeIDs []string) ([]groupVolInfo, string, error) { + vols := make([]groupVolInfo, 0, len(sourceVolumeIDs)) + var commonSymID string + for _, srcVolID := range sourceVolumeIDs { + volName, symID, devID, remoteSymID, remoteVolID, err := s.parseCsiID(srcVolID) + if err != nil { + return nil, "", status.Errorf(codes.InvalidArgument, "failed to parse volume ID: %s", srcVolID) + } + if remoteSymID != "" && remoteVolID != "" { + return nil, "", status.Errorf(codes.InvalidArgument, + "group snapshots are not supported on PowerMax metro volumes: %s", srcVolID) + } + if commonSymID == "" { + commonSymID = symID + } else if commonSymID != symID { + return nil, "", status.Error(codes.InvalidArgument, + "all source volumes must belong to the same PowerMax array") + } + vols = append(vols, groupVolInfo{csiVolID: srcVolID, volName: volName, symID: symID, devID: devID}) + } + return vols, commonSymID, nil +} + +func (s *service) getPmaxClient(symID string) (pmax.Pmax, error) { + pmaxClient, err := s.GetPowerMaxClient(symID) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, + "failed to get PowerMax client for array %s: %s", symID, err.Error()) + } + return pmaxClient, nil +} + +func (s *service) getStorageGroupSnapshotDetails(ctx context.Context, pmaxClient pmax.Pmax, symID, sgName, snapName string) (*types.StorageGroupSnap, error) { + // Get the storage group snapshot information + sgSnapIDs, err := pmaxClient.GetStorageGroupSnapshotSnapIDs(ctx, symID, sgName, snapName) + if err != nil { + if types.IsNotFoundError(err) { + return nil, status.Errorf(codes.NotFound, + "Group snapshot %s not found on array %s", snapName, symID) + } + return nil, status.Errorf(codes.Internal, + "failed to get storage group snapshot IDs: %s", err.Error()) + } + + // If no snapshot IDs exist, snapshot doesn't exist + if len(sgSnapIDs.SnapIDs) == 0 { + return nil, status.Errorf(codes.NotFound, + "group snapshot %s not found on array %s", snapName, symID) + } + + // Get the storage group snapshot details using the first snap ID + snapID := strconv.FormatInt(sgSnapIDs.SnapIDs[0], 10) + sgSnapshot, err := pmaxClient.GetStorageGroupSnapshotSnap(ctx, symID, sgName, snapName, snapID) + if err != nil { + if types.IsNotFoundError(err) { + return nil, status.Errorf(codes.NotFound, + "Group snapshot details %s not found on array %s", snapName, symID) + } + return nil, status.Errorf(codes.Internal, + "failed to get storage group snapshot details: %s", err.Error()) + } + + if sgSnapshot == nil { + return nil, status.Errorf(codes.NotFound, + "group snapshot %s not found on array %s", snapName, symID) + } + + return sgSnapshot, nil +} + +func (s *service) validateSnapshotLicense(ctx context.Context, symID string, pmaxClient pmax.Pmax) error { + if err := s.IsSnapshotLicensed(ctx, symID, pmaxClient); err != nil { + return status.Errorf(codes.FailedPrecondition, + "Snapshot is not licensed on array %s: %s", symID, err.Error()) + } + return nil +} + +func (s *service) getGroupVolumeDetails(ctx context.Context, pmaxClient pmax.Pmax, symID string, vols []groupVolInfo) ([]*types.Volume, map[string]string, string, error) { + if len(vols) == 0 { + return nil, nil, "", status.Error(codes.InvalidArgument, "no volumes provided") + } + + // Get SG name from first volume only + firstVol, err := pmaxClient.GetVolumeByID(ctx, symID, vols[0].devID) + if err != nil { + if types.IsNotFoundError(err) { + return nil, nil, "", status.Errorf(codes.NotFound, + "volume %s not found on array %s", vols[0].devID, symID) + } + return nil, nil, "", status.Errorf(codes.Internal, + "failed to get volume %s on array %s: %s", vols[0].devID, symID, err.Error()) + } + + sgName, err := resolveStorageGroup([]*types.Volume{firstVol}, s.getClusterPrefix()) + if err != nil { + return nil, nil, "", err + } + + // Get all volumes in the SG to validate all requested volumes are present + sgVolIDs, err := pmaxClient.GetVolumeIDListInStorageGroup(ctx, symID, sgName) + if err != nil { + if types.IsNotFoundError(err) { + return nil, nil, "", status.Errorf(codes.NotFound, + "StorageGroup %s not found on array %s", sgName, symID) + } + return nil, nil, "", status.Errorf(codes.Internal, + "failed to get volume list from StorageGroup %s on array %s: %s", sgName, symID, err.Error()) + } + + // Validate all requested volumes are in the SG + sgVolSet := make(map[string]bool, len(sgVolIDs)) + for _, volID := range sgVolIDs { + sgVolSet[volID] = true + } + + for _, v := range vols { + if !sgVolSet[v.devID] { + return nil, nil, "", status.Errorf(codes.FailedPrecondition, + "volume %s is not in storage group %s", v.devID, sgName) + } + } + + // Return minimal volDetails (we'll get capacities from snapshot response later) + volDetails := make([]*types.Volume, 0, len(vols)) + devIDToCSIVolID := make(map[string]string, len(vols)) + for _, v := range vols { + volDetails = append(volDetails, &types.Volume{VolumeID: v.devID}) + devIDToCSIVolID[v.devID] = v.csiVolID + } + + return volDetails, devIDToCSIVolID, sgName, nil +} + +// buildGroupSnapshotID constructs the opaque group snapshot ID. +// Format: {symID}/{sgName}/{snapName} +func buildGroupSnapshotID(symID, sgName, snapName string) string { + return fmt.Sprintf("%s/%s/%s", symID, sgName, snapName) +} + +// buildGroupSnapName returns a CSI-compliant name that keeps the csi--grp- prefix +// intact and appends a trimmed suffix derived from request Name after removing the "groupsnapshot-" prefix. +// The suffix is collapsed to a single hyphen when multiple appear consecutively and is truncated so the +// total length stays below MaxSnapIdentifierLength. +func buildGroupSnapName(clusterPrefix, reqName string) string { + // Format: csi--grp- + // NOTE: Some PowerMax endpoints enforce snapshot identifiers to be *strictly* less than + // MaxSnapIdentifierLength (not <=). Reserve 1 character to guarantee that. + prefix := fmt.Sprintf("%s%s-grp-", CsiVolumePrefix, clusterPrefix) + maxTotalLen := MaxSnapIdentifierLength - 1 + maxSuffixLen := maxTotalLen - len(prefix) + if maxSuffixLen <= 0 { + return truncateString(prefix, maxTotalLen) + } + + suffix := strings.TrimPrefix(reqName, "groupsnapshot-") + suffix = strings.Trim(suffix, "-") + if len(suffix) > maxSuffixLen { + suffix = suffix[:maxSuffixLen] + suffix = strings.TrimRight(suffix, "-") + } + return fmt.Sprintf("%s%s", prefix, suffix) +} + +// parseGroupSnapshotID splits a group snapshot ID into its components. +func parseGroupSnapshotID(id string) (symID, sgName, snapName string, err error) { + parts := strings.Split(id, "/") + if len(parts) != 3 { + return "", "", "", fmt.Errorf("expected format symID/sgName/snapName, got %q", id) + } + return parts[0], parts[1], parts[2], nil +} + +// buildMemberSnapshotID constructs the per-member snapshot ID. +// Format: {snapName}-{symID}-{devID} +func buildMemberSnapshotID(snapName, symID, devID string) string { + return fmt.Sprintf("%s-%s-%s", snapName, symID, devID) +} + +// checkGroupSnapshotIdempotency checks whether the snapshot already exists on +// the backend volumes. It returns: +// - (response, nil) if the snapshot exists on ALL volumes (idempotent success). +// - (nil, error) if it exists on SOME but not all (partial / conflict). +// - (nil, nil) if it exists on NONE (caller should proceed to create). +func (s *service) checkGroupSnapshotIdempotency( + ctx context.Context, + pmaxClient pmax.Pmax, + symID, snapName, groupSnapshotID, sgName string, + volDetails []*types.Volume, + devIDToCSIVolID map[string]string, +) (*csi.CreateVolumeGroupSnapshotResponse, error) { + // First get the storage group snapshot snap IDs + sgSnapIDs, err := pmaxClient.GetStorageGroupSnapshotSnapIDs(ctx, symID, sgName, snapName) + if err != nil { + if types.IsNotFoundError(err) { + // Snapshot doesn't exist, return nil so caller can create it + return nil, nil + } + // For non-NotFound errors, we can't determine if snapshot exists + // Return error to let caller handle the failure appropriately + return nil, status.Errorf(codes.Internal, + "failed to check group snapshot idempotency: %s", err.Error()) + } + + if sgSnapIDs == nil || len(sgSnapIDs.SnapIDs) == 0 { + // No storage group snapshot found + return nil, nil + } + + // Get the storage group snapshot details using the first snap ID + snapID := strconv.FormatInt(sgSnapIDs.SnapIDs[0], 10) + sgSnapshot, err := pmaxClient.GetStorageGroupSnapshotSnap(ctx, symID, sgName, snapName, snapID) + if err != nil { + if types.IsNotFoundError(err) { + // Snapshot details don't exist, return nil so caller can create it + return nil, nil + } + // For non-NotFound errors, we can't determine if snapshot exists + // Return error to let caller handle the failure appropriately + return nil, status.Errorf(codes.Internal, + "failed to check group snapshot idempotency: %s", err.Error()) + } + if sgSnapshot == nil { + // No storage group snapshot found + return nil, nil + } + + // Check if the storage group snapshot includes all volumes + if len(sgSnapshot.SourceVolume) == 0 { + // No source volumes found + return nil, nil + } + + if len(sgSnapshot.SourceVolume) < len(volDetails) { + return nil, status.Errorf(codes.AlreadyExists, + "group snapshot name %q already exists on %d of %d volumes in SG %s; "+ + "cannot create with different source volumes", + snapName, len(sgSnapshot.SourceVolume), len(volDetails), sgName) + } + + // All volumes already have this snapshot — rebuild the response. + // Use volume capacities from the snapshot response instead of volDetails + creationTime := timestamppb.Now() + + // Build volume capacity map from snapshot response + capacityMap := make(map[string]float64) + for _, srcVol := range sgSnapshot.SourceVolume { + capacityMap[srcVol.Name] = srcVol.CapacityGb + } + + memberSnapshots := make([]*csi.Snapshot, 0, len(volDetails)) + for _, vol := range volDetails { + srcVolID := devIDToCSIVolID[vol.VolumeID] + if srcVolID == "" { + srcVolID = vol.VolumeID + } + + // Use capacity from snapshot response + capacityGb, exists := capacityMap[vol.VolumeID] + if !exists { + return nil, status.Errorf(codes.Internal, + "volume %s not found in existing snapshot response", vol.VolumeID) + } + sizeBytes := int64(capacityGb * 1024 * 1024 * 1024) + + memberSnapshots = append(memberSnapshots, &csi.Snapshot{ + SnapshotId: buildMemberSnapshotID(snapName, symID, vol.VolumeID), + SourceVolumeId: srcVolID, + SizeBytes: sizeBytes, + CreationTime: creationTime, + ReadyToUse: true, + }) + } + + return &csi.CreateVolumeGroupSnapshotResponse{ + GroupSnapshot: &csi.VolumeGroupSnapshot{ + GroupSnapshotId: groupSnapshotID, + Snapshots: memberSnapshots, + CreationTime: creationTime, + ReadyToUse: true, + }, + }, nil +} + +// resolveStorageGroup determines which StorageGroup to use for the group snapshot. +// Returns the SG name, whether it's a temporary SG that needs creation, and any error. +func resolveStorageGroup(volumes []*types.Volume, clusterPrefix string) (string, error) { + csiSGPrefix := fmt.Sprintf("%s-%s-", CSIPrefix, clusterPrefix) + + var commonSG string + for _, vol := range volumes { + var candidate string + for _, sg := range vol.StorageGroupIDList { + if strings.HasPrefix(sg, csiSGPrefix) { + candidate = sg + break + } + } + if candidate == "" { + return "", status.Error(codes.FailedPrecondition, + "volume must already belong to a CSI-managed storage group") + } + if commonSG == "" { + commonSG = candidate + } else if candidate != commonSG { + return "", status.Error(codes.FailedPrecondition, + "volumes have inconsistent StorageGroup membership; all must be in the same SG") + } + } + if commonSG == "" { + return "", status.Error(codes.FailedPrecondition, + "volumes must belong to a CSI-managed storage group") + } + return commonSG, nil +} + +// isSnapshotReady checks if any of the snapshot state strings contain "Established". +func isSnapshotReady(states []string) bool { + for _, state := range states { + if strings.Contains(state, "Established") { + return true + } + } + return false +} diff --git a/service/group_controller_test.go b/service/group_controller_test.go new file mode 100644 index 00000000..a6e12798 --- /dev/null +++ b/service/group_controller_test.go @@ -0,0 +1,1203 @@ +/* + Copyright © 2026 Dell Inc. or its subsidiaries. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package service + +import ( + "context" + "errors" + "net/http" + "sync" + "testing" + + "github.com/dell/csi-powermax/v2/pkg/symmetrix" + "github.com/dell/csi-powermax/v2/pkg/symmetrix/mocks" + types "github.com/dell/gopowermax/v2/types/v100" + csi "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/golang/mock/gomock" + gmock "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" +) + +// Helper functions for common mock setups +func setupGetStorageGroupAndSnapshot(c *mocks.MockPmaxClient, symID, sgName, snapName string) { + c.EXPECT().GetStorageGroup(gmock.Any(), symID, sgName).Times(1).Return(&types.StorageGroup{}, nil) + c.EXPECT().GetStorageGroupSnapshotSnapIDs(gmock.Any(), symID, sgName, snapName).Times(1).Return( + &types.SnapID{SnapIDs: []int64{12345}}, nil) + c.EXPECT().GetStorageGroupSnapshotSnap(gmock.Any(), symID, sgName, snapName, "12345").Times(1).Return( + &types.StorageGroupSnap{ + Name: snapName, + SourceVolume: []types.SourceVolume{ + {Name: "011AB", Capacity: 1000, CapacityGb: 1.0}, + {Name: "011CD", Capacity: 2000, CapacityGb: 2.0}, + }, + }, nil) +} + +func setupNotFoundErrorResponse() *types.Error { + return &types.Error{Message: "Not Found", HTTPStatusCode: http.StatusNotFound} +} + +// --- Helper function tests --- + +func TestParseGroupSnapshotID(t *testing.T) { + tests := []struct { + name string + id string + wantSymID string + wantSG string + wantSnap string + wantErr bool + wantErrMsg string + }{ + { + name: "valid ID", + id: "000120000001/csi-ABC-Diamond-SRP_1-SG/csi-GRP-ABC-mysnap", + wantSymID: "000120000001", + wantSG: "csi-ABC-Diamond-SRP_1-SG", + wantSnap: "csi-GRP-ABC-mysnap", + }, + { + name: "too few parts", + id: "000120000001/sgName", + wantErr: true, + wantErrMsg: "expected format", + }, + { + name: "too many parts", + id: "a/b/c/d/e", + wantErr: true, + wantErrMsg: "expected format", + }, + { + name: "empty string", + id: "", + wantErr: true, + wantErrMsg: "expected format", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + symID, sg, snap, err := parseGroupSnapshotID(tt.id) + if tt.wantErr { + assert.Error(t, err) + if tt.wantErrMsg != "" { + assert.Contains(t, err.Error(), tt.wantErrMsg) + } + } else { + assert.NoError(t, err) + assert.Equal(t, tt.wantSymID, symID) + assert.Equal(t, tt.wantSG, sg) + assert.Equal(t, tt.wantSnap, snap) + } + }) + } +} + +func TestBuildGroupSnapshotID(t *testing.T) { + id := buildGroupSnapshotID("000120000001", "mySG", "mySnap") + assert.Equal(t, "000120000001/mySG/mySnap", id) +} + +func TestBuildMemberSnapshotID(t *testing.T) { + id := buildMemberSnapshotID("csi-ABC-grp-snap", "000120000001", "011AB") + assert.Equal(t, "csi-ABC-grp-snap-000120000001-011AB", id) +} + +func TestBuildGroupSnapName(t *testing.T) { + tests := []struct { + name string + clusterPrefix string + reqName string + wantPrefix string + wantLenLt int + }{ + { + name: "short name", + clusterPrefix: "ABC", + reqName: "snap1", + wantPrefix: "csi-ABC-grp-snap1", + }, + { + name: "long name gets truncated", + clusterPrefix: "ABC", + reqName: "this-is-a-very-long-snapshot-name-that-exceeds-the-limit", + wantLenLt: MaxSnapIdentifierLength, + }, + { + name: "groupsnapshot with UUID", + clusterPrefix: "ABC", + reqName: "groupsnapshot-440da11b-234c-4147-8168-b5b142580865", + wantPrefix: "csi-ABC-grp-440da11b-234c-4147", + }, + { + name: "groupsnapshot with consecutive hyphens", + clusterPrefix: "ABC", + reqName: "groupsnapshot--name--with--many--hyphens", + wantPrefix: "csi-ABC-grp-name--with--many--h", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := buildGroupSnapName(tt.clusterPrefix, tt.reqName) + if tt.wantPrefix != "" { + assert.Equal(t, tt.wantPrefix, result) + } + if tt.wantLenLt > 0 { + assert.Less(t, len(result), tt.wantLenLt) + } + // Must always start with the prefix + assert.Contains(t, result, CsiVolumePrefix+tt.clusterPrefix+"-grp-") + }) + } +} + +func TestIsSnapshotReady(t *testing.T) { + assert.True(t, isSnapshotReady([]string{"Established"})) + assert.True(t, isSnapshotReady([]string{"CopyInProgress", "Established"})) + assert.False(t, isSnapshotReady([]string{"CopyInProgress"})) + assert.False(t, isSnapshotReady([]string{})) + assert.False(t, isSnapshotReady(nil)) +} + +func TestResolveStorageGroup(t *testing.T) { + prefix := "ABC" + csiSGName := "csi-ABC-Diamond-SRP_1-SG" + + tests := []struct { + name string + volumes []*types.Volume + wantSG string + wantErr bool + wantErrMsg string + }{ + { + name: "all volumes in same SG", + volumes: []*types.Volume{ + {StorageGroupIDList: []string{csiSGName}}, + {StorageGroupIDList: []string{csiSGName, "other-sg"}}, + }, + wantSG: csiSGName, + }, + { + name: "all volumes with no CSI SG", + volumes: []*types.Volume{ + {StorageGroupIDList: []string{"other-sg"}}, + {StorageGroupIDList: []string{}}, + }, + wantErr: true, + wantErrMsg: "volume must already belong", + }, + { + name: "mixed membership - some in SG, some not", + volumes: []*types.Volume{ + {StorageGroupIDList: []string{csiSGName}}, + {StorageGroupIDList: []string{}}, + }, + wantErr: true, + wantErrMsg: "volume must already belong", + }, + { + name: "volumes in different CSI SGs", + volumes: []*types.Volume{ + {StorageGroupIDList: []string{csiSGName}}, + {StorageGroupIDList: []string{"csi-ABC-Silver-SRP_1-SG"}}, + }, + wantErr: true, + wantErrMsg: "inconsistent StorageGroup", + }, + { + name: "single volume in SG", + volumes: []*types.Volume{ + {StorageGroupIDList: []string{csiSGName}}, + }, + wantSG: csiSGName, + }, + { + name: "single volume with no SG", + volumes: []*types.Volume{ + {StorageGroupIDList: []string{}}, + }, + wantErr: true, + wantErrMsg: "volume must already belong", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sg, err := resolveStorageGroup(tt.volumes, prefix) + if tt.wantErr { + assert.Error(t, err) + if tt.wantErrMsg != "" { + assert.Contains(t, err.Error(), tt.wantErrMsg) + } + } else { + assert.NoError(t, err) + assert.Equal(t, tt.wantSG, sg) + } + }) + } +} + +// --- RPC tests --- +// ... (rest of the code remains the same) + +func newTestService(clusterPrefix string) *service { + return &service{ + opts: Opts{ + ClusterPrefix: clusterPrefix, + }, + mutex: sync.Mutex{}, + cacheMutex: sync.Mutex{}, + nodeProbeMutex: sync.Mutex{}, + probeStatusMutex: sync.Mutex{}, + pollingFrequencyMutex: sync.Mutex{}, + waitGroup: sync.WaitGroup{}, + } +} + +func initMockClient(t *testing.T, symIDs ...string) *mocks.MockPmaxClient { + t.Helper() + c := mocks.NewMockPmaxClient(gmock.NewController(t)) + for _, id := range symIDs { + c.EXPECT().WithSymmetrixID(id).AnyTimes().Return(c) + } + c.EXPECT().GetHTTPClient().AnyTimes().Return(&http.Client{}) + err := symmetrix.Initialize(symIDs, c) + if err != nil { + t.Fatalf("failed to initialize mock client: %s", err) + } + return c +} + +func cleanMockClients(symIDs ...string) { + for _, id := range symIDs { + symmetrix.RemoveClient(id) + } +} + +func TestGroupControllerGetCapabilities(t *testing.T) { + s := newTestService("ABC") + resp, err := s.GroupControllerGetCapabilities(context.Background(), &csi.GroupControllerGetCapabilitiesRequest{}) + assert.NoError(t, err) + assert.NotNil(t, resp) + assert.Len(t, resp.Capabilities, 1) + assert.Equal(t, + csi.GroupControllerServiceCapability_RPC_CREATE_DELETE_GET_VOLUME_GROUP_SNAPSHOT, + resp.Capabilities[0].GetRpc().GetType(), + ) +} + +func Test_service_CreateVolumeGroupSnapshot(t *testing.T) { + const ( + sym1 = "000120000001" + sym2 = "000120000002" + dev1 = "011AB" + dev2 = "011CD" + sgName = "csi-ABC-Diamond-SRP_1-SG" + ) + vol1ID := CsiVolumePrefix + clusterPrefix + "-pmax-vol1-ns1-nsx-" + sym1 + "-" + dev1 + vol2ID := CsiVolumePrefix + clusterPrefix + "-pmax-vol2-ns1-nsx-" + sym1 + "-" + dev2 + + tests := []struct { + name string + req *csi.CreateVolumeGroupSnapshotRequest + before func(c *mocks.MockPmaxClient) + wantErr bool + wantErrMsg string + }{ + { + name: "empty name", + req: &csi.CreateVolumeGroupSnapshotRequest{ + Name: "", + SourceVolumeIds: []string{vol1ID}, + }, + before: func(_ *mocks.MockPmaxClient) {}, + wantErr: true, + wantErrMsg: "required: Name", + }, + { + name: "no source volumes", + req: &csi.CreateVolumeGroupSnapshotRequest{ + Name: "test-snap", + SourceVolumeIds: []string{}, + }, + before: func(_ *mocks.MockPmaxClient) {}, + wantErr: true, + wantErrMsg: "required: SourceVolumeIds", + }, + { + name: "invalid volume ID", + req: &csi.CreateVolumeGroupSnapshotRequest{ + Name: "test-snap", + SourceVolumeIds: []string{"bad-id"}, + }, + before: func(_ *mocks.MockPmaxClient) {}, + wantErr: true, + wantErrMsg: "failed to parse volume ID", + }, + { + name: "remote volume component not supported", + req: &csi.CreateVolumeGroupSnapshotRequest{ + Name: "test-snap", + SourceVolumeIds: []string{ + CsiVolumePrefix + clusterPrefix + "-pmax-vol1-ns1-nsx-" + sym1 + ":" + sym2 + "-" + dev1 + ":" + dev2, + }, + }, + before: func(_ *mocks.MockPmaxClient) {}, + wantErr: true, + wantErrMsg: "group snapshots are not supported on PowerMax metro volumes", + }, + { + name: "volumes on different arrays", + req: &csi.CreateVolumeGroupSnapshotRequest{ + Name: "test-snap", + SourceVolumeIds: []string{ + vol1ID, + CsiVolumePrefix + clusterPrefix + "-pmax-vol3-ns1-nsx-" + sym2 + "-" + dev2, + }, + }, + before: func(_ *mocks.MockPmaxClient) {}, + wantErr: true, + wantErrMsg: "same PowerMax array", + }, + { + name: "array not licensed for snapshots", + req: &csi.CreateVolumeGroupSnapshotRequest{ + Name: "snap1", + SourceVolumeIds: []string{vol1ID}, + }, + before: func(c *mocks.MockPmaxClient) { + c.EXPECT().IsAllowedArray(sym1).Times(1).Return(false, errors.New("not licensed")) + }, + wantErr: true, + wantErrMsg: "Snapshot is not licensed", + }, + { + name: "volume not found on array", + req: &csi.CreateVolumeGroupSnapshotRequest{ + Name: "snap1", + SourceVolumeIds: []string{vol1ID}, + }, + before: func(c *mocks.MockPmaxClient) { + c.EXPECT().IsAllowedArray(sym1).Times(1).Return(true, nil) + c.EXPECT().GetReplicationCapabilities(gmock.Any()).Times(1).Return( + &types.SymReplicationCapabilities{ + SymmetrixCapability: []types.SymmetrixCapability{ + {SymmetrixID: sym1, SnapVxCapable: true}, + }, + }, nil) + c.EXPECT().GetVolumeByID(gmock.Any(), sym1, dev1).Times(1).Return(nil, errors.New("not found")) + }, + wantErr: true, + wantErrMsg: "not found", + }, + { + name: "happy path - volumes in same SG", + req: &csi.CreateVolumeGroupSnapshotRequest{ + Name: "snap1", + SourceVolumeIds: []string{vol1ID, vol2ID}, + }, + before: func(c *mocks.MockPmaxClient) { + c.EXPECT().IsAllowedArray(sym1).Times(1).Return(true, nil) + c.EXPECT().GetReplicationCapabilities(gmock.Any()).Times(1).Return( + &types.SymReplicationCapabilities{ + SymmetrixCapability: []types.SymmetrixCapability{ + {SymmetrixID: sym1, SnapVxCapable: true}, + }, + }, nil) + // Optimized: GetVolumeByID called only for first volume to discover SG + c.EXPECT().GetVolumeByID(gmock.Any(), sym1, dev1).Times(1).Return( + &types.Volume{ + VolumeID: dev1, + CapacityGB: 1.0, + StorageGroupIDList: []string{sgName}, + }, nil) + // Optimized: GetVolumeIDListInStorageGroup called once to validate all volumes + c.EXPECT().GetVolumeIDListInStorageGroup(gmock.Any(), sym1, sgName).Times(1).Return([]string{dev1, dev2}, nil) + // Idempotency check: snapshot does not exist yet + c.EXPECT().GetStorageGroupSnapshotSnapIDs(gmock.Any(), sym1, sgName, "csi-ABC-grp-snap1").Times(1).Return(nil, &types.Error{Message: "Not Found", HTTPStatusCode: http.StatusNotFound}) + c.EXPECT().CreateSnapshot(gmock.Any(), sym1, "csi-ABC-grp-snap1", gmock.Any(), int64(0)).Times(1).Return(nil) + // Optimized: GetStorageGroupSnapshotSnapIDs and GetStorageGroupSnapshotSnap called to get volume capacities + c.EXPECT().GetStorageGroupSnapshotSnapIDs(gmock.Any(), sym1, sgName, "csi-ABC-grp-snap1").Times(1).Return( + &types.SnapID{SnapIDs: []int64{12345}}, nil) + c.EXPECT().GetStorageGroupSnapshotSnap(gmock.Any(), sym1, sgName, "csi-ABC-grp-snap1", "12345").Times(1).Return( + &types.StorageGroupSnap{ + Name: "csi-ABC-grp-snap1", + SourceVolume: []types.SourceVolume{ + {Name: dev1, Capacity: 1000, CapacityGb: 1.0}, + {Name: dev2, Capacity: 2000, CapacityGb: 2.0}, + }, + }, nil) + }, + wantErr: false, + }, + { + name: "idempotent - snapshot already exists on all volumes", + req: &csi.CreateVolumeGroupSnapshotRequest{ + Name: "snap1", + SourceVolumeIds: []string{vol1ID, vol2ID}, + }, + before: func(c *mocks.MockPmaxClient) { + c.EXPECT().IsAllowedArray(sym1).Times(1).Return(true, nil) + c.EXPECT().GetReplicationCapabilities(gmock.Any()).Times(1).Return( + &types.SymReplicationCapabilities{ + SymmetrixCapability: []types.SymmetrixCapability{ + {SymmetrixID: sym1, SnapVxCapable: true}, + }, + }, nil) + // Optimized: GetVolumeByID called only for first volume to discover SG + c.EXPECT().GetVolumeByID(gmock.Any(), sym1, dev1).Times(1).Return( + &types.Volume{ + VolumeID: dev1, + CapacityGB: 1.0, + StorageGroupIDList: []string{sgName}, + }, nil) + // Optimized: GetVolumeIDListInStorageGroup called once to validate all volumes + c.EXPECT().GetVolumeIDListInStorageGroup(gmock.Any(), sym1, sgName).Times(1).Return([]string{dev1, dev2}, nil) + // Idempotency check: snapshot already exists on all volumes + c.EXPECT().GetStorageGroupSnapshotSnapIDs(gmock.Any(), sym1, sgName, "csi-ABC-grp-snap1").Times(1).Return( + &types.SnapID{SnapIDs: []int64{12345}}, nil) + c.EXPECT().GetStorageGroupSnapshotSnap(gmock.Any(), sym1, sgName, "csi-ABC-grp-snap1", "12345").Times(1).Return( + &types.StorageGroupSnap{ + Name: "csi-ABC-grp-snap1", + SourceVolume: []types.SourceVolume{ + {Name: dev1, Capacity: 1000, CapacityGb: 1.0}, + {Name: dev2, Capacity: 2000, CapacityGb: 2.0}, + }, + }, nil) + // CreateSnapshot should NOT be called + }, + wantErr: false, + }, + { + name: "partial match - snapshot exists on some volumes", + req: &csi.CreateVolumeGroupSnapshotRequest{ + Name: "snap1", + SourceVolumeIds: []string{vol1ID, vol2ID}, + }, + before: func(c *mocks.MockPmaxClient) { + c.EXPECT().IsAllowedArray(sym1).Times(1).Return(true, nil) + c.EXPECT().GetReplicationCapabilities(gmock.Any()).Times(1).Return( + &types.SymReplicationCapabilities{ + SymmetrixCapability: []types.SymmetrixCapability{ + {SymmetrixID: sym1, SnapVxCapable: true}, + }, + }, nil) + // Optimized: GetVolumeByID called only for first volume to discover SG + c.EXPECT().GetVolumeByID(gmock.Any(), sym1, dev1).Times(1).Return( + &types.Volume{ + VolumeID: dev1, + CapacityGB: 1.0, + StorageGroupIDList: []string{sgName}, + }, nil) + // Optimized: GetVolumeIDListInStorageGroup called once to validate all volumes + c.EXPECT().GetVolumeIDListInStorageGroup(gmock.Any(), sym1, sgName).Times(1).Return([]string{dev1, dev2}, nil) + // Snapshot exists but with fewer source volumes (partial match) + c.EXPECT().GetStorageGroupSnapshotSnapIDs(gmock.Any(), sym1, sgName, "csi-ABC-grp-snap1").Times(1).Return( + &types.SnapID{SnapIDs: []int64{12345}}, nil) + c.EXPECT().GetStorageGroupSnapshotSnap(gmock.Any(), sym1, sgName, "csi-ABC-grp-snap1", "12345").Times(1).Return( + &types.StorageGroupSnap{ + Name: "csi-ABC-grp-snap1", + SourceVolume: []types.SourceVolume{ + {Name: dev1, Capacity: 1000, CapacityGb: 1.0}, // Only one volume + }, + }, nil) + // CreateSnapshot should NOT be called + }, + wantErr: true, + wantErrMsg: "already exists on 1 of 2 volumes", + }, + { + name: "create SG snapshot API fails", + req: &csi.CreateVolumeGroupSnapshotRequest{ + Name: "snap1", + SourceVolumeIds: []string{vol1ID}, + }, + before: func(c *mocks.MockPmaxClient) { + c.EXPECT().IsAllowedArray(sym1).Times(1).Return(true, nil) + c.EXPECT().GetReplicationCapabilities(gmock.Any()).Times(1).Return( + &types.SymReplicationCapabilities{ + SymmetrixCapability: []types.SymmetrixCapability{ + {SymmetrixID: sym1, SnapVxCapable: true}, + }, + }, nil) + // Optimized: GetVolumeByID called only for first volume to discover SG + c.EXPECT().GetVolumeByID(gmock.Any(), sym1, dev1).Times(1).Return( + &types.Volume{ + VolumeID: dev1, + CapacityGB: 1.0, + StorageGroupIDList: []string{sgName}, + }, nil) + // Optimized: GetVolumeIDListInStorageGroup called once to validate all volumes + c.EXPECT().GetVolumeIDListInStorageGroup(gmock.Any(), sym1, sgName).Times(1).Return([]string{dev1}, nil) + // Idempotency check: snapshot does not exist yet + c.EXPECT().GetStorageGroupSnapshotSnapIDs(gmock.Any(), sym1, sgName, "csi-ABC-grp-snap1").Times(1).Return(nil, &types.Error{Message: "Not Found", HTTPStatusCode: http.StatusNotFound}) + c.EXPECT().CreateSnapshot(gmock.Any(), sym1, "csi-ABC-grp-snap1", gmock.Any(), int64(0)).Times(1).Return(errors.New("create snapshot failed")) + }, + wantErr: true, + wantErrMsg: "failed to create multi-volume snapshot", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := newTestService(clusterPrefix) + // Clear snapshot license cache between tests + RemoveReplicationCapability(sym1) + + c := initMockClient(t, sym1, sym2) + defer cleanMockClients(sym1, sym2) + tt.before(c) + + resp, err := s.CreateVolumeGroupSnapshot(context.Background(), tt.req) + if tt.wantErr { + assert.Error(t, err) + if tt.wantErrMsg != "" { + assert.Contains(t, err.Error(), tt.wantErrMsg) + } + assert.Nil(t, resp) + } else { + assert.NoError(t, err) + assert.NotNil(t, resp) + assert.NotNil(t, resp.GroupSnapshot) + assert.NotEmpty(t, resp.GroupSnapshot.GroupSnapshotId) + assert.Len(t, resp.GroupSnapshot.Snapshots, len(tt.req.SourceVolumeIds)) + if len(tt.req.SourceVolumeIds) > 0 { + wantSources := make(map[string]bool, len(tt.req.SourceVolumeIds)) + for _, id := range tt.req.SourceVolumeIds { + wantSources[id] = true + } + for _, sn := range resp.GroupSnapshot.Snapshots { + assert.True(t, wantSources[sn.SourceVolumeId], "expected SourceVolumeId to be a CSI volume handle") + } + } + assert.True(t, resp.GroupSnapshot.ReadyToUse) + } + }) + } +} + +func Test_service_DeleteVolumeGroupSnapshot(t *testing.T) { + const sym1 = "000120000001" + sgName := "csi-ABC-Diamond-SRP_1-SG" + snapName := "csi-GRP-ABC-snap1" + groupSnapID := buildGroupSnapshotID(sym1, sgName, snapName) + + tests := []struct { + name string + req *csi.DeleteVolumeGroupSnapshotRequest + before func(c *mocks.MockPmaxClient) + wantErr bool + wantErrMsg string + }{ + { + name: "empty group snapshot ID", + req: &csi.DeleteVolumeGroupSnapshotRequest{ + GroupSnapshotId: "", + }, + before: func(_ *mocks.MockPmaxClient) {}, + wantErr: true, + wantErrMsg: "required: GroupSnapshotId", + }, + { + name: "invalid group snapshot ID format", + req: &csi.DeleteVolumeGroupSnapshotRequest{ + GroupSnapshotId: "bad-format", + }, + before: func(_ *mocks.MockPmaxClient) {}, + wantErr: true, + wantErrMsg: "invalid group snapshot ID format", + }, + { + name: "happy path - delete succeeds", + req: &csi.DeleteVolumeGroupSnapshotRequest{ + GroupSnapshotId: groupSnapID, + }, + before: func(c *mocks.MockPmaxClient) { + setupGetStorageGroupAndSnapshot(c, sym1, sgName, snapName) + c.EXPECT().DeleteSnapshotS(gmock.Any(), sym1, snapName, gmock.Any(), int64(0)).Times(1).Return(nil) + }, + wantErr: false, + }, + { + name: "idempotent - snapshot not found", + req: &csi.DeleteVolumeGroupSnapshotRequest{ + GroupSnapshotId: groupSnapID, + }, + before: func(c *mocks.MockPmaxClient) { + c.EXPECT().GetStorageGroup(gmock.Any(), sym1, sgName).Times(1).Return(&types.StorageGroup{}, nil) + nf := setupNotFoundErrorResponse() + c.EXPECT().GetStorageGroupSnapshotSnapIDs(gmock.Any(), sym1, sgName, snapName).Times(1).Return(nil, nf) + }, + wantErr: false, + }, + { + name: "delete API error (non-not-found)", + req: &csi.DeleteVolumeGroupSnapshotRequest{ + GroupSnapshotId: groupSnapID, + }, + before: func(c *mocks.MockPmaxClient) { + setupGetStorageGroupAndSnapshot(c, sym1, sgName, snapName) + c.EXPECT().DeleteSnapshotS(gmock.Any(), sym1, snapName, gmock.Any(), int64(0)).Times(1).Return(errors.New("internal error")) + }, + wantErr: true, + wantErrMsg: "failed to delete multi-volume snapshot", + }, + { + name: "snapshot_ids prefix mismatch", + req: &csi.DeleteVolumeGroupSnapshotRequest{ + GroupSnapshotId: groupSnapID, + SnapshotIds: []string{"wrong-prefix-011AB"}, + }, + before: func(c *mocks.MockPmaxClient) { + setupGetStorageGroupAndSnapshot(c, sym1, sgName, snapName) + }, + wantErr: true, + wantErrMsg: "does not match group snapshot", + }, + { + name: "snapshot_ids valid - delete succeeds", + req: &csi.DeleteVolumeGroupSnapshotRequest{ + GroupSnapshotId: groupSnapID, + SnapshotIds: []string{"csi-GRP-ABC-snap1-000120000001-011AB"}, + }, + before: func(c *mocks.MockPmaxClient) { + setupGetStorageGroupAndSnapshot(c, sym1, sgName, snapName) + c.EXPECT().DeleteSnapshotS(gmock.Any(), sym1, snapName, gmock.Any(), int64(0)).Times(1).Return(nil) + }, + wantErr: false, + }, + { + name: "StorageGroup not found - empty deletion succeeds", + req: &csi.DeleteVolumeGroupSnapshotRequest{ + GroupSnapshotId: groupSnapID, + }, + before: func(c *mocks.MockPmaxClient) { + nf := setupNotFoundErrorResponse() + c.EXPECT().GetStorageGroup(gmock.Any(), sym1, sgName).Times(1).Return(nil, nf) + }, + wantErr: false, + }, + { + name: "GetStorageGroup fails with non-NotFound error", + req: &csi.DeleteVolumeGroupSnapshotRequest{ + GroupSnapshotId: groupSnapID, + }, + before: func(c *mocks.MockPmaxClient) { + c.EXPECT().GetStorageGroup(gmock.Any(), sym1, sgName).Times(1).Return(nil, errors.New("internal error")) + }, + wantErr: true, + wantErrMsg: "failed to get StorageGroup", + }, + { + name: "GetStorageGroupSnapshotSnapIDs returns NotFound - empty deletion succeeds", + req: &csi.DeleteVolumeGroupSnapshotRequest{ + GroupSnapshotId: groupSnapID, + }, + before: func(c *mocks.MockPmaxClient) { + c.EXPECT().GetStorageGroup(gmock.Any(), sym1, sgName).Times(1).Return(&types.StorageGroup{}, nil) + nf := setupNotFoundErrorResponse() + c.EXPECT().GetStorageGroupSnapshotSnapIDs(gmock.Any(), sym1, sgName, snapName).Times(1).Return(nil, nf) + }, + wantErr: false, + }, + { + name: "GetStorageGroupSnapshotSnapIDs returns internal error", + req: &csi.DeleteVolumeGroupSnapshotRequest{ + GroupSnapshotId: groupSnapID, + }, + before: func(c *mocks.MockPmaxClient) { + c.EXPECT().GetStorageGroup(gmock.Any(), sym1, sgName).Times(1).Return(&types.StorageGroup{}, nil) + c.EXPECT().GetStorageGroupSnapshotSnapIDs(gmock.Any(), sym1, sgName, snapName).Times(1).Return(nil, errors.New("internal error")) + }, + wantErr: true, + wantErrMsg: "failed to get storage group snapshot IDs", + }, + { + name: "GetStorageGroupSnapshotSnapIDs returns empty snap IDs", + req: &csi.DeleteVolumeGroupSnapshotRequest{ + GroupSnapshotId: groupSnapID, + }, + before: func(c *mocks.MockPmaxClient) { + c.EXPECT().GetStorageGroup(gmock.Any(), sym1, sgName).Times(1).Return(&types.StorageGroup{}, nil) + c.EXPECT().GetStorageGroupSnapshotSnapIDs(gmock.Any(), sym1, sgName, snapName).Times(1).Return(&types.SnapID{SnapIDs: []int64{}}, nil) + }, + wantErr: false, + }, + { + name: "GetStorageGroupSnapshotSnap returns NotFound - assume snapshot doesn't exist", + req: &csi.DeleteVolumeGroupSnapshotRequest{ + GroupSnapshotId: groupSnapID, + }, + before: func(c *mocks.MockPmaxClient) { + c.EXPECT().GetStorageGroup(gmock.Any(), sym1, sgName).Times(1).Return(&types.StorageGroup{}, nil) + c.EXPECT().GetStorageGroupSnapshotSnapIDs(gmock.Any(), sym1, sgName, snapName).Times(1).Return( + &types.SnapID{SnapIDs: []int64{12345}}, nil) + nf := &types.Error{Message: "Not Found", HTTPStatusCode: http.StatusNotFound} + c.EXPECT().GetStorageGroupSnapshotSnap(gmock.Any(), sym1, sgName, snapName, "12345").Times(1).Return(nil, nf) + }, + wantErr: false, + }, + { + name: "GetStorageGroupSnapshotSnap returns internal error", + req: &csi.DeleteVolumeGroupSnapshotRequest{ + GroupSnapshotId: groupSnapID, + }, + before: func(c *mocks.MockPmaxClient) { + c.EXPECT().GetStorageGroup(gmock.Any(), sym1, sgName).Times(1).Return(&types.StorageGroup{}, nil) + c.EXPECT().GetStorageGroupSnapshotSnapIDs(gmock.Any(), sym1, sgName, snapName).Times(1).Return( + &types.SnapID{SnapIDs: []int64{12345}}, nil) + c.EXPECT().GetStorageGroupSnapshotSnap(gmock.Any(), sym1, sgName, snapName, "12345").Times(1).Return(nil, errors.New("internal error")) + }, + wantErr: true, + wantErrMsg: "failed to get storage group snapshot details", + }, + { + name: "GetStorageGroupSnapshotSnap returns nil snapshot", + req: &csi.DeleteVolumeGroupSnapshotRequest{ + GroupSnapshotId: groupSnapID, + }, + before: func(c *mocks.MockPmaxClient) { + c.EXPECT().GetStorageGroup(gmock.Any(), sym1, sgName).Times(1).Return(&types.StorageGroup{}, nil) + c.EXPECT().GetStorageGroupSnapshotSnapIDs(gmock.Any(), sym1, sgName, snapName).Times(1).Return( + &types.SnapID{SnapIDs: []int64{12345}}, nil) + c.EXPECT().GetStorageGroupSnapshotSnap(gmock.Any(), sym1, sgName, snapName, "12345").Times(1).Return(nil, nil) + c.EXPECT().DeleteSnapshotS(gmock.Any(), sym1, snapName, gmock.Any(), int64(0)).Times(1).Return(nil) + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := newTestService(clusterPrefix) + c := initMockClient(t, sym1) + defer cleanMockClients(sym1) + tt.before(c) + + resp, err := s.DeleteVolumeGroupSnapshot(context.Background(), tt.req) + if tt.wantErr { + assert.Error(t, err) + if tt.wantErrMsg != "" { + assert.Contains(t, err.Error(), tt.wantErrMsg) + } + } else { + assert.NoError(t, err) + assert.NotNil(t, resp) + } + }) + } +} + +func Test_service_GetVolumeGroupSnapshot(t *testing.T) { + const sym1 = "000120000001" + sgName := "csi-ABC-Diamond-SRP_1-SG" + snapName := "csi-ABC-grp-snap1" + groupSnapID := buildGroupSnapshotID(sym1, sgName, snapName) + + tests := []struct { + name string + req *csi.GetVolumeGroupSnapshotRequest + before func(c *mocks.MockPmaxClient) + wantErr bool + wantErrMsg string + wantReady bool + wantMembers int + }{ + { + name: "empty group snapshot ID", + req: &csi.GetVolumeGroupSnapshotRequest{ + GroupSnapshotId: "", + }, + before: func(_ *mocks.MockPmaxClient) {}, + wantErr: true, + wantErrMsg: "required: GroupSnapshotId", + }, + { + name: "invalid format", + req: &csi.GetVolumeGroupSnapshotRequest{ + GroupSnapshotId: "invalid", + }, + before: func(_ *mocks.MockPmaxClient) {}, + wantErr: true, + wantErrMsg: "invalid group snapshot ID format", + }, + { + name: "snapshot not found", + req: &csi.GetVolumeGroupSnapshotRequest{ + GroupSnapshotId: groupSnapID, + }, + before: func(c *mocks.MockPmaxClient) { + c.EXPECT().GetStorageGroup(gmock.Any(), sym1, sgName).Times(1).Return(&types.StorageGroup{}, nil) + c.EXPECT().GetVolumeIDListInStorageGroup(gmock.Any(), sym1, sgName).Times(1).Return([]string{"011AB"}, nil) + c.EXPECT().GetStorageGroupSnapshotSnapIDs(gmock.Any(), sym1, sgName, snapName).Times(1).Return( + &types.SnapID{SnapIDs: []int64{}}, nil) + }, + wantErr: true, + wantErrMsg: "group snapshot", + }, + { + name: "happy path - established", + req: &csi.GetVolumeGroupSnapshotRequest{ + GroupSnapshotId: groupSnapID, + }, + before: func(c *mocks.MockPmaxClient) { + c.EXPECT().GetStorageGroup(gmock.Any(), sym1, sgName).Times(1).Return(&types.StorageGroup{}, nil) + c.EXPECT().GetVolumeIDListInStorageGroup(gmock.Any(), sym1, sgName).Times(1).Return([]string{"011AB", "011CD"}, nil) + c.EXPECT().GetStorageGroupSnapshotSnapIDs(gmock.Any(), sym1, sgName, snapName).Times(1).Return( + &types.SnapID{SnapIDs: []int64{12345}}, nil) + c.EXPECT().GetStorageGroupSnapshotSnap(gmock.Any(), sym1, sgName, snapName, "12345").Times(1).Return( + &types.StorageGroupSnap{ + Name: snapName, + SourceVolume: []types.SourceVolume{ + {Name: "011AB", Capacity: 1000, CapacityGb: 1.0}, + {Name: "011CD", Capacity: 2000, CapacityGb: 2.0}, + }, + }, nil) + }, + wantErr: false, + wantReady: true, + wantMembers: 2, + }, + { + name: "partial - one member missing", + req: &csi.GetVolumeGroupSnapshotRequest{ + GroupSnapshotId: groupSnapID, + }, + before: func(c *mocks.MockPmaxClient) { + c.EXPECT().GetStorageGroup(gmock.Any(), sym1, sgName).Times(1).Return(&types.StorageGroup{}, nil) + c.EXPECT().GetVolumeIDListInStorageGroup(gmock.Any(), sym1, sgName).Times(1).Return([]string{"011AB", "011CD"}, nil) + c.EXPECT().GetStorageGroupSnapshotSnapIDs(gmock.Any(), sym1, sgName, snapName).Times(1).Return( + &types.SnapID{SnapIDs: []int64{12345}}, nil) + c.EXPECT().GetStorageGroupSnapshotSnap(gmock.Any(), sym1, sgName, snapName, "12345").Times(1).Return( + &types.StorageGroupSnap{ + Name: snapName, + SourceVolume: []types.SourceVolume{ + {Name: "011AB", Capacity: 1000, CapacityGb: 1.0}, + // Note: 011CD is missing from SourceVolume, simulating partial snapshot + }, + }, nil) + }, + wantErr: false, + wantReady: false, + wantMembers: 1, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := newTestService(clusterPrefix) + c := initMockClient(t, sym1) + defer cleanMockClients(sym1) + tt.before(c) + + resp, err := s.GetVolumeGroupSnapshot(context.Background(), tt.req) + if tt.wantErr { + assert.Error(t, err) + if tt.wantErrMsg != "" { + assert.Contains(t, err.Error(), tt.wantErrMsg) + } + } else { + assert.NoError(t, err) + assert.NotNil(t, resp) + assert.NotNil(t, resp.GroupSnapshot) + assert.Equal(t, tt.wantReady, resp.GroupSnapshot.ReadyToUse) + assert.Len(t, resp.GroupSnapshot.Snapshots, tt.wantMembers) + } + }) + } +} + +func Test_service_checkGroupSnapshotIdempotency(t *testing.T) { + const ( + symID = "000120000001" + snapName = "test-snap" + groupSnapshotID = "test-group-snap" + sgName = "test-sg" + vol1ID = "00001" + vol2ID = "00002" + csiVol1ID = "csi-vol1" + csiVol2ID = "csi-vol2" + ) + + tests := []struct { + name string + volDetails []*types.Volume + devIDToCSIVolID map[string]string + mockSetup func(*mocks.MockPmaxClient) + wantResponse bool + wantErr bool + wantErrMsg string + }{ + { + name: "snapshot does not exist - GetStorageGroupSnapshotSnapIDs returns error", + volDetails: []*types.Volume{ + {VolumeID: vol1ID, CapacityGB: 1.0}, + {VolumeID: vol2ID, CapacityGB: 2.0}, + }, + devIDToCSIVolID: map[string]string{vol1ID: csiVol1ID, vol2ID: csiVol2ID}, + mockSetup: func(c *mocks.MockPmaxClient) { + c.EXPECT().GetStorageGroupSnapshotSnapIDs(gmock.Any(), symID, sgName, snapName).Times(1).Return(nil, &types.Error{Message: "Not Found", HTTPStatusCode: http.StatusNotFound}) + }, + wantResponse: false, + wantErr: false, + }, + { + name: "snapshot exists - full match (all source volumes)", + volDetails: []*types.Volume{ + {VolumeID: vol1ID, CapacityGB: 1.0}, + {VolumeID: vol2ID, CapacityGB: 2.0}, + }, + devIDToCSIVolID: map[string]string{vol1ID: csiVol1ID, vol2ID: csiVol2ID}, + mockSetup: func(c *mocks.MockPmaxClient) { + c.EXPECT().GetStorageGroupSnapshotSnapIDs(gmock.Any(), symID, sgName, snapName).Times(1).Return(&types.SnapID{SnapIDs: []int64{12345}}, nil) + c.EXPECT().GetStorageGroupSnapshotSnap(gmock.Any(), symID, sgName, snapName, "12345").Times(1).Return(&types.StorageGroupSnap{ + Name: snapName, + SourceVolume: []types.SourceVolume{ + {Name: vol1ID, Capacity: 1000, CapacityGb: 1.0}, + {Name: vol2ID, Capacity: 2000, CapacityGb: 2.0}, + }, + }, nil) + }, + wantResponse: true, + wantErr: false, + }, + { + name: "snapshot exists - partial match (fewer source volumes)", + volDetails: []*types.Volume{ + {VolumeID: vol1ID, CapacityGB: 1.0}, + {VolumeID: vol2ID, CapacityGB: 2.0}, + }, + devIDToCSIVolID: map[string]string{vol1ID: csiVol1ID, vol2ID: csiVol2ID}, + mockSetup: func(c *mocks.MockPmaxClient) { + c.EXPECT().GetStorageGroupSnapshotSnapIDs(gmock.Any(), symID, sgName, snapName).Times(1).Return(&types.SnapID{SnapIDs: []int64{12345}}, nil) + c.EXPECT().GetStorageGroupSnapshotSnap(gmock.Any(), symID, sgName, snapName, "12345").Times(1).Return(&types.StorageGroupSnap{ + Name: snapName, + SourceVolume: []types.SourceVolume{ + {Name: vol1ID, Capacity: 1000, CapacityGb: 1.0}, // Only one volume + }, + }, nil) + }, + wantResponse: false, + wantErr: true, + wantErrMsg: "already exists", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gmock.NewController(t) + defer ctrl.Finish() + + mockClient := mocks.NewMockPmaxClient(ctrl) + if tt.mockSetup != nil { + tt.mockSetup(mockClient) + } + + s := &service{} + resp, err := s.checkGroupSnapshotIdempotency( + context.Background(), + mockClient, + symID, + snapName, + groupSnapshotID, + sgName, + tt.volDetails, + tt.devIDToCSIVolID, + ) + + if tt.wantErr { + assert.Error(t, err) + if tt.wantErrMsg != "" { + assert.Contains(t, err.Error(), tt.wantErrMsg) + } + assert.Nil(t, resp) + } else { + assert.NoError(t, err) + if tt.wantResponse { + assert.NotNil(t, resp) + assert.NotNil(t, resp.GroupSnapshot) + assert.Equal(t, groupSnapshotID, resp.GroupSnapshot.GroupSnapshotId) + assert.Len(t, resp.GroupSnapshot.Snapshots, len(tt.volDetails)) + } else { + assert.Nil(t, resp) + } + } + }) + } +} + +func Test_service_getStorageGroupSnapshotDetails(t *testing.T) { + const ( + symID = "000120000001" + sgName = "test-sg" + snapName = "test-snap" + ) + + tests := []struct { + name string + mockSetup func(*mocks.MockPmaxClient) + wantErr bool + wantErrMsg string + wantSnapshot bool + }{ + { + name: "GetStorageGroupSnapshotSnapIDs returns NotFound error", + mockSetup: func(c *mocks.MockPmaxClient) { + c.EXPECT().GetStorageGroupSnapshotSnapIDs(gmock.Any(), symID, sgName, snapName).Times(1).Return(nil, &types.Error{Message: "Not Found", HTTPStatusCode: http.StatusNotFound}) + }, + wantErr: true, + wantErrMsg: "Group snapshot test-snap not found on array 000120000001", + }, + { + name: "happy path - snapshot found", + mockSetup: func(c *mocks.MockPmaxClient) { + c.EXPECT().GetStorageGroupSnapshotSnapIDs(gmock.Any(), symID, sgName, snapName).Times(1).Return(&types.SnapID{SnapIDs: []int64{12345}}, nil) + c.EXPECT().GetStorageGroupSnapshotSnap(gmock.Any(), symID, sgName, snapName, "12345").Times(1).Return(&types.StorageGroupSnap{ + SourceVolume: []types.SourceVolume{ + {Name: "vol1", CapacityGb: 1.0}, + {Name: "vol2", CapacityGb: 2.0}, + }, + }, nil) + }, + wantErr: false, + wantSnapshot: true, + }, + { + name: "GetStorageGroupSnapshotSnapIDs returns empty snap IDs", + mockSetup: func(c *mocks.MockPmaxClient) { + c.EXPECT().GetStorageGroupSnapshotSnapIDs(gmock.Any(), symID, sgName, snapName).Times(1).Return(&types.SnapID{SnapIDs: []int64{}}, nil) + }, + wantErr: true, + wantErrMsg: "group snapshot test-snap not found on array 000120000001", + }, + { + name: "GetStorageGroupSnapshotSnap returns nil snapshot", + mockSetup: func(c *mocks.MockPmaxClient) { + c.EXPECT().GetStorageGroupSnapshotSnapIDs(gmock.Any(), symID, sgName, snapName).Times(1).Return(&types.SnapID{SnapIDs: []int64{12345}}, nil) + c.EXPECT().GetStorageGroupSnapshotSnap(gmock.Any(), symID, sgName, snapName, "12345").Times(1).Return(nil, nil) + }, + wantErr: true, + wantErrMsg: "group snapshot test-snap not found on array 000120000001", + }, + { + name: "GetStorageGroupSnapshotSnapIDs returns internal error", + mockSetup: func(c *mocks.MockPmaxClient) { + c.EXPECT().GetStorageGroupSnapshotSnapIDs(gmock.Any(), symID, sgName, snapName).Times(1).Return(nil, errors.New("internal error")) + }, + wantErr: true, + wantErrMsg: "failed to get storage group snapshot IDs: internal error", + }, + { + name: "GetStorageGroupSnapshotSnap returns internal error", + mockSetup: func(c *mocks.MockPmaxClient) { + c.EXPECT().GetStorageGroupSnapshotSnapIDs(gmock.Any(), symID, sgName, snapName).Times(1).Return(&types.SnapID{SnapIDs: []int64{12345}}, nil) + c.EXPECT().GetStorageGroupSnapshotSnap(gmock.Any(), symID, sgName, snapName, "12345").Times(1).Return(nil, errors.New("internal error")) + }, + wantErr: true, + wantErrMsg: "failed to get storage group snapshot details: internal error", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := newTestService("ABC") + ctrl := gomock.NewController(t) + defer ctrl.Finish() + c := mocks.NewMockPmaxClient(ctrl) + + tt.mockSetup(c) + + sgSnapshot, err := s.getStorageGroupSnapshotDetails(context.Background(), c, symID, sgName, snapName) + + if tt.wantErr { + assert.Error(t, err) + if tt.wantErrMsg != "" { + assert.Contains(t, err.Error(), tt.wantErrMsg) + } + assert.Nil(t, sgSnapshot) + } else { + assert.NoError(t, err) + if tt.wantSnapshot { + assert.NotNil(t, sgSnapshot) + assert.Len(t, sgSnapshot.SourceVolume, 2) + } else { + assert.Nil(t, sgSnapshot) + } + } + }) + } +} + +func Test_service_getGroupVolumeDetails(t *testing.T) { + const symID = "000120000001" + + tests := []struct { + name string + vols []groupVolInfo + mockSetup func(*mocks.MockPmaxClient) + wantErr bool + wantErrMsg string + wantSgName string + }{ + { + name: "no volumes provided", + vols: []groupVolInfo{}, + mockSetup: func(_ *mocks.MockPmaxClient) { + // No setup needed + }, + wantErr: true, + wantErrMsg: "no volumes provided", + }, + { + name: "GetVolumeByID returns NotFound", + vols: []groupVolInfo{ + {csiVolID: "csi-vol1", volName: "vol1", symID: symID, devID: "011AB"}, + }, + mockSetup: func(c *mocks.MockPmaxClient) { + nf := &types.Error{Message: "Not Found", HTTPStatusCode: http.StatusNotFound} + c.EXPECT().GetVolumeByID(gmock.Any(), symID, "011AB").Times(1).Return(nil, nf) + }, + wantErr: true, + wantErrMsg: "volume 011AB not found on array 000120000001", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := newTestService("ABC") + ctrl := gomock.NewController(t) + defer ctrl.Finish() + c := mocks.NewMockPmaxClient(ctrl) + + tt.mockSetup(c) + + volDetails, devIDToCSIVolID, sgNameResult, err := s.getGroupVolumeDetails(context.Background(), c, symID, tt.vols) + + if tt.wantErr { + assert.Error(t, err) + if tt.wantErrMsg != "" { + assert.Contains(t, err.Error(), tt.wantErrMsg) + } + assert.Nil(t, volDetails) + assert.Nil(t, devIDToCSIVolID) + assert.Empty(t, sgNameResult) + } else { + assert.NoError(t, err) + if tt.wantSgName != "" { + assert.Equal(t, tt.wantSgName, sgNameResult) + } + assert.NotNil(t, volDetails) + assert.NotNil(t, devIDToCSIVolID) + } + }) + } +} diff --git a/service/identity.go b/service/identity.go index 645e553a..d01da906 100644 --- a/service/identity.go +++ b/service/identity.go @@ -15,13 +15,12 @@ package service import ( + "context" "fmt" "strings" commonext "github.com/dell/dell-csi-extensions/common" - "golang.org/x/net/context" - csi "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/protobuf/types/known/wrapperspb" @@ -72,6 +71,13 @@ func (s *service) GetPluginCapabilities( }, }, }, + { + Type: &csi.PluginCapability_Service_{ + Service: &csi.PluginCapability_Service{ + Type: csi.PluginCapability_Service_GROUP_CONTROLLER_SERVICE, + }, + }, + }, } } return &rep, nil @@ -103,7 +109,10 @@ func (s *service) Probe( maximumStartupDelay = 1 } // Initialize the node - _ = s.nodeStartup(ctx) + err := s.nodeStartup(ctx) + if err != nil { + log.Errorf("Failed to initialize node service: %v", err) + } } } ready := new(wrapperspb.BoolValue) diff --git a/service/interfaces.go b/service/interfaces.go index e8614fea..f76507e0 100644 --- a/service/interfaces.go +++ b/service/interfaces.go @@ -1,5 +1,5 @@ /* - Copyright © 2021-2025 Dell Inc. or its subsidiaries. All Rights Reserved. + Copyright © 2021-2026 Dell Inc. or its subsidiaries. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -110,7 +110,7 @@ func (s *service) createDbusConnection() error { return nil } -var dbusNewConnectionFunc = func() (*dbus.Conn, error) { +var dbusNewConnectionFunc = func() (dBusConn, error) { return dbus.New() } diff --git a/service/migration.go b/service/migration.go index 8dad825f..03d279a9 100644 --- a/service/migration.go +++ b/service/migration.go @@ -17,6 +17,7 @@ limitations under the License. package service import ( + "context" "fmt" "path" "strconv" @@ -28,7 +29,6 @@ import ( csimgr "github.com/dell/dell-csi-extensions/migration" types "github.com/dell/gopowermax/v2/types/v100" - "golang.org/x/net/context" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" diff --git a/service/mock-data/symmetrix49.json b/service/mock-data/symmetrix49.json new file mode 100644 index 00000000..627a0419 --- /dev/null +++ b/service/mock-data/symmetrix49.json @@ -0,0 +1,15 @@ +{ + "symmetrixId": "000197900049", + "device_count": 1045, + "ucode": "5978.441.441", + "model": "PowerMax_2000", + "local": true, + "all_flash": true, + "disk_count": 8, + "cache_size_mb": 203776, + "data_encryption": "Disabled", + "microcode": "6079.425.0", + "microcode_date": "01-15-2026", + "microcode_registered_build": 90, + "microcode_package_version": "10.4.0.0 (Release 01, Build 6079_425/0090, 2026-01-15 10:00:00)" +} diff --git a/service/mock-data/symmetrixList.json b/service/mock-data/symmetrixList.json index 37916e40..1118ba32 100644 --- a/service/mock-data/symmetrixList.json +++ b/service/mock-data/symmetrixList.json @@ -2,6 +2,7 @@ "symmetrixId": [ "000197802104", "000197900046", - "000197900047" + "000197900047", + "000197900049" ] } diff --git a/service/mount.go b/service/mount.go index 1bf691a6..001cc9db 100644 --- a/service/mount.go +++ b/service/mount.go @@ -88,6 +88,7 @@ func GetDevice(path string) (*Device, error) { func publishVolume( req *csi.NodePublishVolumeRequest, privDir, device string, reqID string, + fsCfg *fsCheckConfig, ) error { id := req.GetVolumeId() @@ -203,9 +204,24 @@ func publishVolume( log.Debug(fmt.Sprintf("MOUNT: %#v", m)) resolvedMountDevice := evalSymlinks(m.Device) if resolvedMountDevice != sysDevice.RealDev { - return status.Errorf(codes.FailedPrecondition, "Private mount point: %s mounted by different device: %s", privTgt, resolvedMountDevice) + // Check if the old device still exists before hard-failing + // This handles stale mounts from disconnected devices + _, statErr := os.Stat(resolvedMountDevice) + if statErr != nil && os.IsNotExist(statErr) { + // Device doesn't exist (ENOENT) - it's a stale mount + // Clean it up and proceed + log.Infof("Detected stale mount from disconnected device %s, cleaning up", resolvedMountDevice) + if err := gofsutil.Unmount(ctx, privTgt); err != nil { + log.Warnf("Unmount of stale mount %s failed: %s", privTgt, err.Error()) + } + // After cleanup, don't set alreadyMounted - let the normal mount flow proceed + } else { + // Device exists or other stat error - this is a real conflict + return status.Errorf(codes.FailedPrecondition, "Private mount point: %s mounted by different device: %s", privTgt, resolvedMountDevice) + } + } else { + alreadyMounted = true } - alreadyMounted = true } } } @@ -216,6 +232,11 @@ func publishVolume( if fs == "xfs" { mntFlags = append(mntFlags, "nouuid") } + // Perform FS check on the unmounted device before mounting + if err := performFSCheck(ctx, sysDevice, fsCfg, accMode, id, target); err != nil { + cleanupPrivateTarget(reqID, privTgt) + return err + } if err := handlePrivFSMount( ctx, accMode, sysDevice, mntFlags, fs, privTgt); err != nil { // K8S may have removed the desired mount point. Clean up the private target. @@ -395,6 +416,56 @@ func getPrivateMountPoint(privDir string, name string) string { return fmt.Sprintf("%s/%s", privDir, name) } +func podMountPathSuffix(path string) string { + cleanPath := filepath.Clean(filepath.ToSlash(path)) + if idx := strings.Index(cleanPath, "/pods/"); idx >= 0 { + return cleanPath[idx:] + } + return "" +} + +func privateMountPathSuffix(path string) string { + cleanPath := filepath.Clean(filepath.ToSlash(path)) + if idx := strings.Index(cleanPath, "/plugins/"); idx >= 0 { + return cleanPath[idx:] + } + return "" +} + +func isPrivateMountVariant(path, privTgt string) bool { + cleanPath := filepath.Clean(filepath.ToSlash(path)) + cleanPrivTgt := filepath.Clean(filepath.ToSlash(privTgt)) + + if cleanPath == cleanPrivTgt || cleanPath == filepath.Clean("/noderoot"+cleanPrivTgt) { + return true + } + + privSuffix := privateMountPathSuffix(cleanPrivTgt) + if privSuffix == "" { + return false + } + + pathSuffix := privateMountPathSuffix(cleanPath) + return pathSuffix != "" && pathSuffix == privSuffix +} + +func isRequestedTargetMountVariant(path, target string) bool { + cleanPath := filepath.Clean(filepath.ToSlash(path)) + cleanTarget := filepath.Clean(filepath.ToSlash(target)) + + if cleanPath == cleanTarget || cleanPath == filepath.Clean("/noderoot"+cleanTarget) { + return true + } + + targetSuffix := podMountPathSuffix(cleanTarget) + if targetSuffix == "" { + return false + } + + pathSuffix := podMountPathSuffix(cleanPath) + return pathSuffix != "" && pathSuffix == targetSuffix +} + func contains(list []string, item string) bool { for _, x := range list { if x == item { @@ -472,7 +543,6 @@ func unpublishVolume( return lastUnmounted, status.Error(codes.InvalidArgument, "target path required") } - dupTarget := filepath.Join("/noderoot", target) // make sure device is valid sysDevice, err := GetDevice(device) @@ -503,9 +573,9 @@ func unpublishVolume( for _, m := range mnts { // Added check for sysDevice.FullPath as that is used by multipath mapper if m.Source == sysDevice.RealDev || m.Device == sysDevice.RealDev || m.Device == sysDevice.FullPath { - if m.Path == privTgt { + if isPrivateMountVariant(m.Path, privTgt) { privMnt = true - } else if m.Path == target || m.Path == dupTarget { + } else if isRequestedTargetMountVariant(m.Path, target) { tgtMnt = append(tgtMnt, m.Path) } } @@ -560,14 +630,77 @@ func unmountPrivMount( privTgtDup := filepath.Join("/noderoot", target) - // remove private mount if we can (if there are no other mounts - if len(mnts) == 1 || len(mnts) == 2 { - for i, m := range mnts { - if m.Path == target || m.Path == privTgtDup { - if err := gofsutil.Unmount(ctx, m.Path); err != nil { - return false, err + // In bind mount environments the same private mount appears under multiple + // kubelet root paths (e.g., /var/lib/kubelet, /data/kubelet). Identify all + // private mount variants vs real consumer mounts from other pods. + hasOtherMounts := false + for _, m := range mnts { + if m.Path != "" && !isPrivateMountVariant(m.Path, target) { + hasOtherMounts = true + break + } + } + + if !hasOtherMounts { + // In bind mount environments, kernel mount propagation is asynchronous. + // Use exponential backoff to wait for propagation to complete before giving up. + maxRetries := 10 + backoff := 50 * time.Millisecond + + for retry := 0; retry < maxRetries; retry++ { + hasOtherMounts = false + for _, m := range mnts { + if m.Path != "" && !isPrivateMountVariant(m.Path, target) { + hasOtherMounts = true + break + } + } + if hasOtherMounts { + log.Debugf("Detected non-private mounts while unmounting %s; skipping private mount cleanup", target) + break + } + + allUnmounted := true + + for i, m := range mnts { + if m.Path != "" && isPrivateMountVariant(m.Path, target) { + if err := gofsutil.Unmount(ctx, m.Path); err != nil { + if m.Path == target || m.Path == privTgtDup { + // Critical path unmount failed - mark for retry + allUnmounted = false + if retry == maxRetries-1 { + // Last retry - return error + return false, err + } + log.Infof("Unmount of %s failed (retry %d/%d): %s", m.Path, retry+1, maxRetries, err.Error()) + } else { + // Non-critical path unmount failed - log but continue + log.Infof("Unmount of %s: %s", m.Path, err.Error()) + } + } else { + mnts[i].Path = "" + } + } + } + + // If all unmounted successfully, break out of retry loop + if allUnmounted { + log.Debugf("All mounts unmounted successfully after %d retries", retry+1) + break + } + + // Wait for kernel mount propagation with exponential backoff + if retry < maxRetries-1 { + time.Sleep(backoff) + // Refresh mount table to get current state + mnts, err = getDevMounts(dev) + if err != nil { + log.Warnf("Failed to refresh mount table during retry: %s", err.Error()) + } + // Exponential backoff: 50ms, 100ms, 200ms, 400ms, 800ms (max) + if backoff < 800*time.Millisecond { + backoff *= 2 } - mnts[i].Path = "" } } diff --git a/service/mount_test.go b/service/mount_test.go index eea61b86..f7a391a3 100644 --- a/service/mount_test.go +++ b/service/mount_test.go @@ -15,9 +15,14 @@ package service import ( + "context" + "os" + "path/filepath" "reflect" + "strings" "testing" + "github.com/dell/gofsutil" csi "github.com/container-storage-interface/spec/lib/go/csi" ) @@ -61,6 +66,261 @@ func Test_singleAccessMode(t *testing.T) { } } +func setupUnmountVolumeTest(t *testing.T) (string, string, string) { + t.Helper() + gofsutil.UseMockFS() + gofsutil.GOFSMock.InduceGetMountsError = false + gofsutil.GOFSMock.InduceUnmountError = false + gofsutil.GOFSMockMounts = make([]gofsutil.Info, 0) + + t.Cleanup(func() { + gofsutil.GOFSMock.InduceGetMountsError = false + gofsutil.GOFSMock.InduceUnmountError = false + gofsutil.GOFSMockMounts = make([]gofsutil.Info, 0) + }) + + tmpDir := t.TempDir() + devicePath := filepath.Join(tmpDir, "dev") + if err := os.WriteFile(devicePath, []byte("x"), 0o600); err != nil { + t.Fatalf("failed to create mock device file: %v", err) + } + + realDev, err := filepath.EvalSymlinks(devicePath) + if err != nil { + t.Fatalf("failed to resolve device path: %v", err) + } + + privDir := filepath.Join(tmpDir, "private") + if err := os.MkdirAll(privDir, 0o750); err != nil { + t.Fatalf("failed to create private mount dir: %v", err) + } + + oldEmulateBlockDevice := unitTestEmulateBlockDevice + unitTestEmulateBlockDevice = true + t.Cleanup(func() { + unitTestEmulateBlockDevice = oldEmulateBlockDevice + }) + + return devicePath, realDev, privDir +} + +func getMockMountPaths() map[string]bool { + paths := map[string]bool{} + for _, m := range gofsutil.GOFSMockMounts { + paths[m.Path] = true + } + return paths +} + +func TestIsRequestedTargetMountVariant(t *testing.T) { + target := "/var/lib/kubelet/pods/pod-2/volumes/kubernetes.io~csi/pvc-123/mount" + dataVariant := strings.Replace(target, "/var/lib/kubelet", "/data/kubelet", 1) + otherPod := "/var/lib/kubelet/pods/pod-1/volumes/kubernetes.io~csi/pvc-123/mount" + + tests := []struct { + name string + path string + target string + want bool + }{ + { + name: "exact target path", + path: target, + target: target, + want: true, + }, + { + name: "noderoot target path", + path: "/noderoot" + target, + target: target, + want: true, + }, + { + name: "bind mount variant for same pod", + path: dataVariant, + target: target, + want: true, + }, + { + name: "different pod path should not match", + path: otherPod, + target: target, + want: false, + }, + { + name: "non-pod custom path only matches exact target", + path: "/tmp/target-a", + target: "/tmp/target-b", + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isRequestedTargetMountVariant(tt.path, tt.target); got != tt.want { + t.Fatalf("isRequestedTargetMountVariant(%s, %s) = %v, want %v", tt.path, tt.target, got, tt.want) + } + }) + } +} + +func TestIsPrivateMountVariant(t *testing.T) { + privTgt := "/var/lib/kubelet/plugins/powermax.emc.dell.com/disks/csi-vol-abc123" + bindVariant := strings.Replace(privTgt, "/var/lib/kubelet", "/data/kubelet", 1) + + tests := []struct { + name string + path string + want bool + }{ + { + name: "exact private mount path", + path: privTgt, + want: true, + }, + { + name: "noderoot private mount path", + path: "/noderoot" + privTgt, + want: true, + }, + { + name: "bind mount private path", + path: bindVariant, + want: true, + }, + { + name: "different volume private path should not match", + path: "/var/lib/kubelet/plugins/powermax.emc.dell.com/disks/csi-vol-abc123-backup", + want: false, + }, + { + name: "pod consumer mount should not match", + path: "/var/lib/kubelet/pods/pod-1/volumes/kubernetes.io~csi/pvc-123/mount", + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isPrivateMountVariant(tt.path, privTgt); got != tt.want { + t.Fatalf("isPrivateMountVariant(%s, %s) = %v, want %v", tt.path, privTgt, got, tt.want) + } + }) + } +} + +func TestUnmountPrivMountSkipsWhenConsumerMountExists(t *testing.T) { + devicePath, realDev, privDir := setupUnmountVolumeTest(t) + + volumeID := "pvc-123" + privTgt := getPrivateMountPoint(privDir, volumeID) + consumerMount := "/var/lib/kubelet/pods/pod-1/volumes/kubernetes.io~csi/pvc-123/mount" + + gofsutil.GOFSMockMounts = []gofsutil.Info{ + {Device: realDev, Source: realDev, Path: privTgt}, + {Device: realDev, Source: privTgt, Path: consumerMount}, + } + + dev := &Device{FullPath: devicePath, RealDev: realDev} + lastUnmounted, err := unmountPrivMount(context.Background(), dev, privTgt) + if err != nil { + t.Fatalf("unmountPrivMount returned error: %v", err) + } + if lastUnmounted { + t.Fatalf("expected lastUnmounted=false when consumer mount is still present") + } + + paths := getMockMountPaths() + if !paths[privTgt] || !paths[consumerMount] { + t.Fatalf("expected mounts to remain untouched when consumer mount exists, got %#v", gofsutil.GOFSMockMounts) + } +} + +func TestUnpublishVolumeUnmountsOnlyRequestedTargetVariants(t *testing.T) { + devicePath, realDev, privDir := setupUnmountVolumeTest(t) + + volumeID := "pvc-123" + privTgt := getPrivateMountPoint(privDir, volumeID) + + targetPod2 := "/var/lib/kubelet/pods/pod-2/volumes/kubernetes.io~csi/pvc-123/mount" + targetPod2Data := strings.Replace(targetPod2, "/var/lib/kubelet", "/data/kubelet", 1) + targetPod1 := "/var/lib/kubelet/pods/pod-1/volumes/kubernetes.io~csi/pvc-123/mount" + targetPod1Data := strings.Replace(targetPod1, "/var/lib/kubelet", "/data/kubelet", 1) + + gofsutil.GOFSMockMounts = []gofsutil.Info{ + {Device: realDev, Source: realDev, Path: privTgt}, + {Device: realDev, Source: realDev, Path: "/noderoot" + privTgt}, + {Device: realDev, Source: privTgt, Path: targetPod2}, + {Device: realDev, Source: privTgt, Path: "/noderoot" + targetPod2}, + {Device: realDev, Source: privTgt, Path: targetPod2Data}, + {Device: realDev, Source: privTgt, Path: "/noderoot" + targetPod2Data}, + {Device: realDev, Source: privTgt, Path: targetPod1}, + {Device: realDev, Source: privTgt, Path: "/noderoot" + targetPod1}, + {Device: realDev, Source: privTgt, Path: targetPod1Data}, + {Device: realDev, Source: privTgt, Path: "/noderoot" + targetPod1Data}, + } + + req := &csi.NodeUnpublishVolumeRequest{ + VolumeId: volumeID, + TargetPath: targetPod2, + } + + lastUnmounted, err := unpublishVolume(req, privDir, devicePath, "req-1") + if err != nil { + t.Fatalf("unpublishVolume returned error: %v", err) + } + if lastUnmounted { + t.Fatalf("expected lastUnmounted=false when another pod is still mounted") + } + + paths := getMockMountPaths() + for _, removed := range []string{targetPod2, "/noderoot" + targetPod2, targetPod2Data, "/noderoot" + targetPod2Data} { + if paths[removed] { + t.Fatalf("requested target variant %s should have been unmounted", removed) + } + } + + for _, remaining := range []string{targetPod1, "/noderoot" + targetPod1, targetPod1Data, "/noderoot" + targetPod1Data, privTgt} { + if !paths[remaining] { + t.Fatalf("mount %s should remain for still-running pod", remaining) + } + } +} + +func TestUnpublishVolumeLastUnmountedTrueWhenAllPodMountsGone(t *testing.T) { + devicePath, realDev, privDir := setupUnmountVolumeTest(t) + + volumeID := "pvc-123" + privTgt := getPrivateMountPoint(privDir, volumeID) + target := "/var/lib/kubelet/pods/pod-2/volumes/kubernetes.io~csi/pvc-123/mount" + targetData := strings.Replace(target, "/var/lib/kubelet", "/data/kubelet", 1) + + gofsutil.GOFSMockMounts = []gofsutil.Info{ + {Device: realDev, Source: realDev, Path: privTgt}, + {Device: realDev, Source: realDev, Path: "/noderoot" + privTgt}, + {Device: realDev, Source: privTgt, Path: target}, + {Device: realDev, Source: privTgt, Path: "/noderoot" + target}, + {Device: realDev, Source: privTgt, Path: targetData}, + {Device: realDev, Source: privTgt, Path: "/noderoot" + targetData}, + } + + req := &csi.NodeUnpublishVolumeRequest{ + VolumeId: volumeID, + TargetPath: target, + } + + lastUnmounted, err := unpublishVolume(req, privDir, devicePath, "req-2") + if err != nil { + t.Fatalf("unpublishVolume returned error: %v", err) + } + if !lastUnmounted { + t.Fatalf("expected lastUnmounted=true when no other pod mounts remain") + } + if len(gofsutil.GOFSMockMounts) != 0 { + t.Fatalf("expected all mounts to be unmounted, got %#v", gofsutil.GOFSMockMounts) + } +} + func Test_validateVolumeCapability(t *testing.T) { type args struct { volCap *csi.VolumeCapability diff --git a/service/node.go b/service/node.go index d5bd2dd1..b947e46c 100644 --- a/service/node.go +++ b/service/node.go @@ -1,5 +1,5 @@ /* - Copyright © 2021-2025 Dell Inc. or its subsidiaries. All Rights Reserved. + Copyright © 2021-2026 Dell Inc. or its subsidiaries. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,7 +17,6 @@ package service import ( "context" "fmt" - "io/ioutil" "math/rand" "net" "os" @@ -71,6 +70,8 @@ var ( sysBlock = "/sys/block" // changed for unit testing dev = "/dev/" maxDisconnectRetries = 3 + // Maximum number of re-tries for some Unisphere calls (reduced in unit-tests) + pmaxQueryAttempts = 20 ) type maskingViewTargetInfo struct { @@ -294,6 +295,7 @@ func (s *service) NodeStageVolume( } log.WithFields(f).Infof("NodeStageVolume completed for devicePath: %s", devicePath) + return &csi.NodeStageVolumeResponse{}, nil } @@ -523,16 +525,25 @@ func (s *service) NodeUnstageVolume( return nil, status.Error(codes.InvalidArgument, err.Error()) } - if err := s.disconnectVolume(reqID, symID, devID, volumeWWN); err != nil { - return nil, err - } - - // Remove the mount private directory if present, and the directory + // Remove the mount private directory before disconnecting volume + // This ensures the device is not in use when multipath flush is attempted privTgt := getPrivateMountPoint(s.privDir, id) + if err := gofsutil.Unmount(context.Background(), privTgt); err != nil { + log.Infof("Unmount of private target %s: %s (may already be unmounted)", privTgt, err.Error()) + } + // Unmount /noderoot variant for bind mount environments + if err := gofsutil.Unmount(context.Background(), "/noderoot"+privTgt); err != nil { + log.Debugf("Unmount of /noderoot variant: %s", err.Error()) + } err = removeWithRetry(privTgt) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } + + if err := s.disconnectVolume(reqID, symID, devID, volumeWWN); err != nil { + return nil, err + } + s.removeWWNFile(id) if s.opts.IsVsphereEnabled { @@ -796,7 +807,12 @@ func (s *service) NodePublishVolume( } log.WithFields(f).Info("Calling publishVolume") - if err := publishVolume(req, s.privDir, symlinkPath, reqID); err != nil { + fsCfg := &fsCheckConfig{ + enabled: s.opts.FsCheckEnabled, + mode: s.opts.FsCheckMode, + k8sUtils: s.k8sUtils, + } + if err := publishVolume(req, s.privDir, symlinkPath, reqID, fsCfg); err != nil { return nil, err } return &csi.NodePublishVolumeResponse{}, nil @@ -1639,18 +1655,21 @@ func (s *service) nodeStartup(ctx context.Context) error { s.useNFS = true return nil } + + err = s.nodeHostSetup(ctx, portWWNs, IQNs, hostNQN, s.opts.ManagedArrays) + if err != nil { + return err + } // #nosec G20 } else { err := s.setVMHost() if err != nil { return err } log.Debug("vmHost created successfully") + s.useFC = true + s.nodeIsInitialized = true } - err = s.nodeHostSetup(ctx, portWWNs, IQNs, hostNQN, s.opts.ManagedArrays) - if err != nil { - return err - } // #nosec G20 go s.startAPIService(ctx) return err } @@ -1857,6 +1876,7 @@ func (s *service) nodeHostSetup(ctx context.Context, portWWNs []string, IQNs []s } nodeChroot, _ := csictx.LookupEnv(context.Background(), EnvNodeChroot) + if s.useNVMeTCP { // check nvme module availability on the host err = s.setupArrayForNVMeTCP(ctx, symID, validNVMeTCPs, pmaxClient) @@ -1869,8 +1889,8 @@ func (s *service) nodeHostSetup(ctx context.Context, portWWNs []string, IQNs []s s.useFC = false s.useIscsi = false } - } + if s.useFC { formattedFCs := make([]string, 0) for _, initiatorID := range validFCs { @@ -1888,8 +1908,8 @@ func (s *service) nodeHostSetup(ctx context.Context, portWWNs []string, IQNs []s s.useNVMeTCP = false s.useIscsi = false } - } + if s.useIscsi { err := s.ensureISCSIDaemonStarted() if err != nil { @@ -1905,11 +1925,15 @@ func (s *service) nodeHostSetup(ctx context.Context, portWWNs []string, IQNs []s s.useNVMeTCP = false s.useNFS = false } - } } + // Even if all block protocols failed, we mark the node + // as initialized so that the node can be used for NFS mounts. + // If this behavor will need to change, this flag would be removed from here + // and instead would be set in each of the protocol setup blocks above on success. s.nodeIsInitialized = true + return nil } @@ -1954,69 +1978,89 @@ func (s *service) setupArrayForIscsi(ctx context.Context, array string, IQNs []s return nil } -// setupArrayForIscsi is called to set up a node for iscsi operation. +// setupArrayForNVMeTCP is called to set up a node for NVMe operation. func (s *service) setupArrayForNVMeTCP(ctx context.Context, array string, NQNs []string, pmaxClient pmax.Pmax) error { hostName, _, mvName := s.GetNVMETCPHostSGAndMVIDFromNodeID(s.opts.NodeName) log := log.WithContext(ctx) log.Infof("setting up array %s for NVMeTCP, host name: %s masking view ID: %s %v", array, hostName, mvName, NQNs) - // Discover targets on the host - err := s.setupNVMeTCPTargetDiscovery(ctx, array, pmaxClient) + nvmeHostID, err := s.nvmetcpClient.GetHostID() if err != nil { - log.Error(err.Error()) - return err + return fmt.Errorf("failed to get local NVMe host ID: %v", err) } - updatesHostNQNs, err := s.updateNQNWithHostID(ctx, array, NQNs, pmaxClient) - if err != nil || updatesHostNQNs == nil { - return fmt.Errorf(" Error updating NQN with HostID, len of updatedNQN: %d", len(updatesHostNQNs)) + + initiatorIDs, err := makeNVMeInitiatorIDs(NQNs, nvmeHostID) + if err != nil { + return fmt.Errorf("failed to make NVMe initiator IDs (hostNQN:hostID): %v", err) } + log.Infof("Using NVMe initiator IDs (hostNQN:hostID): %v", initiatorIDs) - // Create or update the NVMe Host and Initiators dummy - _, err = s.createOrUpdateNVMeTCPHost(ctx, array, hostName, updatesHostNQNs, pmaxClient) + // Create or update the NVMe Host with initiator reference + _, err = s.createOrUpdateNVMeTCPHost(ctx, array, hostName, initiatorIDs, pmaxClient) if err != nil { log.Error(err.Error()) return err } - // Create or update the NVMe Host and Initiators - _, err = s.getAndConfigureMaskingViewTargetsNVMeTCP(ctx, array, mvName, pmaxClient) - if err != nil && !(strings.Contains(err.Error(), "Masking View") && strings.Contains(err.Error(), "cannot be found")) { - log.Warn(err.Error()) + // Discover targets on the host and connect initiators + err = s.setupNVMeTCPTargetDiscovery(ctx, array, pmaxClient) + if err != nil { + log.Error(err.Error()) return err } + + // It may take some time for the array to start reporting the newly logged in initiators. + // To accommodate this, retry a few times before erroring out. + + // Wait for the NQNs to appear as in the array initiators list + for attempt := 1; attempt <= pmaxQueryAttempts; attempt++ { + if attempt > 1 { // First attempt does not need to wait + // Sleep 10 seconds or until context is closed + select { + case <-ctx.Done(): + return fmt.Errorf("failed to validate NVMe initiators on array: context timeout") + case <-time.After(10 * time.Second): + } + } + log.Infof("Attempting to validate NVMe initiators on array (%d)", attempt) + err = s.validateNVMeInitiators(ctx, array, initiatorIDs, pmaxClient) + if err == nil { + break // Success + } + log.Errorf("Not all NVMe initiators were found on array %s: %v", array, err) + } + return nil } -func (s *service) updateNQNWithHostID(ctx context.Context, symID string, NQNs []string, pmaxClient pmax.Pmax) ([]string, error) { +func (s *service) validateNVMeInitiators(ctx context.Context, symID string, initiatorIDs []string, pmaxClient pmax.Pmax) error { log := log.WithContext(ctx) - updatesHostNQNs := make([]string, 0) - // Process the NQN to append hostId - hostInitiators, err := pmaxClient.GetInitiatorList(ctx, symID, "", false, false) - if err != nil { - log.Error("Failed to fetch initiator list for the SYM :" + symID) - return nil, err - } - log.Infof("Host Initiators: %+v", hostInitiators) - for _, hostInitiator := range hostInitiators.InitiatorIDs { - // hostInitiator = OR-1C:001:nqn.2014-08.org.nvmexpress:uuid:csi_master:76B04D56EAB26A2E1509A7E98D3DFDB6 - for _, nqn := range NQNs { - // nqn = [nqn.2014-08.org.nvmexpress:uuid:csi_master] - if strings.Contains(hostInitiator, nqn) { - initiator, err := pmaxClient.GetInitiatorByID(ctx, symID, hostInitiator) - if err != nil { - log.Errorf("Failed to fetch InitiatorID details for the initiator %s", hostInitiator) - } else { - hostID := initiator.HostID - nqn = nqn + ":" + hostID - log.Infof("updated host nqn is: %s", nqn) - } - updatesHostNQNs = append(updatesHostNQNs, nqn) + // Get all initiators from the array and search for our local NQNs + allInitiators, err := pmaxClient.GetInitiatorList(ctx, symID, "", false, false) + if err != nil || allInitiators == nil { + return fmt.Errorf("failed to get all initiators: %v", err) + } + log.Debugf("All initiators on array: %v", allInitiators.InitiatorIDs) + + for _, localID := range initiatorIDs { + // localID = [nqn.2014-08.org.nvmexpress:uuid:27212f42-27d7-3250-5c80-b02adbbb66b5:76B04D56EAB26A2E1509A7E98D3DFDB6] + nqnFound := false + for _, initiatorID := range allInitiators.InitiatorIDs { + // initiatorID = OR-1C:001:nqn.2014-08.org.nvmexpress:uuid:27212f42-27d7-3250-5c80-b02adbbb66b5:76B04D56EAB26A2E1509A7E98D3DFDB6 + if strings.HasSuffix(initiatorID, localID) { + nqnFound = true + break } } + if !nqnFound { + return fmt.Errorf("Initiator %s not found", localID) + } } - log.Infof("Updated NQNs are: %+v", updatesHostNQNs) - return updatesHostNQNs, nil + + log.Infof("All local NVMe initiators are registered on the array") + + return nil } // getAndConfigureMaskingViewTargets - Returns a list of ISCSITargets for a given masking view @@ -2564,7 +2608,7 @@ func (s *service) createOrUpdateFCHost(ctx context.Context, array string, nodeNa log.Infof("Array %s FC Host %s does not exist. Creating it.", array, nodeName) host, err = s.retryableCreateHost(ctx, array, nodeName, hostInitiators, nil, pmaxClient) if err != nil { - return host, err + return nil, err } } else { // make sure we don't update an iscsi host @@ -2575,7 +2619,7 @@ func (s *service) createOrUpdateFCHost(ctx context.Context, array string, nodeNa log.Infof("updating host: %s initiators to: %s", nodeName, hostInitiators) _, err := s.retryableUpdateHostInitiators(ctx, array, host, portWWNs, pmaxClient) if err != nil { - return host, err + return nil, err } } } @@ -2586,13 +2630,13 @@ func (s *service) createOrUpdateIscsiHost(ctx context.Context, array string, nod log := log.WithContext(ctx) log.Debug(fmt.Sprintf("Processing Iscsi Host array: %s, nodeName: %s, initiators: %v", array, nodeName, IQNs)) if array == "" { - return &types.Host{}, fmt.Errorf("createOrUpdateHost: No array specified") + return nil, fmt.Errorf("createOrUpdateHost: No array specified") } if nodeName == "" { - return &types.Host{}, fmt.Errorf("createOrUpdateHost: No nodeName specified") + return nil, fmt.Errorf("createOrUpdateHost: No nodeName specified") } if len(IQNs) == 0 { - return &types.Host{}, fmt.Errorf("createOrUpdateHost: No IQNs specified") + return nil, fmt.Errorf("createOrUpdateHost: No IQNs specified") } host, err := pmaxClient.GetHostByID(ctx, array, nodeName) @@ -2602,7 +2646,7 @@ func (s *service) createOrUpdateIscsiHost(ctx context.Context, array string, nod log.Infof("ISCSI Host %s does not exist. Creating it.", nodeName) host, err = s.retryableCreateHost(ctx, array, nodeName, IQNs, nil, pmaxClient) if err != nil { - return &types.Host{}, fmt.Errorf("Unable to create Host: %v", err) + return nil, fmt.Errorf("Unable to create Host: %v", err) } } else { // Make sure we don't update an FC host @@ -2611,7 +2655,7 @@ func (s *service) createOrUpdateIscsiHost(ctx context.Context, array string, nod if len(hostInitiators) != 0 && !stringSlicesEqual(hostInitiators, IQNs) { log.Infof("updating host: %s initiators to: %s", nodeName, IQNs) if _, err := s.retryableUpdateHostInitiators(ctx, array, host, IQNs, pmaxClient); err != nil { - return host, err + return nil, err } } } @@ -2621,14 +2665,15 @@ func (s *service) createOrUpdateIscsiHost(ctx context.Context, array string, nod func (s *service) createOrUpdateNVMeTCPHost(ctx context.Context, array string, nodeName string, NQNs []string, pmaxClient pmax.Pmax) (*types.Host, error) { log := log.WithContext(ctx) log.Debug(fmt.Sprintf("Processing NVMeTCP Host array: %s, nodeName: %s, initiators: %v", array, nodeName, NQNs)) + if array == "" { - return &types.Host{}, fmt.Errorf("createOrUpdateHost: No array specified") + return nil, fmt.Errorf("createOrUpdateHost: No array specified") } if nodeName == "" { - return &types.Host{}, fmt.Errorf("createOrUpdateHost: No nodeName specified") + return nil, fmt.Errorf("createOrUpdateHost: No nodeName specified") } if len(NQNs) == 0 { - return &types.Host{}, fmt.Errorf("createOrUpdateHost: No NQNs specified") + return nil, fmt.Errorf("createOrUpdateHost: No NQNs specified") } // process the NQNs @@ -2640,7 +2685,7 @@ func (s *service) createOrUpdateNVMeTCPHost(ctx context.Context, array string, n log.Infof("NVMe Host %s does not exist. Creating it.", nodeName) host, err = s.retryableCreateHost(ctx, array, nodeName, NQNs, nil, pmaxClient) if err != nil { - return &types.Host{}, fmt.Errorf("Unable to create Host: %v", err) + return nil, fmt.Errorf("unable to create host: %v", err) } } else { // Make sure we fetch only the NVMe hosts @@ -2649,40 +2694,57 @@ func (s *service) createOrUpdateNVMeTCPHost(ctx context.Context, array string, n if len(hostInitiators) != 0 && !stringSlicesEqual(hostInitiators, NQNs) { log.Infof("updating host: %s initiators to: %s", nodeName, NQNs) if _, err := s.retryableUpdateHostInitiators(ctx, array, host, NQNs, pmaxClient); err != nil { - return host, err + return nil, err } } } return host, nil } +func makeNVMeInitiatorIDs(NQNs []string, nvmeHostID string) ([]string, error) { + // Normalize the local NVMe host ID to match the format used by the array for initiator hostID. + // Example: c32abcdf-35f9-4800-88ad-396225c90b70 -> C32ABCDF35F9480088AD396225C90B70 + nvmeHostID = strings.ReplaceAll(nvmeHostID, "-", "") + nvmeHostID = strings.ToUpper(nvmeHostID) + + initiatorIDs := make([]string, len(NQNs)) + + // NVMe initiator ID format used in PowerMax API Host object has to include the NVMe host identity + for i, nqn := range NQNs { + if !strings.HasSuffix(nqn, ":"+nvmeHostID) { + initiatorIDs[i] = nqn + ":" + nvmeHostID + } else { + initiatorIDs[i] = nqn + } + } + + return initiatorIDs, nil +} + // retryableCreateHost func (s *service) retryableCreateHost(ctx context.Context, array string, nodeName string, hostInitiators []string, _ *types.HostFlags, pmaxClient pmax.Pmax) (*types.Host, error) { - log := log.WithContext(ctx) var err error var host *types.Host - deadline := time.Now().Add(time.Duration(s.GetPmaxTimeoutSeconds()) * time.Second) - for tries := 0; time.Now().Before(deadline); tries++ { - host, err = pmaxClient.CreateHost(ctx, array, nodeName, hostInitiators, nil) - if err != nil { - // Retry on this error - if strings.Contains(err.Error(), "is not in the format of a valid NQN:HostID") { - hostInitiators, err = s.updateNQNWithHostID(ctx, array, hostInitiators, pmaxClient) - if err != nil { - log.Debug(fmt.Sprintf("failed to update host nqn; retrying...")) - } + + // Retry up to pmaxQueryAttempts times to create host on array + for attempt := 1; attempt <= pmaxQueryAttempts; attempt++ { + if attempt > 1 { // First attempt does not need to wait + // Sleep 5 seconds or until context is closed + select { + case <-ctx.Done(): + return nil, fmt.Errorf("failed to create host on array: context timeout") + case <-time.After(5 * time.Second): } - log.Debug(fmt.Sprintf("failed to create Host; retrying...")) - // #nosec G115 - time.Sleep(time.Second << uint(tries)) // incremental back-off - continue } - break - } - if err != nil { - return &types.Host{}, fmt.Errorf("Unable to create Host: %s", err) + log.Infof("Attempting to create host %s with initiators %v on array %s (%d)", + nodeName, hostInitiators, array, attempt) + host, err = pmaxClient.CreateHost(ctx, array, nodeName, hostInitiators, nil) + if err == nil { + return host, nil + } + log.Errorf("Failed to create host on array: %v", err) } - return host, nil + return nil, fmt.Errorf("failed to create host on array after %d attempts", pmaxQueryAttempts) } // retryableUpdateHostInitiators wraps UpdateHostInitiators in a retry loop @@ -2690,23 +2752,26 @@ func (s *service) retryableUpdateHostInitiators(ctx context.Context, array strin log := log.WithContext(ctx) var err error var updatedHost *types.Host - deadline := time.Now().Add(time.Duration(s.GetPmaxTimeoutSeconds()) * time.Second) - for tries := 0; time.Now().Before(deadline); tries++ { + + // Retry pmaxQueryAttempts times to update host on array + for attempt := 1; attempt <= pmaxQueryAttempts; attempt++ { + if attempt > 1 { // First attempt does not need to wait + // Sleep 5 seconds or until context is closed + select { + case <-ctx.Done(): + return nil, fmt.Errorf("failed to update host initiators on array: context timeout") + case <-time.After(5 * time.Second): + } + } + log.Infof("Attempting to update host %s with initiators %v on array %s (%d)", + host.HostID, initiators, array, attempt) updatedHost, err = pmaxClient.UpdateHostInitiators(ctx, array, host, initiators) - if err != nil { - // Retry on this error - log.Debug(fmt.Sprintf("failed to update Host; retrying...")) - // #nosec G115 - time.Sleep(time.Second << uint(tries)) // incremental back-off - continue + if err == nil { + return updatedHost, nil } - host = updatedHost - break - } - if err != nil { - return &types.Host{}, fmt.Errorf("Unable to update Host: %v", err) + log.Errorf("Failed to update host initiators on array: %v", err) } - return updatedHost, nil + return nil, fmt.Errorf("failed to update host initiators on array after %d attempts", pmaxQueryAttempts) } // retryableGetSymmetrixIDList returns the list of arrays @@ -3232,7 +3297,7 @@ func (s *service) getAndConfigureArrayISCSITargets(ctx context.Context, arrayTar // writeWWNFile writes a volume's WWN to a file copy on the node func (s *service) writeWWNFile(id, volumeWWN string) error { wwnFileName := fmt.Sprintf("%s/%s.wwn", s.privDir, id) - err := ioutil.WriteFile(wwnFileName, []byte(volumeWWN), 0o644) // #nosec G306 + err := os.WriteFile(wwnFileName, []byte(volumeWWN), 0o644) // #nosec G306 if err != nil { return status.Errorf(codes.Internal, "Could not write WWN file %s: %v", wwnFileName, err) } @@ -3243,7 +3308,7 @@ func (s *service) writeWWNFile(id, volumeWWN string) error { func (s *service) readWWNFile(id string) (string, error) { // READ volume WWN wwnFileName := fmt.Sprintf("%s/%s.wwn", s.privDir, id) - wwnBytes, err := ioutil.ReadFile(wwnFileName) // #nosec G304 + wwnBytes, err := os.ReadFile(wwnFileName) // #nosec G304 if err != nil { return "", status.Errorf(codes.Internal, "Could not read WWN file %s: %v", wwnFileName, err) } diff --git a/service/node_test.go b/service/node_test.go index dd6aae54..8657c19d 100644 --- a/service/node_test.go +++ b/service/node_test.go @@ -32,13 +32,14 @@ import ( "github.com/dell/gofsutil" "github.com/dell/goiscsi" "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" gonvme "github.com/dell/gonvme" pmax "github.com/dell/gopowermax/v2" types "github.com/dell/gopowermax/v2/types/v100" "github.com/golang/mock/gomock" gmock "github.com/golang/mock/gomock" - "github.com/stretchr/testify/require" ) func TestGetNVMeTCPTargets(t *testing.T) { @@ -863,6 +864,64 @@ func TestConnectRDMDevice(t *testing.T) { } } +// TestConnectDevice_Vsphere verifies connectDevice routing when vSphere is +// enabled. With useFC=true (set by nodeStartup) the call must reach +// connectRDMDevice; without it the call falls through to iSCSI and fails. +func TestConnectDevice_Vsphere(t *testing.T) { + tests := []struct { + name string + useFC bool + wantErr bool + wantInDev bool // expect devicePath to contain "/dev/" + }{ + { + name: "useFC routes to RDM", + useFC: true, + wantErr: false, + wantInDev: true, + }, + { + name: "without useFC falls through to iSCSI and fails", + useFC: false, + wantErr: true, + }, + } + + data := publishContextData{ + deviceWWN: "mockWWN", + volumeLUNAddress: "10", + fcTargets: []FCTargetInfo{ + {WWPN: "mockWWPN"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &service{ + opts: Opts{ + IsVsphereEnabled: true, + }, + useFC: tt.useFC, + } + if tt.useFC { + s.fcConnector = &mockFCGobrick{} + } else { + s.iscsiConnector = &mockISCSIGobrick{} + } + + devicePath, err := s.connectDevice(context.Background(), data) + if tt.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + if tt.wantInDev { + require.Contains(t, devicePath, "/dev/") + } + }) + } +} + func TestGetHostForVsphere(t *testing.T) { ctx := context.Background() vsphereHostName := "vsphere-host" @@ -1023,14 +1082,10 @@ func TestCreateOrUpdateNVMeTCPHost(t *testing.T) { c := mocks.NewMockPmaxClient(gmock.NewController(t)) c.EXPECT().GetHostByID(gmock.All(), "array1", "host1").AnyTimes().Return(nil, errors.New("host not found")) c.EXPECT().CreateHost(gmock.All(), "array1", "host1", gmock.Any(), gmock.Any()).AnyTimes().Return(nil, errors.New("create host failed")) - c.EXPECT().GetInitiatorList(gmock.All(), "array1", "", false, false).AnyTimes().Return(&types.InitiatorList{}, nil) - c.EXPECT().GetInitiatorByID(gmock.All(), "array1", "nqn.1988-11.com.dell.mock:e6e2d5b871f1403E169D00001").AnyTimes().Return( - &types.Initiator{InitiatorID: "nqn.1988-11.com.dell.mock:e6e2d5b871f1403E169D00001"}, nil) - c.EXPECT().UpdateHostInitiators(gmock.All(), "array1", "host1", gmock.Any()).AnyTimes().Return(&types.Host{HostID: "host1"}, nil) return c }, - wantErr: false, - want: &types.Host{}, + wantErr: true, + want: nil, }, { // This is really bizarre condition in the code to test, but it is what it is. @@ -1053,7 +1108,7 @@ func TestCreateOrUpdateNVMeTCPHost(t *testing.T) { c.EXPECT().UpdateHostInitiators(gmock.All(), "array1", "host1", gmock.Any()).AnyTimes().Return(&types.Host{HostID: "host1"}, nil) return c }, - wantErr: false, + wantErr: true, want: nil, }, { @@ -1066,7 +1121,7 @@ func TestCreateOrUpdateNVMeTCPHost(t *testing.T) { return c }, wantErr: true, - want: &types.Host{}, + want: nil, }, { name: "nodename empty case", @@ -1078,7 +1133,7 @@ func TestCreateOrUpdateNVMeTCPHost(t *testing.T) { return c }, wantErr: true, - want: &types.Host{}, + want: nil, }, { name: "len NQNs zero case", @@ -1090,7 +1145,7 @@ func TestCreateOrUpdateNVMeTCPHost(t *testing.T) { return c }, wantErr: true, - want: &types.Host{}, + want: nil, }, { name: "Host exists, add new initiators", @@ -1125,6 +1180,9 @@ func TestCreateOrUpdateNVMeTCPHost(t *testing.T) { for _, tc := range testCases { tc.pmaxClient = tc.getClient() t.Run(tc.name, func(t *testing.T) { + // Disable re-tries for tests + pmaxQueryAttempts = 1 + defer func() { pmaxQueryAttempts = 30 }() s := &service{ opts: Opts{ UseProxy: true, @@ -1132,11 +1190,13 @@ func TestCreateOrUpdateNVMeTCPHost(t *testing.T) { nvmetcpClient: gonvme.NewMockNVMe(map[string]string{}), nvmeTargets: &sync.Map{}, loggedInNVMeArrays: map[string]bool{}, - pmaxTimeoutSeconds: 1, } got, err := s.createOrUpdateNVMeTCPHost(context.Background(), tc.array, tc.nodeName, tc.NQNs, tc.pmaxClient) - if err == nil && tc.wantErr { - t.Errorf("Expected: %v, but got no error", tc.wantErr) + if tc.wantErr && err == nil { + t.Errorf("Expected error, but got nil") + } + if !tc.wantErr && err != nil { + t.Errorf("Expected no error, but got: %v", err) } if !reflect.DeepEqual(got, tc.want) { t.Errorf("Expected: %v, but got: %v", tc.want, got) @@ -1332,6 +1392,207 @@ func TestPerformNVMETCPLoginOnSymID(t *testing.T) { } } +func TestSetupArrayForNVMeTCP(t *testing.T) { + testCases := []struct { + name string + NQNs []string + initFunc func() + setupClient func(c *mocks.MockPmaxClient) + wantErr bool + wantContains string + }{ + { + name: "GetHostID fails", + NQNs: []string{"nqn.test:001"}, + initFunc: func() { + gonvme.GONVMEMock.InduceInitiatorError = true + }, + setupClient: func(_ *mocks.MockPmaxClient) {}, + wantErr: true, + wantContains: "failed to get local NVMe host ID", + }, + { + name: "setupNVMeTCPTargetDiscovery fails", + NQNs: []string{"nqn.test:001"}, + initFunc: func() { + gonvme.GONVMEMock.InduceInitiatorError = false + getIPInterfaces = func(_ context.Context, _ string, _ []string, _ pmax.Pmax) (map[string]int32, error) { + return nil, fmt.Errorf("IP interface fetch error") + } + }, + setupClient: func(c *mocks.MockPmaxClient) { + c.EXPECT().GetHostByID(gomock.Any(), "array1", gomock.Any()).Return(nil, errors.New("host not found")) + c.EXPECT().CreateHost(gomock.Any(), "array1", gomock.Any(), gomock.Any(), gomock.Any()).Return(&types.Host{HostID: "testhost"}, nil) + }, + wantErr: true, + }, + { + name: "validateNVMeInitiators succeeds on first attempt", + NQNs: []string{"nqn.test:001"}, + initFunc: func() { + gonvme.GONVMEMock.InduceInitiatorError = false + gonvme.GONVMEMock.InduceDiscoveryError = false + getIPInterfaces = func(_ context.Context, _ string, _ []string, _ pmax.Pmax) (map[string]int32, error) { + return map[string]int32{"1.2.3.4": 4420}, nil + } + }, + setupClient: func(c *mocks.MockPmaxClient) { + c.EXPECT().GetHostByID(gomock.Any(), "array1", gomock.Any()).Return(nil, errors.New("host not found")) + c.EXPECT().CreateHost(gomock.Any(), "array1", gomock.Any(), gomock.Any(), gomock.Any()).Return(&types.Host{HostID: "testhost"}, nil) + c.EXPECT().GetInitiatorList(gomock.Any(), "array1", "", false, false).Return(&types.InitiatorList{ + InitiatorIDs: []string{"nqn.test:001:A2D57D74A1984E6BAA7897AF9CD00F31"}, + }, nil) + }, + wantErr: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + origIPInterfaces := getIPInterfaces + origQueryAttempts := pmaxQueryAttempts + pmaxQueryAttempts = 1 + defer func() { + gonvme.GONVMEMock.InduceInitiatorError = false + gonvme.GONVMEMock.InduceDiscoveryError = false + getIPInterfaces = origIPInterfaces + pmaxQueryAttempts = origQueryAttempts + }() + + tc.initFunc() + + ctrl := gomock.NewController(t) + c := mocks.NewMockPmaxClient(ctrl) + tc.setupClient(c) + + svc := &service{ + opts: Opts{ + NodeName: "node1", + }, + nvmetcpClient: gonvme.NewMockNVMe(map[string]string{}), + } + + err := svc.setupArrayForNVMeTCP(context.Background(), "array1", tc.NQNs, c) + if tc.wantErr { + assert.Error(t, err) + if tc.wantContains != "" { + assert.Contains(t, err.Error(), tc.wantContains) + } + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestSetupNVMeTCPTargetDiscovery_EmptyIPInterfaces(t *testing.T) { + orig := getIPInterfaces + getIPInterfaces = func(_ context.Context, _ string, _ []string, _ pmax.Pmax) (map[string]int32, error) { + return map[string]int32{}, nil + } + defer func() { getIPInterfaces = orig }() + + svc := &service{ + opts: Opts{PortGroups: []string{"pg1"}}, + nvmetcpClient: gonvme.NewMockNVMe(map[string]string{}), + } + ctrl := gomock.NewController(t) + c := mocks.NewMockPmaxClient(ctrl) + + err := svc.setupNVMeTCPTargetDiscovery(context.Background(), "array1", c) + assert.Error(t, err) + assert.Contains(t, err.Error(), "couldn't find any IP interfaces") +} + +func TestRetryableUpdateHostInitiators(t *testing.T) { + testCases := []struct { + name string + attempts int + ctxFunc func() context.Context + wantContains string + }{ + { + name: "all attempts fail", + attempts: 1, + ctxFunc: func() context.Context { return context.Background() }, + wantContains: "failed to update host initiators on array after", + }, + { + name: "context cancelled on second attempt", + attempts: 2, + ctxFunc: func() context.Context { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + return ctx + }, + wantContains: "context timeout", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + orig := pmaxQueryAttempts + pmaxQueryAttempts = tc.attempts + defer func() { pmaxQueryAttempts = orig }() + + ctrl := gomock.NewController(t) + c := mocks.NewMockPmaxClient(ctrl) + host := &types.Host{HostID: "host1"} + c.EXPECT().UpdateHostInitiators(gomock.Any(), "array1", host, gomock.Any()).Return(nil, errors.New("update failed")) + + svc := &service{} + result, err := svc.retryableUpdateHostInitiators(tc.ctxFunc(), "array1", host, []string{"nqn.test:001"}, c) + assert.Nil(t, result) + assert.Error(t, err) + assert.Contains(t, err.Error(), tc.wantContains) + }) + } +} + +func TestRetryableCreateHost(t *testing.T) { + testCases := []struct { + name string + attempts int + ctxFunc func() context.Context + wantContains string + }{ + { + name: "all attempts fail", + attempts: 1, + ctxFunc: func() context.Context { return context.Background() }, + wantContains: "failed to create host on array after", + }, + { + name: "context cancelled on second attempt", + attempts: 2, + ctxFunc: func() context.Context { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + return ctx + }, + wantContains: "context timeout", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + orig := pmaxQueryAttempts + pmaxQueryAttempts = tc.attempts + defer func() { pmaxQueryAttempts = orig }() + + ctrl := gomock.NewController(t) + c := mocks.NewMockPmaxClient(ctrl) + c.EXPECT().CreateHost(gomock.Any(), "array1", "node1", gomock.Any(), gomock.Any()).Return(nil, errors.New("create failed")) + + svc := &service{} + result, err := svc.retryableCreateHost(tc.ctxFunc(), "array1", "node1", []string{"nqn.test:001"}, nil, c) + assert.Nil(t, result) + assert.Error(t, err) + assert.Contains(t, err.Error(), tc.wantContains) + }) + } +} + func TestCreateTopologyMap(t *testing.T) { var listener net.Listener testCases := []struct { @@ -2745,3 +3006,176 @@ func (c *iscsiClientMock) DiscoverTargets(portal string, _ bool) ([]goiscsi.ISCS } return []goiscsi.ISCSITarget{}, nil } + +func TestValidateNVMeInitiators(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + tests := []struct { + name string + initiatorIDs []string + arrayList []string + listError bool + expectedError bool + errContains string + description string + }{ + { + name: "All initiators found on array", + initiatorIDs: []string{"nqn.2014-08.org.nvmexpress:uuid:test:ABCD1234"}, + arrayList: []string{"FA-1E:1:nqn.2014-08.org.nvmexpress:uuid:test:ABCD1234"}, + expectedError: false, + description: "Single initiator found on array", + }, + { + name: "Multiple initiators all found", + initiatorIDs: []string{ + "nqn.2014-08.org.nvmexpress:uuid:test1:ABCD1234", + "nqn.2014-08.org.nvmexpress:uuid:test2:ABCD1234", + }, + arrayList: []string{ + "FA-1E:1:nqn.2014-08.org.nvmexpress:uuid:test1:ABCD1234", + "FA-1E:2:nqn.2014-08.org.nvmexpress:uuid:test2:ABCD1234", + }, + expectedError: false, + description: "Multiple initiators all found on array", + }, + { + name: "Initiator not found on array", + initiatorIDs: []string{"nqn.2014-08.org.nvmexpress:uuid:test:ABCD1234"}, + arrayList: []string{"FA-1E:1:nqn.2014-08.org.nvmexpress:uuid:other:FFFF0000"}, + expectedError: true, + errContains: "not found", + description: "Initiator not present in array list", + }, + { + name: "Empty array initiator list", + initiatorIDs: []string{"nqn.2014-08.org.nvmexpress:uuid:test:ABCD1234"}, + arrayList: []string{}, + expectedError: true, + errContains: "not found", + description: "No initiators on array yet", + }, + { + name: "GetInitiatorList error", + initiatorIDs: []string{"nqn.2014-08.org.nvmexpress:uuid:test:ABCD1234"}, + listError: true, + expectedError: true, + errContains: "failed to get all initiators", + description: "Error getting initiator list from array", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + mockClient := mocks.NewMockPmaxClient(ctrl) + svc := &service{ + cacheMutex: sync.Mutex{}, + loggedInNVMeArrays: make(map[string]bool), + } + + if tc.listError { + mockClient.EXPECT().GetInitiatorList(gomock.Any(), "000197900046", "", false, false). + Return(nil, errors.New("simulated error")).Times(1) + } else { + mockClient.EXPECT().GetInitiatorList(gomock.Any(), "000197900046", "", false, false). + Return(&types.InitiatorList{InitiatorIDs: tc.arrayList}, nil).Times(1) + } + + err := svc.validateNVMeInitiators(context.Background(), "000197900046", tc.initiatorIDs, mockClient) + + if tc.expectedError { + assert.Error(t, err, tc.description) + assert.Contains(t, err.Error(), tc.errContains, tc.description) + } else { + assert.NoError(t, err, tc.description) + } + }) + } +} + +func TestMakeNVMeInitiatorIDs(t *testing.T) { + tests := []struct { + name string + nqns []string + hostID string + expected []string + shouldFail bool + }{ + { + name: "Single NQN with UUID host ID", + nqns: []string{"nqn.2014-08.org.nvmexpress:uuid:27212f42-27d7-3250-5c80-b02adbbb66b5"}, + hostID: "c32abcdf-35f9-4800-88ad-396225c90b70", + expected: []string{"nqn.2014-08.org.nvmexpress:uuid:27212f42-27d7-3250-5c80-b02adbbb66b5:C32ABCDF35F9480088AD396225C90B70"}, + }, + { + name: "Multiple NQNs", + nqns: []string{ + "nqn.2014-08.org.nvmexpress:uuid:aaa", + "nqn.2014-08.org.nvmexpress:uuid:bbb", + }, + hostID: "c32abcdf-35f9-4800-88ad-396225c90b70", + expected: []string{ + "nqn.2014-08.org.nvmexpress:uuid:aaa:C32ABCDF35F9480088AD396225C90B70", + "nqn.2014-08.org.nvmexpress:uuid:bbb:C32ABCDF35F9480088AD396225C90B70", + }, + }, + { + name: "NQN already has host ID appended", + nqns: []string{"nqn.2014-08.org.nvmexpress:uuid:test:C32ABCDF35F9480088AD396225C90B70"}, + hostID: "c32abcdf-35f9-4800-88ad-396225c90b70", + expected: []string{"nqn.2014-08.org.nvmexpress:uuid:test:C32ABCDF35F9480088AD396225C90B70"}, + }, + { + name: "Empty NQN list", + nqns: []string{}, + hostID: "c32abcdf-35f9-4800-88ad-396225c90b70", + expected: []string{}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + result, err := makeNVMeInitiatorIDs(tc.nqns, tc.hostID) + assert.NoError(t, err) + assert.Equal(t, tc.expected, result) + }) + } +} + +func TestNVMeInitiatorRetryLogic(t *testing.T) { + // This test validates the retry logic behavior described in the refactored code. + // validateNVMeInitiators is called in a retry loop by setupArrayForNVMeTCP. + + t.Run("Array propagation delay scenario", func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := mocks.NewMockPmaxClient(ctrl) + svc := &service{ + cacheMutex: sync.Mutex{}, + loggedInNVMeArrays: make(map[string]bool), + } + + initiatorIDs := []string{"nqn.2014-08.org.nvmexpress:uuid:test:ABCD1234"} + + // Scenario 1: Empty list (simulating propagation delay — initiator not yet visible) + mockClient.EXPECT().GetInitiatorList(gomock.Any(), "000197900046", "", false, false). + Return(&types.InitiatorList{InitiatorIDs: []string{}}, nil).Times(1) + + err := svc.validateNVMeInitiators(context.Background(), "000197900046", initiatorIDs, mockClient) + + // Should fail because the initiator is not found + assert.Error(t, err) + assert.Contains(t, err.Error(), "not found") + + // Scenario 2: Initiator appears (simulating successful propagation) + mockClient.EXPECT().GetInitiatorList(gomock.Any(), "000197900046", "", false, false). + Return(&types.InitiatorList{InitiatorIDs: []string{"FA-1E:1:nqn.2014-08.org.nvmexpress:uuid:test:ABCD1234"}}, nil).Times(1) + + err = svc.validateNVMeInitiators(context.Background(), "000197900046", initiatorIDs, mockClient) + + // Should succeed now + assert.NoError(t, err) + }) +} diff --git a/service/replication_test.go b/service/replication_test.go index e5f151e5..f0d53353 100644 --- a/service/replication_test.go +++ b/service/replication_test.go @@ -15,6 +15,7 @@ package service import ( + "context" "errors" "fmt" "reflect" @@ -27,7 +28,6 @@ import ( types "github.com/dell/gopowermax/v2/types/v100" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" - "golang.org/x/net/context" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -463,7 +463,8 @@ func TestService_GetOrCreateRDFGroup(t *testing.T) { SymmID: "remote-sym-id", }, }, - }, nil) + }, nil, + ) mockPmaxClient.EXPECT().GetLocalRDFPortDetails(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(&types.RDFPortDetails{}, nil) mockPmaxClient.EXPECT().ExecuteCreateRDFGroup(gomock.Any(), "local-sym-id", gomock.Any()).Return(nil) diff --git a/service/service.go b/service/service.go index 45efd89d..746a9154 100644 --- a/service/service.go +++ b/service/service.go @@ -59,7 +59,6 @@ const ( Name = "csi-powermax.dellemc.com" // Name is the name of the CSI plug-in. ApplicationName = "CSI Driver for Dell EMC PowerMax" // ApplicationName is the name used to register with Powermax REST APIs defaultPrivDir = "/dev/disk/csi-powermax" - defaultPmaxTimeout = 120 defaultLockCleanupDuration = 4 csiPrefix = "csi-" logFields = "logFields" @@ -99,6 +98,7 @@ var ( // Service is the CSI Mock service provider. type Service interface { csi.ControllerServer + csi.GroupControllerServer csi.IdentityServer csi.NodeServer csiext.ReplicationServer @@ -114,7 +114,7 @@ type Opts struct { ProxyServiceHost string ProxyServicePort string User string - Password string // #nosec G117 + Password string `json:"-"` SystemName string NodeName string NodeFullName string @@ -157,6 +157,10 @@ type Opts struct { StorageArrays map[string]StorageArrayConfig dynamicSGEnabled bool sgVolumeLimit int + // FsCheckEnabled enables file system check before mount + FsCheckEnabled bool + // FsCheckMode is the FS check operation mode: "checkOnly" or "checkAndRepair" + FsCheckMode string } // StorageArrayConfig represents the configuration of a storage array in the config file @@ -178,12 +182,17 @@ type TopologyConfig struct { } type service struct { + // satisfies the Service interface and provides unimplemented defaults to functions not implemented + csi.UnimplementedControllerServer + csi.UnimplementedGroupControllerServer + csi.UnimplementedIdentityServer + csi.UnimplementedNodeServer + opts Opts mode string - // amount of time to retry unisphere calls - pmaxTimeoutSeconds int64 // replace this with Unisphere client adminClient pmax.Pmax + adminClient104 pmax.Pmax deletionWorker *deletionWorker iscsiClient goiscsi.ISCSIinterface nvmetcpClient gonvme.NVMEinterface @@ -225,8 +234,11 @@ type service struct { allowedTopologyKeys map[string][]string // map of nodes to allowed topology keys deniedTopologyKeys map[string][]string // map of nodes to denied topology keys - k8sUtils k8sutils.UtilsInterface - snapCleaner *snapCleanupWorker + k8sUtils k8sutils.UtilsInterface + snapCleaner *snapCleanupWorker + spaceReclaimMgr *SpaceReclamationManager + versionCache *versionCache + versionCacheOnce sync.Once } // New returns a new Service. @@ -236,9 +248,9 @@ func New() Service { iscsiTargets: map[string][]string{}, loggedInNVMeArrays: map[string]bool{}, nvmeTargets: new(sync.Map), + versionCache: newVersionCache(), } svc.sgSvc = newStorageGroupService(svc) - svc.pmaxTimeoutSeconds = defaultPmaxTimeout svc.probeStatus = new(sync.Map) return svc } @@ -730,6 +742,35 @@ func (s *service) BeforeServe( } log.Infof("%s set to %v", EnvSGVolumeLimit, sgVolumeLimit) } + + // File system check configuration + if fsCheckEnabled, ok := csictx.LookupEnv(ctx, EnvFsCheckEnabled); ok { + b, err := strconv.ParseBool(fsCheckEnabled) + if err != nil { + log.Warnf("Invalid value %q for %s, defaulting to false", fsCheckEnabled, EnvFsCheckEnabled) + s.opts.FsCheckEnabled = false + } else { + s.opts.FsCheckEnabled = b + } + } + log.Infof("FS check enabled: %v", s.opts.FsCheckEnabled) + + s.opts.FsCheckMode = "checkOnly" // default + if fsCheckMode, ok := csictx.LookupEnv(ctx, EnvFsCheckMode); ok { + switch fsCheckMode { + case "checkOnly", "checkAndRepair": + s.opts.FsCheckMode = fsCheckMode + default: + log.Warnf("Invalid value %q for %s, defaulting to %q", fsCheckMode, EnvFsCheckMode, "checkOnly") + } + } + log.Infof("FS check mode: %s", s.opts.FsCheckMode) + // Initialize space reclamation for node mode + if s.isNode() && s.k8sUtils != nil { + initSpaceReclamation(ctx, s, s.k8sUtils.GetClient()) + log.Infof("Space reclamation initialized") + } + return nil } @@ -855,16 +896,6 @@ func (s *service) getTransportProtocolFromEnv() string { } } -// get the amount of time to retry pmax calls -func (s *service) GetPmaxTimeoutSeconds() int64 { - return s.pmaxTimeoutSeconds -} - -// SetPmaxTimeoutSeconds sets the maximum amount of time to retry pmax calls -func (s *service) SetPmaxTimeoutSeconds(seconds int64) { - s.pmaxTimeoutSeconds = seconds -} - // parseCommaSeperatedList validates and splits a comma seperated list func (s *service) parseCommaSeperatedList(values string) ([]string, error) { results := make([]string, 0) @@ -919,6 +950,34 @@ func (s *service) createPowerMaxClients(ctx context.Context) error { "unable to login to Unisphere: %s", err.Error()) } + // Create a separate 10.4-specific client for CreateVolume/PublishVolume operations + c104, err := pmax.NewClientWithArgs(endPoint, applicationName, s.opts.Insecure, !s.opts.DisableCerts, tlsCertFile) + if err != nil { + log.Warnf("unable to create 10.4 PowerMax client: %s, will use main client", err.Error()) + s.adminClient104 = s.adminClient + } else { + for i := 0; i < maxAuthenticateRetryCount; i++ { + err = c104.Authenticate(ctx, &pmax.ConfigConnect{ + Endpoint: endPoint, + Username: s.opts.User, + Password: s.opts.Password, + Version: "104", + }) + if err == nil { + break + } + log.Infof("Error authenticating 10.4 client: %s", err) + time.Sleep(10 * time.Second) + } + if err != nil { + log.Warnf("unable to authenticate 10.4 client: %s, will use main client", err.Error()) + s.adminClient104 = s.adminClient + } else { + s.adminClient104 = c104 + log.Infof("10.4 client created and authenticated successfully") + } + } + // Filter out a list of locally connected list of arrays, and // initialize the PowerMax client for those array only managedArrays := make([]string, 0, len(s.opts.ManagedArrays)) @@ -975,6 +1034,41 @@ func (s *service) getDriverName() string { return s.opts.DriverName } +func (s *service) isDynamicSGEnabled() bool { + return s.opts.dynamicSGEnabled +} + +func (s *service) getReplicationPrefix() string { + return s.opts.ReplicationPrefix +} + +func (s *service) getReplicationContextPrefix() string { + return s.opts.ReplicationContextPrefix +} + +func (s *service) isSnapshotLicensed(ctx context.Context, symID string, pmaxClient pmax.Pmax) error { + return s.IsSnapshotLicensed(ctx, symID, pmaxClient) +} + +func (s *service) getDynamicSG(ctx context.Context, arrayID, baseSGName string) (string, bool, error) { + return getDynamicSG(ctx, arrayID, baseSGName, s) +} + +func (s *service) getStorageArrayLabels(arrayID string) map[string]string { + if array, ok := s.opts.StorageArrays[arrayID]; ok { + labels := make(map[string]string) + for k, v := range array.Labels { + labels[k] = v.(string) + } + return labels + } + return nil +} + +func (s *service) isBlockEnabled() bool { + return s.opts.EnableBlock +} + func setLogFields(ctx context.Context, fields csmlog.Fields) context.Context { if ctx == nil { ctx = context.Background() diff --git a/service/service_test.go b/service/service_test.go index 783454ff..99dcf5b5 100644 --- a/service/service_test.go +++ b/service/service_test.go @@ -32,8 +32,9 @@ import ( ) var ( - testStatus int - testStartTime time.Time + testStatus int + testStartTime time.Time + lastDeletionWorker *deletionWorker ) func TestMain(m *testing.M) { @@ -56,7 +57,7 @@ func TestGoDog(t *testing.T) { runOptions := godog.Options{ Format: "pretty", Paths: []string{"features"}, - Tags: "v1.0.0, v1.1.0, v1.2.0, v1.3.0, v1.4.0, v1.5.0, v1.6.0, v2.2.0, v2.3.0, v2.4.0, v2.5.0, v2.6.0, v2.7.0, v2.8.0, v2.9.0, v2.11.0, v2.12.0, v2.13.0, v2.14.0, v2.15.0, v2.16.0", + Tags: "v1.0.0, v1.1.0, v1.2.0, v1.3.0, v1.4.0, v1.5.0, v1.6.0, v2.2.0, v2.3.0, v2.4.0, v2.5.0, v2.6.0, v2.7.0, v2.8.0, v2.9.0, v2.11.0, v2.12.0, v2.13.0, v2.14.0, v2.15.0, v2.17.0", // Tags: "wip", // Tags: "resiliency", // uncomment to run all node resiliency related tests, } diff --git a/service/service_unit_test.go b/service/service_unit_test.go index bec25822..34ec9f4c 100644 --- a/service/service_unit_test.go +++ b/service/service_unit_test.go @@ -1,5 +1,5 @@ /* -Copyright © 2021-2025 Dell Inc. or its subsidiaries. All Rights Reserved. +Copyright © 2021-2026 Dell Inc. or its subsidiaries. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -774,14 +774,14 @@ func TestCreateDbusConnection(t *testing.T) { tests := []struct { name string dBusConn *mockDbusConnection - mockdbusNewWithContextFunc func() (*dbus.Conn, error) + mockdbusNewWithContextFunc func() (dBusConn, error) expectedErr error }{ { name: "Successful connection", dBusConn: nil, expectedErr: nil, - mockdbusNewWithContextFunc: func() (*dbus.Conn, error) { + mockdbusNewWithContextFunc: func() (dBusConn, error) { return &dbus.Conn{}, nil }, }, @@ -789,7 +789,7 @@ func TestCreateDbusConnection(t *testing.T) { name: "Error connection", dBusConn: nil, expectedErr: errMockErr, - mockdbusNewWithContextFunc: func() (*dbus.Conn, error) { + mockdbusNewWithContextFunc: func() (dBusConn, error) { return nil, errMockErr }, }, diff --git a/service/snap.go b/service/snap.go index 0d11b97e..d833c97b 100644 --- a/service/snap.go +++ b/service/snap.go @@ -1,5 +1,5 @@ /* - Copyright © 2021-2024 Dell Inc. or its subsidiaries. All Rights Reserved. + Copyright © 2021-2026 Dell Inc. or its subsidiaries. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -627,6 +627,12 @@ func snapCleanupThread(ctx context.Context, scw *snapCleanupWorker, s *service) }*/ for _, symID := range s.opts.ManagedArrays { + select { + case <-ctx.Done(): + log.Infof("snap cleanup worker context canceled before processing %s", symID) + return + default: + } pmaxClient, err := s.GetPowerMaxClient(symID) if err != nil { log.Error(err.Error()) @@ -711,7 +717,12 @@ func snapCleanupThread(ctx context.Context, scw *snapCleanupWorker, s *service) } ReleaseLock(lockHandle, reqID, lockNum) } - time.Sleep(scw.PollingInterval) + select { + case <-ctx.Done(): + log.Infof("snap cleanup worker context canceled while sleeping, exiting") + return + case <-time.After(scw.PollingInterval): + } } } diff --git a/service/space_reclamation.go b/service/space_reclamation.go new file mode 100644 index 00000000..813a8c81 --- /dev/null +++ b/service/space_reclamation.go @@ -0,0 +1,985 @@ +// Copyright © 2024-2026 Dell Inc. or its subsidiaries. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package service + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/dell/gofsutil" + pmax "github.com/dell/gopowermax/v2" + "github.com/robfig/cron/v3" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/tools/record" +) + +// ---- Annotation key constants ---- + +const ( + // LabelPrefix is the prefix for space reclamation PVC labels. + LabelPrefix = "space-reclamation.csi.dell.com/" + // LabelEnabled controls per-PVC opt-in/opt-out via labels. + LabelEnabled = LabelPrefix + "enabled" + // LabelBlockReclaim controls raw block PV reclamation opt-in. + LabelBlockReclaim = LabelPrefix + "block-reclaim" + + // AnnotationPrefix is the prefix for space reclamation PVC annotations (for result metadata). + AnnotationPrefix = "space-reclamation.csi.dell.com/" + // AnnotationLastRunTime records the last reclamation timestamp. + AnnotationLastRunTime = AnnotationPrefix + "last-run-time" + // AnnotationBytesAvailable records bytes available after reclamation. + AnnotationBytesAvailable = AnnotationPrefix + "bytes-available" + // AnnotationDuration records the reclamation duration in seconds. + AnnotationDuration = AnnotationPrefix + "duration-seconds" + // AnnotationStatus records the reclamation status. + AnnotationStatus = AnnotationPrefix + "status" + // AnnotationErrorMsg records any error message. + AnnotationErrorMsg = AnnotationPrefix + "error-message" + // AnnotationNode records the node where reclamation ran. + AnnotationNode = AnnotationPrefix + "node" +) + +// ---- Event reason constants ---- + +const ( + // EventReasonCompleted is the event reason for successful reclamation. + EventReasonCompleted = "SpaceReclamationCompleted" + // EventReasonFailed is the event reason for failed reclamation. + EventReasonFailed = "SpaceReclamationFailed" + // EventReasonTimeout is the event reason for timed-out reclamation. + EventReasonTimeout = "SpaceReclamationTimeout" + // EventReasonUnsupported is the event reason for unsupported devices. + EventReasonUnsupported = "SpaceReclamationUnsupported" +) + +// ---- Volume mode constants ---- + +// VolumeMode distinguishes filesystem from raw block volumes. +type VolumeMode corev1.PersistentVolumeMode + +// Volume mode constants +const ( + VolumeModeFilesystem VolumeMode = VolumeMode(corev1.PersistentVolumeFilesystem) + VolumeModeBlock VolumeMode = VolumeMode(corev1.PersistentVolumeBlock) +) + +// ---- Configuration ---- + +// SpaceReclamationConfig holds configuration for the space reclamation feature. +type SpaceReclamationConfig struct { + // Enabled gates the entire subsystem. + Enabled bool + // Schedule is a cron expression (5-field). Default: "0 2 * * 0". + Schedule string + // MaxConcurrentVolumes is the max parallel reclamation jobs per node. Default: 2. + MaxConcurrentVolumes int + // TimeoutSeconds is the per-volume timeout. Default: 14400. + TimeoutSeconds int + // NodeName is the Kubernetes node name (from downward API or env var). + NodeName string +} + +// getEnvString reads an environment variable and returns a default if unset or empty. +func getEnvString(key, defaultVal string) string { + val := os.Getenv(key) + if val == "" { + return defaultVal + } + return val +} + +// getEnvBool reads an environment variable as a boolean, returning a default on error or empty. +func getEnvBool(key string, defaultVal bool) bool { + val := os.Getenv(key) + if val == "" { + return defaultVal + } + b, err := strconv.ParseBool(val) + if err != nil { + return defaultVal + } + return b +} + +// getEnvInt reads an environment variable as an int, returning a default on error, empty, or negative. +func getEnvInt(key string, defaultVal int) int { + val := os.Getenv(key) + if val == "" { + return defaultVal + } + i, err := strconv.Atoi(val) + if err != nil { + return defaultVal + } + if i < 0 { + return defaultVal + } + return i +} + +// ReadSpaceReclamationConfig reads configuration from environment variables. +func ReadSpaceReclamationConfig() SpaceReclamationConfig { + cfg := SpaceReclamationConfig{ + Enabled: getEnvBool(EnvSpaceReclamationEnabled, false), + Schedule: getEnvString(EnvSpaceReclamationSchedule, "0 2 * * 0"), + MaxConcurrentVolumes: getEnvInt(EnvSpaceReclamationMaxConcurrent, 2), + TimeoutSeconds: getEnvInt(EnvSpaceReclamationTimeout, 14400), + NodeName: getEnvString(EnvNodeName, ""), + } + return cfg +} + +// ---- Volume Info ---- + +// VolumeInfo stores metadata about a staged volume for reclamation. +type VolumeInfo struct { + VolumeID string + StagingPath string // Mount point for filesystem PVs; device path for block PVs + DevicePath string // Underlying block device (e.g., /dev/sda, /dev/dm-0) + VolumeMode VolumeMode // Filesystem or Block + PVCName string + PVCNamespace string + PVC *corev1.PersistentVolumeClaim // PVC object (fetched in RunOnce, reused in reclaimVolume) +} + +// ---- Reclamation Result ---- + +// ReclamationResult represents the outcome of a reclamation operation. +type ReclamationResult struct { + Status string // "success", "error", "timeout", "unsupported" + BytesAvailable int64 + Duration time.Duration + ErrorMessage string // populated on failure + NodeName string +} + +// ---- PVC Annotator ---- + +// PVCAnnotator updates PVC annotations with reclamation results. +type PVCAnnotator struct { + client kubernetes.Interface + maxRetry int +} + +// NewPVCAnnotator creates a new PVCAnnotator. +func NewPVCAnnotator(client kubernetes.Interface) *PVCAnnotator { + return &PVCAnnotator{ + client: client, + maxRetry: 3, + } +} + +// Annotate updates the PVC with reclamation result annotations. +// It handles 404 (PVC not found) and 409 (conflict, retry) responses. +func (a *PVCAnnotator) Annotate(ctx context.Context, pvcName, pvcNamespace string, result *ReclamationResult) error { + var lastErr error + for attempt := 0; attempt <= a.maxRetry; attempt++ { + // GET the latest PVC + pvc, err := a.client.CoreV1().PersistentVolumeClaims(pvcNamespace).Get(ctx, pvcName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get PVC %s/%s: %w", pvcNamespace, pvcName, err) + } + + // Merge annotations + if pvc.Annotations == nil { + pvc.Annotations = make(map[string]string) + } + pvc.Annotations[AnnotationStatus] = result.Status + pvc.Annotations[AnnotationLastRunTime] = time.Now().UTC().Format(time.RFC3339) + pvc.Annotations[AnnotationBytesAvailable] = strconv.FormatInt(result.BytesAvailable, 10) + pvc.Annotations[AnnotationDuration] = fmt.Sprintf("%.3f", result.Duration.Seconds()) + pvc.Annotations[AnnotationNode] = result.NodeName + if result.ErrorMessage != "" { + pvc.Annotations[AnnotationErrorMsg] = result.ErrorMessage + } else { + // Clear error message on success to remove stale error states + delete(pvc.Annotations, AnnotationErrorMsg) + } + + // UPDATE the PVC + _, err = a.client.CoreV1().PersistentVolumeClaims(pvcNamespace).Update(ctx, pvc, metav1.UpdateOptions{}) + if err == nil { + return nil + } + lastErr = err + // Retry on conflict (409) + if strings.Contains(err.Error(), "the object has been modified") || strings.Contains(err.Error(), "Conflict") { + continue + } + return fmt.Errorf("failed to update PVC %s/%s: %w", pvcNamespace, pvcName, err) + } + return lastErr +} + +// ---- Event Emitter ---- + +// EventEmitter creates Kubernetes Events on PVCs. +type EventEmitter struct { + recorder record.EventRecorder +} + +// NewEventEmitter creates a new EventEmitter with a Kubernetes event recorder. +func NewEventEmitter(clientset kubernetes.Interface, driverName string) *EventEmitter { + if clientset == nil { + return &EventEmitter{} + } + eventBroadcaster := record.NewBroadcaster() + eventBroadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{ + Interface: clientset.CoreV1().Events(""), + }) + recorder := eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: driverName}) + return &EventEmitter{recorder: recorder} +} + +// EmitSuccess records a successful reclamation event on the PVC. +func (e *EventEmitter) EmitSuccess(pvc *corev1.PersistentVolumeClaim, bytesAvailable int64) { + if e.recorder == nil { + return + } + msg := fmt.Sprintf("Space reclamation completed: %d bytes available", bytesAvailable) + e.recorder.Event(pvc, corev1.EventTypeNormal, EventReasonCompleted, msg) +} + +// EmitFailure records a failed reclamation event on the PVC. +func (e *EventEmitter) EmitFailure(pvc *corev1.PersistentVolumeClaim, err error) { + if e.recorder == nil { + return + } + msg := fmt.Sprintf("Space reclamation failed: %v", err) + e.recorder.Event(pvc, corev1.EventTypeWarning, EventReasonFailed, msg) +} + +// EmitTimeout records a timed-out reclamation event on the PVC. +func (e *EventEmitter) EmitTimeout(pvc *corev1.PersistentVolumeClaim, timeout time.Duration) { + if e.recorder == nil { + return + } + msg := fmt.Sprintf("Space reclamation timed out after %v", timeout) + e.recorder.Event(pvc, corev1.EventTypeWarning, EventReasonTimeout, msg) +} + +// EmitUnsupported records an unsupported-device reclamation event on the PVC. +func (e *EventEmitter) EmitUnsupported(pvc *corev1.PersistentVolumeClaim, reason string) { + if e.recorder == nil { + return + } + msg := fmt.Sprintf("Device does not support space reclamation: %s", reason) + e.recorder.Event(pvc, corev1.EventTypeWarning, EventReasonUnsupported, msg) +} + +// ---- Eligibility ---- + +// IsEligible determines if a volume is eligible for reclamation based on +// global config and per-PVC labels. +// Returns (eligible, reason) where reason explains why not eligible (empty if eligible). +func IsEligible(globalEnabled bool, labels map[string]string, volumeMode VolumeMode) (bool, string) { + // Block mode requires explicit opt-in via label + if volumeMode == VolumeModeBlock { + if labels == nil { + return false, "block mode missing required label" + } + val, ok := labels[LabelBlockReclaim] + if !ok { + return false, "block mode missing required label" + } + if !strings.EqualFold(val, "true") { + return false, fmt.Sprintf("block mode label is '%s' (must be 'true')", val) + } + return true, "" + } + + // Filesystem mode: explicit label takes precedence, otherwise follow global config + if labels == nil { + if globalEnabled { + return true, "" + } + return false, "global disabled" + } + val, ok := labels[LabelEnabled] + if !ok { + if globalEnabled { + return true, "" + } + return false, "global disabled" + } + if strings.EqualFold(val, "true") { + return true, "" + } + return false, fmt.Sprintf("label is '%s' (must be 'true' to override global)", val) +} + +// ---- Injectable function variables (overridable in tests) ---- + +// wwnToDevicePathFunc is overridable in tests to avoid real /dev/disk/by-id lookups. +var wwnToDevicePathFunc = func(ctx context.Context, wwn string) (string, string, error) { + return gofsutil.WWNToDevicePathX(ctx, wwn) +} + +// checkDiscardCapabilityFunc checks if a device supports discard operations. +var checkDiscardCapabilityFunc = func(ctx context.Context, devicePath string) (supported bool, maxBytes int64, reason string) { + discardCap, err := gofsutil.CheckDiscardSupport(ctx, devicePath) + if err != nil { + return false, 0, fmt.Sprintf("failed to check discard support: %v", err) + } + if !discardCap.Supported { + return false, discardCap.DiscardMaxBytes, discardCap.Reason + } + return true, discardCap.DiscardMaxBytes, "" +} + +// getVolumeWWNFunc is overridable in tests to avoid real PowerMax API calls. +// Default implementation calls the manager's getVolumeWWN method. +var getVolumeWWNFunc = func(m *SpaceReclamationManager, ctx context.Context, pvName, volumeHandle string) (symID, devID, wwn string, err error) { + return m.getVolumeWWN(ctx, pvName, volumeHandle) +} + +// normalizeDevicePathFunc resolves symlinks to get the canonical device path. +// This handles cases where mount table shows /dev/mapper/mpatha but WWN resolution returns /dev/dm-1. +// Returns the original path if symlink resolution fails. Overridable in tests. +var normalizeDevicePathFunc = func(path string) string { + resolved, err := filepath.EvalSymlinks(path) + if err != nil { + return path + } + return resolved +} + +// normalizeDevicePath is a convenience wrapper around normalizeDevicePathFunc. +func normalizeDevicePath(path string) string { + return normalizeDevicePathFunc(path) +} + +// ---- Space Reclamation Manager ---- + +// SpaceReclamationManager orchestrates periodic space reclamation on staged volumes. +type SpaceReclamationManager struct { + config SpaceReclamationConfig + annotator *PVCAnnotator + emitter *EventEmitter + k8sClient kubernetes.Interface + semaphore chan struct{} + volumeLocks sync.Map + ctx context.Context + cronSched *cron.Cron + running atomic.Bool // Flag to prevent overlapping RunOnce cycles + svc *service // Reference to service for PowerMax client access +} + +// NewSpaceReclamationManager creates a new SpaceReclamationManager. +// Returns error if the cron schedule expression is invalid. +func NewSpaceReclamationManager( + ctx context.Context, + config SpaceReclamationConfig, + k8sClient kubernetes.Interface, + nodeName string, + svc *service, +) (*SpaceReclamationManager, error) { + // Validate the cron expression by attempting to parse it + parser := cron.NewParser(cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow) + _, err := parser.Parse(config.Schedule) + if err != nil { + return nil, fmt.Errorf("invalid cron schedule %q: %w", config.Schedule, err) + } + + config.NodeName = nodeName + + semSize := config.MaxConcurrentVolumes + if semSize <= 0 { + semSize = 1 + } + + mgr := &SpaceReclamationManager{ + config: config, + annotator: NewPVCAnnotator(k8sClient), + emitter: NewEventEmitter(k8sClient, Name), + k8sClient: k8sClient, + semaphore: make(chan struct{}, semSize), + ctx: ctx, + svc: svc, + } + return mgr, nil +} + +// Start begins the cron-based reclamation scheduler. +func (m *SpaceReclamationManager) Start() error { + m.cronSched = cron.New(cron.WithParser(cron.NewParser( + cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow, + ))) + log.Info("SpaceReclamation: cron scheduler created") + _, err := m.cronSched.AddFunc(m.config.Schedule, m.RunOnce) + if err != nil { + log.Errorf("SpaceReclamation: failed to add cron job: %v", err) + return fmt.Errorf("failed to add cron job: %w", err) + } + log.Info("SpaceReclamation: cron job added") + m.cronSched.Start() + log.Infof("SpaceReclamation: scheduler running with schedule %q", m.config.Schedule) + return nil +} + +// Stop halts the cron scheduler. +func (m *SpaceReclamationManager) Stop() { + if m.cronSched != nil { + m.cronSched.Stop() + } +} + +// buildDeviceToMountMap builds a map of device paths to mount paths from the system mount table. +// It filters to only CSI-related mounts and prefers private mounts over pod mount paths. +// For filesystem volumes, the PowerMax driver mounts devices to /var/lib/kubelet/plugins/powermax.emc.dell.com/disks/ (private mount) +// and then bind-mounts to pod paths. The private mount is more stable and persists even when pods are recreated. +// Device paths are normalized by resolving symlinks to handle /dev/mapper/mpatha vs /dev/dm-1 differences. +func (m *SpaceReclamationManager) buildDeviceToMountMap(ctx context.Context) (map[string]string, error) { + mounts, err := gofsutil.GetMounts(ctx) + if err != nil { + log.Errorf("SpaceReclamation: failed to get mounts: %v", err) + return nil, fmt.Errorf("failed to get mounts: %w", err) + } + log.Infof("SpaceReclamation: found %d total mounts", len(mounts)) + + deviceToMount := make(map[string]string, len(mounts)) + for _, mnt := range mounts { + // Filter to only CSI-related mounts: + // 1. Pod mounts: /var/lib/kubelet/pods/... + // 2. Private mounts: /var/lib/kubelet/plugins/powermax.emc.dell.com/disks/... + isPodMount := strings.Contains(mnt.Path, "/var/lib/kubelet/pods/") + isPrivateMount := strings.Contains(mnt.Path, "/var/lib/kubelet/plugins/powermax.emc.dell.com/disks/") + + if !isPodMount && !isPrivateMount { + continue + } + + log.Infof("SpaceReclamation: CSI mount - Device: %s, Path: %s", mnt.Device, mnt.Path) + + // Normalize device path by resolving symlinks (e.g., /dev/mapper/mpatha -> /dev/dm-1) + // This ensures consistent comparison with WWN-resolved device paths + normalizedDevice := normalizeDevicePath(mnt.Device) + + // Prefer private mounts (/var/lib/kubelet/plugins/powermax.emc.dell.com/disks/) over pod mount paths (/pods/) + // Private mounts are more stable and persist even when pods are recreated + currentPath, exists := deviceToMount[normalizedDevice] + if !exists { + deviceToMount[normalizedDevice] = mnt.Path + } else if isPrivateMount && !strings.Contains(currentPath, "/var/lib/kubelet/plugins/powermax.emc.dell.com/disks/") { + // Replace with private mount if current is not a private mount + deviceToMount[normalizedDevice] = mnt.Path + } + } + return deviceToMount, nil +} + +// shouldSkipPV checks if a PV should be skipped based on basic filters. +// Returns (shouldSkip, reason). +func (m *SpaceReclamationManager) shouldSkipPV(pv *corev1.PersistentVolume) (bool, string) { + // Filter: only process PVs managed by this driver + if pv.Spec.CSI == nil || pv.Spec.CSI.Driver != Name { + return true, "not managed by this driver" + } + // Filter: only process Bound PVs + if pv.Status.Phase != corev1.VolumeBound { + return true, "not bound" + } + // Skip RWX volumes - space reclamation not supported for multi-node access + for _, accessMode := range pv.Spec.AccessModes { + if accessMode == corev1.ReadWriteMany { + return true, "SpaceReclamation: skipping RWX volume" + } + } + // Skip NFS volumes - space reclamation is handled at the NFS server level + if pv.Spec.CSI.FSType == "nfs" { + return true, "NFS handled at server level" + } + if pv.Spec.ClaimRef == nil { + return true, "no claim reference" + } + return false, "" +} + +// getVolumeWWN retrieves the WWN for a PowerMax volume. +func (m *SpaceReclamationManager) getVolumeWWN(ctx context.Context, pvName, volumeHandle string) (symID, devID, wwn string, err error) { + // Parse the CSI volume ID to get symID and devID + _, symID, devID, _, _, err = m.parseCsiID(volumeHandle) + if err != nil { + return "", "", "", fmt.Errorf("failed to parse volumeHandle %s: %w", volumeHandle, err) + } + log.Infof("SpaceReclamation: PV %s parsed to symID=%s, devID=%s", pvName, symID, devID) + + // Get PowerMax client and retrieve volume details to get WWN + pmaxClient, err := m.getPowerMaxClient(symID) + if err != nil { + return "", "", "", fmt.Errorf("failed to get PowerMax client for array %s: %w", symID, err) + } + log.Info("SpaceReclamation: PowerMax client successfully initialized") + + // Retrieve volume from PowerMax to get EffectiveWWN + vol, err := pmaxClient.GetVolumeByID(ctx, symID, devID) + if err != nil { + return "", "", "", fmt.Errorf("failed to get volume %s/%s from array: %w", symID, devID, err) + } + log.Infof("SpaceReclamation: Retrieved Volume Details: %v", vol) + + wwn = vol.EffectiveWWN + if wwn == "" { + return "", "", "", fmt.Errorf("volume %s/%s has no EffectiveWWN", symID, devID) + } + log.Infof("SpaceReclamation: PV %s retrieved WWN %s from PowerMax", pvName, wwn) + return symID, devID, wwn, nil +} + +// selectBestDevice selects the best device from a list based on protocol and mount status. +// For filesystem mode, it verifies the device is in the mount table. +func (m *SpaceReclamationManager) selectBestDevice(pvName, wwn string, devices []string, deviceToMount map[string]string, volMode VolumeMode) (string, error) { + // First, try to find a device that's already mounted (for both filesystem and block) + for _, dev := range devices { + candidatePath := "/dev/" + dev + // Normalize the candidate path to match the normalized keys in deviceToMount + normalizedCandidate := normalizeDevicePath(candidatePath) + if _, mounted := deviceToMount[normalizedCandidate]; mounted { + log.Infof("SpaceReclamation: PV %s resolved to device %s (found in mount table)", pvName, candidatePath) + return candidatePath, nil + } + } + + // For filesystem mode, if no mounted device found, fail + if volMode == VolumeModeFilesystem { + return "", fmt.Errorf("no device from WWN %s found in mount table", wwn) + } + + // For block mode, select the best candidate even if not in mount table + // (block devices don't appear in mount table with their actual device path) + var namespaceDevice string + // Single iteration to classify devices and find the best match + var isNVMe bool + var multipathDevice string + var firstDevice string + + for _, dev := range devices { + // Store first device for fallback + if firstDevice == "" { + firstDevice = dev + } + + // Check for NVMe devices + if strings.HasPrefix(dev, "nvme") { + isNVMe = true + // NVMe namespace devices have format nvmen (e.g., nvme0n4) + // Controller devices have format nvmecn (e.g., nvme0c0n4) + if !strings.Contains(dev, "c") { + namespaceDevice = dev + break // Found NVMe namespace device, stop searching + } + } + + // Check for multipath devices (only relevant for non-NVMe) + if !isNVMe && multipathDevice == "" && strings.HasPrefix(dev, "dm-") { + multipathDevice = dev + } + } + + // Return NVMe namespace device if found + if isNVMe { + if namespaceDevice != "" { + devicePath := "/dev/" + namespaceDevice + log.Infof("SpaceReclamation: PV %s resolved to device %s (NVMe namespace device)", pvName, devicePath) + return devicePath, nil + } + return "", fmt.Errorf("NVMe devices found but no namespace device (only controller devices)") + } + + // For non-NVMe (FC/iSCSI), prefer multipath devices (dm-*) over single paths (sd*) + if multipathDevice != "" { + devicePath := "/dev/" + multipathDevice + log.Infof("SpaceReclamation: PV %s resolved to device %s (multipath device)", pvName, devicePath) + return devicePath, nil + } + + // Fallback: use the first device + devicePath := "/dev/" + firstDevice + log.Infof("SpaceReclamation: PV %s resolved to device %s (first device)", pvName, devicePath) + return devicePath, nil +} + +// resolveDevicePath resolves the device path for a volume from its WWN. +// For filesystem mode, it ensures the device is in the mount table. +func (m *SpaceReclamationManager) resolveDevicePath(ctx context.Context, pvName, wwn string, deviceToMount map[string]string, volMode VolumeMode) (string, error) { + // Try WWNToDevicePath first + _, devicePath, err := wwnToDevicePathFunc(ctx, wwn) + log.Infof("SpaceReclamation: WWNToDevicePath result: devicePath=%s, err=%v", devicePath, err) + + if err == nil && devicePath != "" { + log.Infof("SpaceReclamation: PV %s resolved to device %s via WWN %s", pvName, devicePath, wwn) + // Normalize the device path to handle symlinks (e.g., /dev/mapper/mpatha -> /dev/dm-1) + normalizedDevicePath := normalizeDevicePath(devicePath) + log.Infof("SpaceReclamation: PV %s normalized device path: %s -> %s", pvName, devicePath, normalizedDevicePath) + + // Verify the device is in mount table for filesystem mode + if volMode == VolumeModeFilesystem { + if mountPath, mounted := deviceToMount[normalizedDevicePath]; mounted { + log.Infof("SpaceReclamation: PV %s found in mount table at %s", pvName, mountPath) + // Return the original devicePath (not normalized) for consistency with other code + return devicePath, nil + } + log.Warnf("SpaceReclamation: PV %s normalized device %s not in mount table, trying fallback", pvName, normalizedDevicePath) + // Fall through to GetSysBlockDevicesForVolumeWWN + } else { + // Block mode: accept the device even if not in mount table + return devicePath, nil + } + } + + // Fallback: try GetSysBlockDevicesForVolumeWWN + log.Infof("SpaceReclamation: trying GetSysBlockDevicesForVolumeWWN for PV %s", pvName) + devices, err := gofsutil.GetSysBlockDevicesForVolumeWWN(ctx, wwn) + log.Infof("SpaceReclamation: GetSysBlockDevicesForVolumeWWN result: length(devices)=%d, err=%v", len(devices), err) + if err != nil || len(devices) == 0 { + return "", fmt.Errorf("PV not on this node (WWN: %s)", wwn) + } + + return m.selectBestDevice(pvName, wwn, devices, deviceToMount, volMode) +} + +// processVolume processes a single PV for space reclamation. +// Returns the VolumeInfo if successful, or nil if the volume should be skipped. +func (m *SpaceReclamationManager) processVolume(ctx context.Context, pv *corev1.PersistentVolume, deviceToMount map[string]string) (*VolumeInfo, error) { + // Check basic PV filters + if skip, reason := m.shouldSkipPV(pv); skip { + log.Infof("SpaceReclamation: skipping PV %s (%s)", pv.Name, reason) + return nil, nil + } + + log.Infof("SpaceReclamation: processing PV %s with AccessModes: %v", pv.Name, pv.Spec.AccessModes) + + pvcRef := pv.Spec.ClaimRef + + // Get PVC and check eligibility + pvc, err := m.k8sClient.CoreV1().PersistentVolumeClaims(pvcRef.Namespace).Get(ctx, pvcRef.Name, metav1.GetOptions{}) + if err != nil { + log.Errorf("SpaceReclamation: failed to get PVC: %v", err) + return nil, fmt.Errorf("failed to get PVC %s/%s: %w", pvcRef.Namespace, pvcRef.Name, err) + } + + var volMode VolumeMode + if pvc.Spec.VolumeMode != nil { + volMode = VolumeMode(*pvc.Spec.VolumeMode) + } else { + volMode = VolumeModeFilesystem + } + + log.Infof("SpaceReclamation: PV %s has VolumeMode: %s, Labels: %v", pv.Name, volMode, pvc.Labels) + eligible, reason := IsEligible(m.config.Enabled, pvc.Labels, volMode) + if !eligible { + log.Infof("SpaceReclamation: PV %s is not eligible for reclamation (reason: %s)", pv.Name, reason) + return nil, nil + } + log.Infof("SpaceReclamation: PV %s is eligible for reclamation", pv.Name) + + // Get volume handle + volumeHandle := pv.Spec.CSI.VolumeHandle + if volumeHandle == "" { + return nil, fmt.Errorf("PV %s missing VolumeHandle", pv.Name) + } + log.Infof("SpaceReclamation: PV %s has VolumeHandle: %s", pv.Name, volumeHandle) + + // Get WWN from PowerMax (or test mock) + symID, devID, wwn, err := getVolumeWWNFunc(m, ctx, pv.Name, volumeHandle) + if err != nil { + return nil, fmt.Errorf("failed to get WWN: %w", err) + } + _ = symID // Unused in current implementation but may be needed later + _ = devID // Unused in current implementation but may be needed later + + // Resolve device path + devicePath, err2 := m.resolveDevicePath(ctx, pv.Name, wwn, deviceToMount, volMode) + if err2 != nil { + return nil, fmt.Errorf("failed to resolve device path: %w", err2) + } + + // Resolve staging path for filesystem mode + var stagingPath string + if volMode == VolumeModeFilesystem { + log.Info("Found Filesystem Mode") + if mountPath, mounted := deviceToMount[devicePath]; mounted { + stagingPath = mountPath + log.Infof("SpaceReclamation: PV %s filesystem mode, staging path: %s", pv.Name, stagingPath) + } else { + return nil, fmt.Errorf("filesystem mode but no mount found for device %s", devicePath) + } + } else { + log.Info("Found Block Mode") + log.Infof("SpaceReclamation: PV %s block mode, device path: %s", pv.Name, devicePath) + } + + volInfo := &VolumeInfo{ + VolumeID: volumeHandle, + StagingPath: stagingPath, + DevicePath: devicePath, + VolumeMode: volMode, + PVCName: pvcRef.Name, + PVCNamespace: pvcRef.Namespace, + PVC: pvc, + } + + if volMode == VolumeModeFilesystem { + log.Infof("SpaceReclamation: submitting reclamation job for PV %s (VolumeID: %s, Device: %s, Path: %s, Mode: %s)", + pv.Name, volInfo.VolumeID, devicePath, stagingPath, volMode) + } else { + log.Infof("SpaceReclamation: submitting reclamation job for PV %s (VolumeID: %s, Device: %s, Mode: %s)", + pv.Name, volInfo.VolumeID, devicePath, volMode) + } + + return volInfo, nil +} + +// RunOnce executes one reclamation cycle using on-demand discovery. +// Called by the cron scheduler. +// It discovers eligible volumes by: +// 1. Listing all Bound PowerMax PVs from Kubernetes. +// 2. Checking live PVC labels for eligibility (fail fast, no device work for ineligible PVCs). +// 3. Resolving the device path from WWN using gofsutil.WWNToDevicePathX. +// 4. Resolving the mount path from gofsutil.GetMounts() to determine VolumeMode. +func (m *SpaceReclamationManager) RunOnce() { + log.Info("SpaceReclamation: starting RunOnce cycle") + + // Prevent overlapping cycles + if !m.running.CompareAndSwap(false, true) { + log.Warn("SpaceReclamation: previous scheduled run is still in progress, skipping this cycle") + return + } + defer m.running.Store(false) + + // Create a job-level timeout context for the entire reclamation cycle + timeout := time.Duration(m.config.TimeoutSeconds) * time.Second + ctx, cancel := context.WithTimeout(m.ctx, timeout) + defer cancel() + + // List all PVs in the cluster + pvList, err := m.k8sClient.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{}) + if err != nil { + log.Errorf("SpaceReclamation: failed to list PersistentVolumes: %v", err) + return + } + log.Infof("SpaceReclamation: found %d total PVs", len(pvList.Items)) + + // Build device-to-mount map for O(1) lookup + deviceToMount, err := m.buildDeviceToMountMap(ctx) + if err != nil { + log.Errorf("SpaceReclamation: %v", err) + return + } + + // Process each PV concurrently + var wg sync.WaitGroup + for i := range pvList.Items { + pv := &pvList.Items[i] + + volInfo, err := m.processVolume(ctx, pv, deviceToMount) + if err != nil { + log.Warnf("SpaceReclamation: PV %s error: %v", pv.Name, err) + continue + } + if volInfo == nil { + // Volume was skipped (not eligible or filtered out) + continue + } + + // Submit reclamation job + wg.Add(1) + go func(v *VolumeInfo) { + defer wg.Done() + m.reclaimVolume(ctx, v) + }(volInfo) + } + wg.Wait() + log.Info("SpaceReclamation: completed RunOnce cycle") +} + +// reclaimVolume performs space reclamation on a single volume. +func (m *SpaceReclamationManager) reclaimVolume(ctx context.Context, vol *VolumeInfo) { + log.Infof("SpaceReclamation: starting reclamation for volume %s (PVC: %s/%s, Mode: %s, Device: %s, Path: %s)", + vol.VolumeID, vol.PVCNamespace, vol.PVCName, vol.VolumeMode, vol.DevicePath, vol.StagingPath) + + // Acquire semaphore for concurrency control + select { + case m.semaphore <- struct{}{}: + defer func() { <-m.semaphore }() + case <-ctx.Done(): + return + } + + // Acquire per-volume mutex to prevent duplicate jobs + mu := &sync.Mutex{} + actual, _ := m.volumeLocks.LoadOrStore(vol.VolumeID, mu) + actualMu := actual.(*sync.Mutex) + if !actualMu.TryLock() { + log.Infof("SpaceReclamation: skipping duplicate job for volume %s (already in progress)", vol.VolumeID) + return // Another reclamation is already running for this volume + } + defer actualMu.Unlock() + + // Check if device supports discard operations + var supported bool + var reason string + supported, _, reason = checkDiscardCapabilityFunc(m.ctx, vol.DevicePath) + log.Infof("SpaceReclamation: checked discard capability for device %s (supported: %v, reason: %s)", vol.DevicePath, supported, reason) + if !supported { + log.Infof("SpaceReclamation: volume %s does not support discard (device: %s, reason: %s)", vol.VolumeID, vol.DevicePath, reason) + // Annotate as unsupported + result := &ReclamationResult{ + Status: "unsupported", + ErrorMessage: reason, + NodeName: m.config.NodeName, + } + if m.annotator != nil && m.k8sClient != nil && vol.PVCName != "" { + _ = m.annotator.Annotate(m.ctx, vol.PVCName, vol.PVCNamespace, result) + } + // Emit event for unsupported device + if m.emitter != nil && vol.PVC != nil { + m.emitter.EmitUnsupported(vol.PVC, reason) + } + return + } + + var bytesAvailable int64 + var reclaimErr error + start := time.Now() + + // Execute the reclamation operation + switch vol.VolumeMode { + case VolumeModeFilesystem: + var fstrimResult *gofsutil.FstrimResult + fstrimResult, reclaimErr = gofsutil.Fstrim(ctx, vol.StagingPath) + if reclaimErr == nil && fstrimResult != nil { + bytesAvailable = fstrimResult.BytesTrimmed + } + log.Infof("SpaceReclamation: fstrim on %s - %d bytes available", vol.StagingPath, bytesAvailable) + case VolumeModeBlock: + var blkResult *gofsutil.BlkdiscardResult + blkResult, reclaimErr = gofsutil.Blkdiscard(ctx, vol.DevicePath) + if reclaimErr == nil && blkResult != nil { + bytesAvailable = blkResult.BytesDiscarded + } + log.Infof("SpaceReclamation: blkdiscard on %s - %d bytes available", vol.DevicePath, bytesAvailable) + } + + duration := time.Since(start) + + // Build the result + var result *ReclamationResult + if reclaimErr != nil { + if ctx.Err() == context.DeadlineExceeded { + result = &ReclamationResult{ + Status: "timeout", + ErrorMessage: fmt.Sprintf("operation timed out after %v", time.Duration(m.config.TimeoutSeconds)*time.Second), + NodeName: m.config.NodeName, + Duration: duration, + } + } else { + result = &ReclamationResult{ + Status: "error", + ErrorMessage: reclaimErr.Error(), + NodeName: m.config.NodeName, + Duration: duration, + } + } + } else { + result = &ReclamationResult{ + Status: "success", + BytesAvailable: bytesAvailable, + Duration: duration, + NodeName: m.config.NodeName, + } + } + + // Annotate the PVC with results + // If ctx is already expired (e.g. after a timeout), the Kubernetes API calls inside + // Annotate would immediately fail. Use a fresh context derived from the manager's + // parent context so the annotation is always written regardless of reclamation outcome. + annotateCtx := ctx + if ctx.Err() != nil { + var annotateCancel context.CancelFunc + annotateCtx, annotateCancel = context.WithTimeout(m.ctx, 10*time.Second) + defer annotateCancel() + } + if m.annotator != nil && m.k8sClient != nil && vol.PVCName != "" { + _ = m.annotator.Annotate(annotateCtx, vol.PVCName, vol.PVCNamespace, result) + } + + // Emit Kubernetes event based on result status + if m.emitter != nil && vol.PVC != nil { + switch result.Status { + case "success": + m.emitter.EmitSuccess(vol.PVC, result.BytesAvailable) + case "timeout": + m.emitter.EmitTimeout(vol.PVC, time.Duration(m.config.TimeoutSeconds)*time.Second) + case "error": + m.emitter.EmitFailure(vol.PVC, errors.New(result.ErrorMessage)) + } + } + + log.Infof("SpaceReclamation: completed reclamation for volume %s (PVC: %s/%s) - Status: %s, BytesAvailable: %d, Duration: %v", + vol.VolumeID, vol.PVCNamespace, vol.PVCName, result.Status, result.BytesAvailable, result.Duration) +} + +// parseCsiID parses the CSI volume ID to extract volume name, array ID, and device ID. +func (m *SpaceReclamationManager) parseCsiID(csiID string) (volName string, arrayID string, devID string, remoteSymID, remoteVolID string, err error) { + if m.svc == nil { + return "", "", "", "", "", fmt.Errorf("service reference is nil") + } + return m.svc.parseCsiID(csiID) +} + +// getPowerMaxClient returns a PowerMax client for the specified array. +func (m *SpaceReclamationManager) getPowerMaxClient(symID string) (pmax.Pmax, error) { + if m.svc == nil { + return nil, fmt.Errorf("service reference is nil") + } + return m.svc.GetPowerMaxClient(symID) +} + +// initSpaceReclamation reads env and initializes the space reclamation manager on the service. +// This is called from BeforeServe when in node mode. +// The manager is always initialized to allow per-PVC label-based opt-in, even when globally disabled. +// The cfg.Enabled flag is used in the eligibility check (IsEligible) to determine which PVCs to process. +func initSpaceReclamation(ctx context.Context, s *service, k8sClient kubernetes.Interface) { + cfg := ReadSpaceReclamationConfig() + log.Infof("SpaceReclamation: initializing with config: %v", cfg) + + mgr, err := NewSpaceReclamationManager(ctx, cfg, k8sClient, cfg.NodeName, s) + if err != nil { + log.Errorf("Failed to create SpaceReclamationManager: %v", err) + return + } + log.Infof("SpaceReclamation: created manager, starting...") + if err := mgr.Start(); err != nil { + log.Errorf("Failed to start SpaceReclamationManager: %v", err) + return + } + if cfg.Enabled { + log.Infof("SpaceReclamation: started successfully (globally enabled)") + } else { + log.Infof("SpaceReclamation: started successfully (globally disabled, per-PVC labels will be honored)") + } + s.spaceReclaimMgr = mgr +} diff --git a/service/space_reclamation_test.go b/service/space_reclamation_test.go new file mode 100644 index 00000000..ba5a9c4f --- /dev/null +++ b/service/space_reclamation_test.go @@ -0,0 +1,1186 @@ +// Copyright © 2024-2026 Dell Inc. or its subsidiaries. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package service + +import ( + "context" + "fmt" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/dell/gofsutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + "k8s.io/client-go/tools/record" +) + +// ============================================================================ +// Test Helpers +// ============================================================================ + +// makePVC creates a minimal PVC object for testing. +func makePVC(name, namespace string) *corev1.PersistentVolumeClaim { + return &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Annotations: map[string]string{}, + }, + } +} + +// newTestManager creates a SpaceReclamationManager with test defaults. +func newTestManager(t *testing.T, client *fake.Clientset, cfg SpaceReclamationConfig) *SpaceReclamationManager { + t.Helper() + ctx := context.Background() + + // Set up test WWN resolver before creating manager + // This mock extracts WWN from the volume handle for test volumes + getVolumeWWNFunc = func(_ *SpaceReclamationManager, _ context.Context, _ string, volumeHandle string) (symID, devID, wwn string, err error) { + // Parse test volume handles like "CSM-vol001-000120001647-00001" + // and return the corresponding test WWN + testWWNs := map[string]string{ + "CSM-vol001-000120001647-00001": "60000970000120001647533030303031", + "CSM-volblk001-000120001647-00002": "60000970000120001647533030303032", + "CSM-volunsup001-000120001647-00003": "60000970000120001647533030303033", + } + if wwn, ok := testWWNs[volumeHandle]; ok { + return "000120001647", strings.Split(volumeHandle, "-")[3], wwn, nil + } + return "", "", "", fmt.Errorf("test: unknown volume handle %s", volumeHandle) + } + + // Pass nil for *service parameter in tests since we use mock WWN resolution + mgr, err := NewSpaceReclamationManager(ctx, cfg, client, cfg.NodeName, nil) + require.NoError(t, err) + return mgr +} + +// resetGofsutilMock resets gofsutil mock to a clean state. +func resetGofsutilMock() { + gofsutil.GOFSMock.InduceMountError = false + gofsutil.GOFSMock.InduceUnmountError = false +} + +// defaultGetVolumeWWNFunc stores the original default implementation for restoration after tests. +var defaultGetVolumeWWNFunc = getVolumeWWNFunc + +// resetTestMocks resets all test mocks to their default state. +func resetTestMocks() { + getVolumeWWNFunc = defaultGetVolumeWWNFunc + resetGofsutilMock() +} + +var filesystemMode = corev1.PersistentVolumeFilesystem + +// ============================================================================ +// CONTRACT TESTS (C-*) -- FIRST +// ============================================================================ +// Contract tests validate integration points between components. +// They force wiring that connects new code to existing code. + +// C-001: TestReclamationCycle_CallsFstrimOnFilesystemVolume +// Forces: RunOnce worker dispatch + gofsutil.Fstrim call + PVC annotation update +func TestReclamationCycle_CallsFstrimOnFilesystemVolume(t *testing.T) { + gofsutil.UseMockFS() + defer resetGofsutilMock() + + pvc := makePVC("pvc-test-001", "default") + // Create PV with the PVC reference + // Use proper PowerMax CSI volume handle format: -- + pv := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pv-test-001", + }, + Spec: corev1.PersistentVolumeSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Capacity: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("10Gi"), + }, + PersistentVolumeSource: corev1.PersistentVolumeSource{ + CSI: &corev1.CSIPersistentVolumeSource{ + Driver: Name, + VolumeHandle: "CSM-vol001-000120001647-00001", + VolumeAttributes: map[string]string{ + "WWID": "60000970000120001647533030303031", + }, + }, + }, + ClaimRef: &corev1.ObjectReference{ + Name: "pvc-test-001", + Namespace: "default", + }, + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimDelete, + StorageClassName: "powermax-sc", + VolumeMode: &filesystemMode, + }, + Status: corev1.PersistentVolumeStatus{ + Phase: corev1.VolumeBound, + }, + } + + fakeClient := fake.NewSimpleClientset(pvc, pv) + mgr := newTestManager(t, fakeClient, SpaceReclamationConfig{ + Enabled: true, + Schedule: "* * * * *", + MaxConcurrentVolumes: 2, + TimeoutSeconds: 60, + NodeName: "node-1", + }) + + // Mock mount table with private mount path + gofsutil.GOFSMockMounts = []gofsutil.Info{ + { + Device: "/dev/sda", + Path: "/var/lib/kubelet/plugins/powermax.emc.dell.com/disks/CSM-vol001-000120001647-00001", + Source: "/dev/sda", + }, + } + + // Mock gofsutil functions for on-demand discovery + originalWWNToDevicePathFunc := wwnToDevicePathFunc + wwnToDevicePathFunc = func(_ context.Context, _ string) (string, string, error) { + return "60000970000120001647533030303031", "/dev/sda", nil + } + defer func() { wwnToDevicePathFunc = originalWWNToDevicePathFunc }() + + // Execute one reclamation cycle + mgr.RunOnce() + + // Verify PVC annotations were updated + updatedPVC, err := fakeClient.CoreV1().PersistentVolumeClaims("default").Get( + context.Background(), "pvc-test-001", metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, "success", updatedPVC.Annotations[AnnotationStatus], + "PVC should be annotated with status=success after fstrim") + assert.NotEmpty(t, updatedPVC.Annotations[AnnotationBytesAvailable], + "PVC should have bytes-available annotation") + assert.NotEmpty(t, updatedPVC.Annotations[AnnotationLastRunTime], + "PVC should have last-run-time annotation") + assert.Equal(t, "node-1", updatedPVC.Annotations[AnnotationNode], + "PVC should have node annotation") +} + +// C-002: TestReclamationCycle_CallsBlkdiscardOnBlockVolume +// Forces: RunOnce worker dispatch + gofsutil.Blkdiscard call +func TestReclamationCycle_CallsBlkdiscardOnBlockVolume(t *testing.T) { + gofsutil.UseMockFS() + defer resetGofsutilMock() + + pvc := makePVC("pvc-blk-001", "default") + pvc.Labels = map[string]string{ + LabelBlockReclaim: "true", + } + blockMode := corev1.PersistentVolumeBlock + pvc.Spec.VolumeMode = &blockMode + // Create PV with the PVC reference + // Use proper PowerMax CSI volume handle format: -- + pv := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pv-blk-001", + }, + Spec: corev1.PersistentVolumeSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Capacity: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("10Gi"), + }, + PersistentVolumeSource: corev1.PersistentVolumeSource{ + CSI: &corev1.CSIPersistentVolumeSource{ + Driver: Name, + VolumeHandle: "CSM-volblk001-000120001647-00002", + VolumeAttributes: map[string]string{ + "WWID": "60000970000120001647533030303032", + }, + }, + }, + ClaimRef: &corev1.ObjectReference{ + Name: "pvc-blk-001", + Namespace: "default", + }, + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimDelete, + StorageClassName: "powermax-sc", + VolumeMode: &blockMode, + }, + Status: corev1.PersistentVolumeStatus{ + Phase: corev1.VolumeBound, + }, + } + + fakeClient := fake.NewSimpleClientset(pvc, pv) + mgr := newTestManager(t, fakeClient, SpaceReclamationConfig{ + Enabled: true, + Schedule: "* * * * *", + MaxConcurrentVolumes: 2, + TimeoutSeconds: 60, + NodeName: "node-1", + }) + + // Mock WWN to device mapping for GetSysBlockDevicesForVolumeWWN + gofsutil.GOFSMockWWNToDevice = map[string]string{ + "60000970000120001647533030303032": "/dev/sdb", + } + defer func() { gofsutil.GOFSMockWWNToDevice = nil }() + + // Mock gofsutil functions for on-demand discovery + originalWWNToDevicePathFunc := wwnToDevicePathFunc + wwnToDevicePathFunc = func(_ context.Context, _ string) (string, string, error) { + return "60000970000120001647533030303032", "/dev/sdb", nil + } + defer func() { wwnToDevicePathFunc = originalWWNToDevicePathFunc }() + + // Execute one reclamation cycle + mgr.RunOnce() + + // Verify PVC annotations were updated + updatedPVC, err := fakeClient.CoreV1().PersistentVolumeClaims("default").Get( + context.Background(), "pvc-blk-001", metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, "success", updatedPVC.Annotations[AnnotationStatus], + "PVC should be annotated with status=success after blkdiscard") + assert.NotEmpty(t, updatedPVC.Annotations[AnnotationBytesAvailable], + "PVC should have bytes-available annotation") +} + +// C-003: TestReclamationCycle_SkipsUnsupportedDevice +// Forces: Capability check path + "unsupported" annotation + +func TestReclamationCycle_SkipsUnsupportedDevice(t *testing.T) { + gofsutil.UseMockFS() + defer resetGofsutilMock() + + pvc := makePVC("pvc-unsup-001", "default") + filesystemMode := corev1.PersistentVolumeFilesystem + // Create PV with the PVC reference + // Use proper PowerMax CSI volume handle format: -- + pv := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pv-unsup-001", + }, + Spec: corev1.PersistentVolumeSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Capacity: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("10Gi"), + }, + PersistentVolumeSource: corev1.PersistentVolumeSource{ + CSI: &corev1.CSIPersistentVolumeSource{ + Driver: Name, + VolumeHandle: "CSM-volunsup001-000120001647-00003", + VolumeAttributes: map[string]string{ + "WWID": "60000970000120001647533030303033", + }, + }, + }, + ClaimRef: &corev1.ObjectReference{ + Name: "pvc-unsup-001", + Namespace: "default", + }, + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimDelete, + StorageClassName: "powermax-sc", + VolumeMode: &filesystemMode, + }, + Status: corev1.PersistentVolumeStatus{ + Phase: corev1.VolumeBound, + }, + } + + fakeClient := fake.NewSimpleClientset(pvc, pv) + mgr := newTestManager(t, fakeClient, SpaceReclamationConfig{ + Enabled: true, + Schedule: "* * * * *", + MaxConcurrentVolumes: 2, + TimeoutSeconds: 60, + NodeName: "node-1", + }) + + // Mock mount table with private mount path + gofsutil.GOFSMockMounts = []gofsutil.Info{ + { + Device: "/dev/sdc", + Path: "/var/lib/kubelet/plugins/powermax.emc.dell.com/disks/CSM-volunsup001-000120001647-00003", + Source: "/dev/sdc", + }, + } + + // Mock checkDiscardCapabilityFunc to return unsupported for /dev/sdc + originalCheckDiscardFunc := checkDiscardCapabilityFunc + checkDiscardCapabilityFunc = func(_ context.Context, devicePath string) (bool, int64, string) { + if devicePath == "/dev/sdc" { + return false, 0, "discard_max_bytes is 0" + } + return true, 4294967295, "" + } + defer func() { checkDiscardCapabilityFunc = originalCheckDiscardFunc }() + + // Mock gofsutil functions for on-demand discovery + originalWWNToDevicePathFunc := wwnToDevicePathFunc + wwnToDevicePathFunc = func(_ context.Context, _ string) (string, string, error) { + return "60000970000120001647533030303033", "/dev/sdc", nil + } + defer func() { wwnToDevicePathFunc = originalWWNToDevicePathFunc }() + + // Execute one reclamation cycle + mgr.RunOnce() + + // Verify PVC annotations indicate unsupported + updatedPVC, err := fakeClient.CoreV1().PersistentVolumeClaims("default").Get( + context.Background(), "pvc-unsup-001", metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, "unsupported", updatedPVC.Annotations[AnnotationStatus], + "PVC should be annotated with status=unsupported for unsupported device") +} + +// ============================================================================ +// UNIT TESTS (U-*) -- SECOND +// ============================================================================ + +// --- TestReadSpaceReclamationConfig: default values and env override --- + +func TestReadSpaceReclamationConfig(t *testing.T) { + tests := []struct { + name string + envVars map[string]string + expected SpaceReclamationConfig + }{ + { + name: "AllDefaults", + envVars: map[string]string{}, + expected: SpaceReclamationConfig{ + Enabled: false, + Schedule: "0 2 * * 0", + MaxConcurrentVolumes: 2, + TimeoutSeconds: 14400, + NodeName: "", + }, + }, + { + name: "AllCustom", + envVars: map[string]string{ + "X_CSI_SPACE_RECLAMATION_ENABLED": "true", + "X_CSI_SPACE_RECLAMATION_SCHEDULE": "*/5 * * * *", + "X_CSI_SPACE_RECLAMATION_MAX_CONCURRENT": "4", + "X_CSI_SPACE_RECLAMATION_TIMEOUT": "1800", + "X_CSI_POWERMAX_NODENAME": "node-x", + }, + expected: SpaceReclamationConfig{ + Enabled: true, + Schedule: "*/5 * * * *", + MaxConcurrentVolumes: 4, + TimeoutSeconds: 1800, + NodeName: "node-x", + }, + }, + { + name: "InvalidBool", + envVars: map[string]string{ + "X_CSI_SPACE_RECLAMATION_ENABLED": "notabool", + }, + expected: SpaceReclamationConfig{ + Enabled: false, + Schedule: "0 2 * * 0", + MaxConcurrentVolumes: 2, + TimeoutSeconds: 14400, + }, + }, + { + name: "InvalidInt", + envVars: map[string]string{ + "X_CSI_SPACE_RECLAMATION_MAX_CONCURRENT": "abc", + }, + expected: SpaceReclamationConfig{ + Enabled: false, + Schedule: "0 2 * * 0", + MaxConcurrentVolumes: 2, + TimeoutSeconds: 14400, + }, + }, + { + name: "ZeroConcurrent", + envVars: map[string]string{ + "X_CSI_SPACE_RECLAMATION_MAX_CONCURRENT": "0", + }, + expected: SpaceReclamationConfig{ + Enabled: false, + Schedule: "0 2 * * 0", + MaxConcurrentVolumes: 0, + TimeoutSeconds: 14400, + }, + }, + { + name: "EmptySchedule", + envVars: map[string]string{ + "X_CSI_SPACE_RECLAMATION_SCHEDULE": "", + }, + expected: SpaceReclamationConfig{ + Enabled: false, + Schedule: "0 2 * * 0", + MaxConcurrentVolumes: 2, + TimeoutSeconds: 14400, + }, + }, + { + name: "NegativeTimeout", + envVars: map[string]string{ + "X_CSI_SPACE_RECLAMATION_TIMEOUT": "-1", + }, + expected: SpaceReclamationConfig{ + Enabled: false, + Schedule: "0 2 * * 0", + MaxConcurrentVolumes: 2, + TimeoutSeconds: 14400, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Clear all relevant env vars first + t.Setenv("X_CSI_SPACE_RECLAMATION_ENABLED", "") + t.Setenv("X_CSI_SPACE_RECLAMATION_SCHEDULE", "") + t.Setenv("X_CSI_SPACE_RECLAMATION_MAX_CONCURRENT", "") + t.Setenv("X_CSI_SPACE_RECLAMATION_TIMEOUT", "") + t.Setenv("X_CSI_POWERMAX_NODENAME", "") + + for k, v := range tt.envVars { + t.Setenv(k, v) + } + + cfg := ReadSpaceReclamationConfig() + assert.Equal(t, tt.expected.Enabled, cfg.Enabled, "Enabled mismatch") + assert.Equal(t, tt.expected.Schedule, cfg.Schedule, "Schedule mismatch") + assert.Equal(t, tt.expected.MaxConcurrentVolumes, cfg.MaxConcurrentVolumes, "MaxConcurrentVolumes mismatch") + assert.Equal(t, tt.expected.TimeoutSeconds, cfg.TimeoutSeconds, "TimeoutSeconds mismatch") + assert.Equal(t, tt.expected.NodeName, cfg.NodeName, "NodeName mismatch") + }) + } +} + +// --- TestSpaceReclamationManager_StartStop --- + +func TestSpaceReclamationManager_StartStop(t *testing.T) { + fakeClient := fake.NewSimpleClientset() + mgr := newTestManager(t, fakeClient, SpaceReclamationConfig{ + Enabled: true, + Schedule: "0 2 * * 0", + MaxConcurrentVolumes: 2, + TimeoutSeconds: 60, + NodeName: "node-1", + }) + + err := mgr.Start() + require.NoError(t, err, "Start should succeed") + assert.NotNil(t, mgr.cronSched, "cron scheduler should be initialized") + + // Stop should not panic + assert.NotPanics(t, func() { + mgr.Stop() + }, "Stop should not panic") +} + +// --- TestBuildAnnotations --- + +func TestBuildAnnotations(t *testing.T) { + pvc := makePVC("test-pvc", "default") + fakeClient := fake.NewSimpleClientset(pvc) + annotator := NewPVCAnnotator(fakeClient) + + result := &ReclamationResult{ + Status: "success", + BytesAvailable: 1073741824, + Duration: 500 * time.Millisecond, + NodeName: "node-1", + } + err := annotator.Annotate(context.Background(), "test-pvc", "default", result) + require.NoError(t, err) + + updated, err := fakeClient.CoreV1().PersistentVolumeClaims("default").Get( + context.Background(), "test-pvc", metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, "success", updated.Annotations[AnnotationStatus]) + assert.Equal(t, "1073741824", updated.Annotations[AnnotationBytesAvailable]) + assert.Equal(t, "node-1", updated.Annotations[AnnotationNode]) + assert.NotEmpty(t, updated.Annotations[AnnotationLastRunTime]) + assert.NotEmpty(t, updated.Annotations[AnnotationDuration]) +} + +func TestBuildAnnotations_ErrorResult(t *testing.T) { + pvc := makePVC("test-pvc", "default") + fakeClient := fake.NewSimpleClientset(pvc) + annotator := NewPVCAnnotator(fakeClient) + + result := &ReclamationResult{ + Status: "error", + ErrorMessage: "fstrim failed: permission denied", + NodeName: "node-1", + } + err := annotator.Annotate(context.Background(), "test-pvc", "default", result) + require.NoError(t, err) + + updated, err := fakeClient.CoreV1().PersistentVolumeClaims("default").Get( + context.Background(), "test-pvc", metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, "error", updated.Annotations[AnnotationStatus]) + assert.Contains(t, updated.Annotations[AnnotationErrorMsg], "fstrim failed") +} + +func TestBuildAnnotations_PVCNotFound(t *testing.T) { + fakeClient := fake.NewSimpleClientset() + annotator := NewPVCAnnotator(fakeClient) + + result := &ReclamationResult{Status: "success", BytesAvailable: 100} + err := annotator.Annotate(context.Background(), "nonexistent-pvc", "default", result) + assert.Error(t, err, "annotating non-existent PVC should return error") +} + +func TestBuildAnnotations_ConflictRetry(t *testing.T) { + pvc := makePVC("test-pvc", "default") + fakeClient := fake.NewSimpleClientset(pvc) + + updateCount := 0 + fakeClient.PrependReactor("update", "persistentvolumeclaims", func(_ k8stesting.Action) (bool, runtime.Object, error) { + updateCount++ + if updateCount == 1 { + return true, nil, fmt.Errorf("the object has been modified; please apply your changes to the latest version and try again") + } + return false, nil, nil + }) + + annotator := NewPVCAnnotator(fakeClient) + result := &ReclamationResult{Status: "success", BytesAvailable: 100, NodeName: "node-1"} + err := annotator.Annotate(context.Background(), "test-pvc", "default", result) + + assert.NoError(t, err, "annotator should handle conflict with retry") + assert.GreaterOrEqual(t, updateCount, 2, "should have retried at least once") +} + +// --- TestIsEligible --- + +func TestIsEligible_GlobalEnabledNoAnnotation(t *testing.T) { + annotations := map[string]string{} + eligible, _ := IsEligible(true, annotations, VolumeModeFilesystem) + assert.True(t, eligible, "global enabled + no annotation = eligible") +} + +func TestIsEligible_ExplicitOptOut(t *testing.T) { + annotations := map[string]string{ + LabelEnabled: "false", + } + eligible, _ := IsEligible(true, annotations, VolumeModeFilesystem) + assert.False(t, eligible, "explicit opt-out should make volume ineligible") +} + +func TestIsEligible_ExplicitOptIn(t *testing.T) { + annotations := map[string]string{ + LabelEnabled: "true", + } + eligible, _ := IsEligible(true, annotations, VolumeModeFilesystem) + assert.True(t, eligible, "explicit opt-in should make volume eligible") +} + +func TestIsEligible_GlobalDisabled(t *testing.T) { + annotations := map[string]string{} + eligible, _ := IsEligible(false, annotations, VolumeModeFilesystem) + assert.False(t, eligible, "global disabled = ineligible") +} + +func TestIsEligible_NilAnnotationsMap(t *testing.T) { + eligible, _ := IsEligible(true, nil, VolumeModeFilesystem) + assert.True(t, eligible, "nil annotations with global enabled = eligible") +} + +func TestIsEligible_GlobalDisabledButLabelOptIn(t *testing.T) { + annotations := map[string]string{ + LabelEnabled: "true", + } + eligible, _ := IsEligible(false, annotations, VolumeModeFilesystem) + assert.True(t, eligible, "global disabled but label opt-in should make volume eligible (label precedence)") +} + +func TestIsEligible_BlockModeIgnoresGlobalConfig(t *testing.T) { + annotations := map[string]string{ + LabelBlockReclaim: "true", + } + // Test with globalEnabled=false - should still be eligible because block mode ignores global config + eligible, _ := IsEligible(false, annotations, VolumeModeBlock) + assert.True(t, eligible, "block mode with label should be eligible even when globally disabled") + + // Test with globalEnabled=true - should also be eligible + eligible, _ = IsEligible(true, annotations, VolumeModeBlock) + assert.True(t, eligible, "block mode with label should be eligible when globally enabled") +} + +// --- EventEmitter tests --- + +func TestEventEmitter_EmitSuccess(t *testing.T) { + recorder := record.NewFakeRecorder(10) + emitter := &EventEmitter{recorder: recorder} + pvc := makePVC("test-pvc", "default") + + emitter.EmitSuccess(pvc, 1073741824) + + select { + case event := <-recorder.Events: + assert.Contains(t, event, EventReasonCompleted, "event should contain SpaceReclamationCompleted") + case <-time.After(time.Second): + t.Fatal("expected SpaceReclamationCompleted event not received") + } +} + +func TestEventEmitter_EmitFailure(t *testing.T) { + recorder := record.NewFakeRecorder(10) + emitter := &EventEmitter{recorder: recorder} + pvc := makePVC("test-pvc", "default") + + emitter.EmitFailure(pvc, fmt.Errorf("fstrim failed")) + + select { + case event := <-recorder.Events: + assert.Contains(t, event, EventReasonFailed, "event should contain SpaceReclamationFailed") + case <-time.After(time.Second): + t.Fatal("expected SpaceReclamationFailed event not received") + } +} + +func TestEventEmitter_EmitTimeout(t *testing.T) { + recorder := record.NewFakeRecorder(10) + emitter := &EventEmitter{recorder: recorder} + pvc := makePVC("test-pvc", "default") + + emitter.EmitTimeout(pvc, 3600*time.Second) + + select { + case event := <-recorder.Events: + assert.Contains(t, event, EventReasonTimeout, "event should contain SpaceReclamationTimeout") + case <-time.After(time.Second): + t.Fatal("expected SpaceReclamationTimeout event not received") + } +} + +func TestEventEmitter_EmitUnsupported(t *testing.T) { + recorder := record.NewFakeRecorder(10) + emitter := &EventEmitter{recorder: recorder} + pvc := makePVC("test-pvc", "default") + + emitter.EmitUnsupported(pvc, "discard_max_bytes is 0") + + select { + case event := <-recorder.Events: + assert.Contains(t, event, EventReasonUnsupported, "event should contain SpaceReclamationUnsupported") + case <-time.After(time.Second): + t.Fatal("expected SpaceReclamationUnsupported event not received") + } +} + +// --- Concurrency Tests --- + +func TestSemaphore_LimitsParallelism(t *testing.T) { + sem := make(chan struct{}, 2) + var maxConcurrent int64 + var currentConcurrent int64 + var wg sync.WaitGroup + + for i := 0; i < 5; i++ { + wg.Add(1) + go func() { + defer wg.Done() + sem <- struct{}{} + defer func() { <-sem }() + curr := atomic.AddInt64(¤tConcurrent, 1) + for { + old := atomic.LoadInt64(&maxConcurrent) + if curr <= old || atomic.CompareAndSwapInt64(&maxConcurrent, old, curr) { + break + } + } + time.Sleep(50 * time.Millisecond) + atomic.AddInt64(¤tConcurrent, -1) + }() + } + wg.Wait() + assert.LessOrEqual(t, atomic.LoadInt64(&maxConcurrent), int64(2), + "at most 2 jobs should run concurrently") +} + +func TestPerVolumeMutex_PreventsDuplicateJob(t *testing.T) { + var volumeLocks sync.Map + volID := "vol-dup-001" + + mu := &sync.Mutex{} + actual, loaded := volumeLocks.LoadOrStore(volID, mu) + assert.False(t, loaded, "first lock should not be loaded") + + actualMu := actual.(*sync.Mutex) + actualMu.Lock() + + _, loaded2 := volumeLocks.LoadOrStore(volID, &sync.Mutex{}) + assert.True(t, loaded2, "second lock should find existing entry (duplicate job)") + + actualMu.Unlock() +} + +func TestShutdown_CancelsRunningJobs(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + + jobStarted := make(chan struct{}) + jobDone := make(chan struct{}) + + go func() { + close(jobStarted) + select { + case <-ctx.Done(): + close(jobDone) + case <-time.After(5 * time.Second): + } + }() + + <-jobStarted + cancel() + + select { + case <-jobDone: + assert.True(t, true, "job should be cancelled on shutdown") + case <-time.After(1 * time.Second): + t.Fatal("job was not cancelled within timeout") + } +} + +// --- Manager Initialization Edge Cases --- + +func TestNewSpaceReclamationManager_InvalidCronExpression(t *testing.T) { + cfg := SpaceReclamationConfig{ + Enabled: true, + Schedule: "invalid cron", + MaxConcurrentVolumes: 2, + TimeoutSeconds: 14400, + NodeName: "test-node", + } + fakeClient := fake.NewSimpleClientset() + mgr, err := NewSpaceReclamationManager(context.Background(), cfg, fakeClient, cfg.NodeName, nil) + require.Error(t, err, "invalid cron expression should return error") + require.Nil(t, mgr, "manager should be nil on error") +} + +func TestNewSpaceReclamationManager_ValidConfig(t *testing.T) { + cfg := SpaceReclamationConfig{ + Enabled: true, + Schedule: "0 2 * * 0", + MaxConcurrentVolumes: 2, + TimeoutSeconds: 14400, + NodeName: "test-node", + } + fakeClient := fake.NewSimpleClientset() + mgr, err := NewSpaceReclamationManager(context.Background(), cfg, fakeClient, cfg.NodeName, nil) + require.NoError(t, err) + require.NotNil(t, mgr) +} + +func TestNewSpaceReclamationManager_EmptyNodeName(t *testing.T) { + cfg := SpaceReclamationConfig{ + Enabled: true, + Schedule: "0 2 * * 0", + MaxConcurrentVolumes: 2, + TimeoutSeconds: 14400, + NodeName: "", + } + fakeClient := fake.NewSimpleClientset() + mgr, err := NewSpaceReclamationManager(context.Background(), cfg, fakeClient, cfg.NodeName, nil) + require.NoError(t, err, "empty NodeName should be accepted (graceful degradation)") + require.NotNil(t, mgr, "manager should be created even with empty NodeName") +} + +// --- Environment Variable Constants --- + +func TestEnvVarConstants_Defined(t *testing.T) { + assert.Equal(t, "X_CSI_SPACE_RECLAMATION_ENABLED", EnvSpaceReclamationEnabled) + assert.Equal(t, "X_CSI_SPACE_RECLAMATION_SCHEDULE", EnvSpaceReclamationSchedule) + assert.Equal(t, "X_CSI_SPACE_RECLAMATION_MAX_CONCURRENT", EnvSpaceReclamationMaxConcurrent) + assert.Equal(t, "X_CSI_SPACE_RECLAMATION_TIMEOUT", EnvSpaceReclamationTimeout) +} + +// --- Job-Level Timeout Tests --- + +// TestJobLevelTimeout_ConfigurationApplied verifies timeout configuration is correctly applied +func TestJobLevelTimeout_ConfigurationApplied(t *testing.T) { + cfg := SpaceReclamationConfig{ + Enabled: true, + Schedule: "0 2 * * 0", + MaxConcurrentVolumes: 2, + TimeoutSeconds: 7200, // 2 hours + NodeName: "test-node", + } + + fakeClient := fake.NewSimpleClientset() + mgr := newTestManager(t, fakeClient, cfg) + + // Verify that manager was created with correct timeout config + assert.Equal(t, 7200, mgr.config.TimeoutSeconds, "timeout should be configured correctly") + assert.NotNil(t, mgr.ctx, "manager context should be initialized") +} + +// TestJobLevelTimeout_ShortTimeout verifies manager handles short timeouts +func TestJobLevelTimeout_ShortTimeout(t *testing.T) { + cfg := SpaceReclamationConfig{ + Enabled: true, + Schedule: "0 2 * * 0", + MaxConcurrentVolumes: 1, + TimeoutSeconds: 1, // 1 second timeout + NodeName: "test-node", + } + + fakeClient := fake.NewSimpleClientset() + mgr := newTestManager(t, fakeClient, cfg) + + // Verify short timeout is accepted + assert.Equal(t, 1, mgr.config.TimeoutSeconds, "short timeout should be accepted") +} + +// TestDevicePathNormalization_iSCSIMultipathDevice verifies that device path normalization +// works correctly for iSCSI multipath devices (similar to FC) +func TestDevicePathNormalization_iSCSIMultipathDevice(t *testing.T) { + gofsutil.UseMockFS() + defer resetGofsutilMock() + + pvc := makePVC("pvc-iscsi-001", "default") + pv := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pv-iscsi-001", + }, + Spec: corev1.PersistentVolumeSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Capacity: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("10Gi"), + }, + PersistentVolumeSource: corev1.PersistentVolumeSource{ + CSI: &corev1.CSIPersistentVolumeSource{ + Driver: Name, + VolumeHandle: "CSM-vol001-000120001647-00001", + VolumeAttributes: map[string]string{ + "WWID": "60000970000120001647533030303031", + }, + }, + }, + ClaimRef: &corev1.ObjectReference{ + Name: "pvc-iscsi-001", + Namespace: "default", + }, + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimDelete, + StorageClassName: "powermax-sc", + VolumeMode: &filesystemMode, + }, + Status: corev1.PersistentVolumeStatus{ + Phase: corev1.VolumeBound, + }, + } + + fakeClient := fake.NewSimpleClientset(pvc, pv) + mgr := newTestManager(t, fakeClient, SpaceReclamationConfig{ + Enabled: true, + Schedule: "* * * * *", + MaxConcurrentVolumes: 2, + TimeoutSeconds: 60, + NodeName: "node-1", + }) + + // Mock mount table with /dev/mapper/mpathb (iSCSI multipath device) + gofsutil.GOFSMockMounts = []gofsutil.Info{ + { + Device: "/dev/mapper/mpathb", + Path: "/var/lib/kubelet/plugins/powermax.emc.dell.com/disks/CSM-vol001-000120001647-00001", + Source: "/dev/mapper/mpathb", + }, + } + + // Mock WWNToDevicePath to return /dev/dm-2 + originalWWNToDevicePathFunc := wwnToDevicePathFunc + wwnToDevicePathFunc = func(_ context.Context, _ string) (string, string, error) { + return "60000970000120001647533030303031", "/dev/dm-2", nil + } + defer func() { wwnToDevicePathFunc = originalWWNToDevicePathFunc }() + + // Mock normalizeDevicePath to simulate symlink resolution for iSCSI + originalNormalizeDevicePathFunc := normalizeDevicePathFunc + normalizeDevicePathFunc = func(path string) string { + if path == "/dev/mapper/mpathb" { + return "/dev/dm-2" + } + return path + } + defer func() { normalizeDevicePathFunc = originalNormalizeDevicePathFunc }() + + // Execute one reclamation cycle + mgr.RunOnce() + + // Verify PVC annotations were updated successfully + updatedPVC, err := fakeClient.CoreV1().PersistentVolumeClaims("default").Get( + context.Background(), "pvc-iscsi-001", metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, "success", updatedPVC.Annotations[AnnotationStatus], + "iSCSI multipath device path normalization should work") +} + +// TestDevicePathNormalization_NVMeTCPDevice verifies that device path normalization +// works correctly for NVMe-TCP devices (which don't use multipath symlinks) +func TestDevicePathNormalization_NVMeTCPDevice(t *testing.T) { + gofsutil.UseMockFS() + defer resetGofsutilMock() + + pvc := makePVC("pvc-nvme-001", "default") + pv := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pv-nvme-001", + }, + Spec: corev1.PersistentVolumeSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Capacity: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("10Gi"), + }, + PersistentVolumeSource: corev1.PersistentVolumeSource{ + CSI: &corev1.CSIPersistentVolumeSource{ + Driver: Name, + VolumeHandle: "CSM-vol001-000120001647-00001", + VolumeAttributes: map[string]string{ + "WWID": "60000970000120001647533030303031", + }, + }, + }, + ClaimRef: &corev1.ObjectReference{ + Name: "pvc-nvme-001", + Namespace: "default", + }, + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimDelete, + StorageClassName: "powermax-sc", + VolumeMode: &filesystemMode, + }, + Status: corev1.PersistentVolumeStatus{ + Phase: corev1.VolumeBound, + }, + } + + fakeClient := fake.NewSimpleClientset(pvc, pv) + mgr := newTestManager(t, fakeClient, SpaceReclamationConfig{ + Enabled: true, + Schedule: "* * * * *", + MaxConcurrentVolumes: 2, + TimeoutSeconds: 60, + NodeName: "node-1", + }) + + // Mock mount table with /dev/nvme0n1 (NVMe device - no symlinks) + gofsutil.GOFSMockMounts = []gofsutil.Info{ + { + Device: "/dev/nvme0n1", + Path: "/var/lib/kubelet/plugins/powermax.emc.dell.com/disks/CSM-vol001-000120001647-00001", + Source: "/dev/nvme0n1", + }, + } + + // Mock WWNToDevicePath to return /dev/nvme0n1 (same as mount table) + originalWWNToDevicePathFunc := wwnToDevicePathFunc + wwnToDevicePathFunc = func(_ context.Context, _ string) (string, string, error) { + return "60000970000120001647533030303031", "/dev/nvme0n1", nil + } + defer func() { wwnToDevicePathFunc = originalWWNToDevicePathFunc }() + + // NVMe devices don't have symlinks, so normalization returns the same path + // No need to mock normalizeDevicePathFunc - default behavior is correct + + // Execute one reclamation cycle + mgr.RunOnce() + + // Verify PVC annotations were updated successfully + updatedPVC, err := fakeClient.CoreV1().PersistentVolumeClaims("default").Get( + context.Background(), "pvc-nvme-001", metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, "success", updatedPVC.Annotations[AnnotationStatus], + "NVMe-TCP device path normalization should work (no symlinks)") +} + +// TestDevicePathNormalization_BlockVolume verifies that device path normalization +// works correctly for block volumes (which don't appear in mount table) +func TestDevicePathNormalization_BlockVolume(t *testing.T) { + gofsutil.UseMockFS() + defer resetGofsutilMock() + + pvc := makePVC("pvc-block-001", "default") + pvc.Labels = map[string]string{ + LabelBlockReclaim: "true", + } + blockMode := corev1.PersistentVolumeBlock + pvc.Spec.VolumeMode = &blockMode + + pv := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pv-block-001", + }, + Spec: corev1.PersistentVolumeSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Capacity: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("10Gi"), + }, + PersistentVolumeSource: corev1.PersistentVolumeSource{ + CSI: &corev1.CSIPersistentVolumeSource{ + Driver: Name, + VolumeHandle: "CSM-volblk001-000120001647-00002", + VolumeAttributes: map[string]string{ + "WWID": "60000970000120001647533030303032", + }, + }, + }, + ClaimRef: &corev1.ObjectReference{ + Name: "pvc-block-001", + Namespace: "default", + }, + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimDelete, + StorageClassName: "powermax-sc", + VolumeMode: &blockMode, + }, + Status: corev1.PersistentVolumeStatus{ + Phase: corev1.VolumeBound, + }, + } + + fakeClient := fake.NewSimpleClientset(pvc, pv) + mgr := newTestManager(t, fakeClient, SpaceReclamationConfig{ + Enabled: true, + Schedule: "* * * * *", + MaxConcurrentVolumes: 2, + TimeoutSeconds: 60, + NodeName: "node-1", + }) + + // Block volumes don't appear in mount table + gofsutil.GOFSMockMounts = []gofsutil.Info{} + + // Mock WWNToDevicePath to return /dev/dm-3 (multipath block device) + originalWWNToDevicePathFunc := wwnToDevicePathFunc + wwnToDevicePathFunc = func(_ context.Context, _ string) (string, string, error) { + return "60000970000120001647533030303032", "/dev/dm-3", nil + } + defer func() { wwnToDevicePathFunc = originalWWNToDevicePathFunc }() + + // Execute one reclamation cycle + mgr.RunOnce() + + // Verify PVC annotations were updated successfully + // Block volumes don't need to be in mount table + updatedPVC, err := fakeClient.CoreV1().PersistentVolumeClaims("default").Get( + context.Background(), "pvc-block-001", metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, "success", updatedPVC.Annotations[AnnotationStatus], + "Block volume device path normalization should work (no mount table check)") +} + +// --- Device Path Normalization Tests --- + +// TestDevicePathNormalization_FCMultipathDevice verifies that device path normalization +// correctly handles FC multipath devices where mount table shows /dev/mapper/mpatha +// but WWN resolution returns /dev/dm-1 +func TestDevicePathNormalization_FCMultipathDevice(t *testing.T) { + gofsutil.UseMockFS() + defer resetGofsutilMock() + + pvc := makePVC("pvc-fc-001", "default") + pv := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pv-fc-001", + }, + Spec: corev1.PersistentVolumeSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Capacity: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("10Gi"), + }, + PersistentVolumeSource: corev1.PersistentVolumeSource{ + CSI: &corev1.CSIPersistentVolumeSource{ + Driver: Name, + VolumeHandle: "CSM-vol001-000120001647-00001", + VolumeAttributes: map[string]string{ + "WWID": "60000970000120001647533030303031", + }, + }, + }, + ClaimRef: &corev1.ObjectReference{ + Name: "pvc-fc-001", + Namespace: "default", + }, + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimDelete, + StorageClassName: "powermax-sc", + VolumeMode: &filesystemMode, + }, + Status: corev1.PersistentVolumeStatus{ + Phase: corev1.VolumeBound, + }, + } + + fakeClient := fake.NewSimpleClientset(pvc, pv) + mgr := newTestManager(t, fakeClient, SpaceReclamationConfig{ + Enabled: true, + Schedule: "* * * * *", + MaxConcurrentVolumes: 2, + TimeoutSeconds: 60, + NodeName: "node-1", + }) + + // Mock mount table with /dev/mapper/mpatha (symlink to dm-1) + // This simulates the real-world FC multipath scenario + gofsutil.GOFSMockMounts = []gofsutil.Info{ + { + Device: "/dev/mapper/mpatha", + Path: "/var/lib/kubelet/plugins/powermax.emc.dell.com/disks/CSM-vol001-000120001647-00001", + Source: "/dev/mapper/mpatha", + }, + } + + // Mock WWNToDevicePath to return /dev/dm-1 (the actual device, not the symlink) + // This simulates what gofsutil.WWNToDevicePathX returns in real environments + originalWWNToDevicePathFunc := wwnToDevicePathFunc + wwnToDevicePathFunc = func(_ context.Context, _ string) (string, string, error) { + return "60000970000120001647533030303031", "/dev/dm-1", nil + } + defer func() { wwnToDevicePathFunc = originalWWNToDevicePathFunc }() + + // Mock normalizeDevicePath to simulate symlink resolution + // In real environments, /dev/mapper/mpatha is a symlink to ../dm-1 + originalNormalizeDevicePathFunc := normalizeDevicePathFunc + normalizeDevicePathFunc = func(path string) string { + if path == "/dev/mapper/mpatha" { + return "/dev/dm-1" + } + return path + } + defer func() { normalizeDevicePathFunc = originalNormalizeDevicePathFunc }() + + // Execute one reclamation cycle + mgr.RunOnce() + + // Verify PVC annotations were updated successfully + // This proves that device path normalization worked correctly + updatedPVC, err := fakeClient.CoreV1().PersistentVolumeClaims("default").Get( + context.Background(), "pvc-fc-001", metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, "success", updatedPVC.Annotations[AnnotationStatus], + "PVC should be annotated with status=success after fstrim (device path normalization worked)") + assert.NotEmpty(t, updatedPVC.Annotations[AnnotationBytesAvailable], + "PVC should have bytes-available annotation") + assert.NotEmpty(t, updatedPVC.Annotations[AnnotationLastRunTime], + "PVC should have last-run-time annotation") + assert.Equal(t, "node-1", updatedPVC.Annotations[AnnotationNode], + "PVC should have node annotation") +} diff --git a/service/step_defs_test.go b/service/step_defs_test.go index 39d70b9f..57bc647c 100644 --- a/service/step_defs_test.go +++ b/service/step_defs_test.go @@ -16,6 +16,7 @@ limitations under the License. package service import ( + "context" "encoding/json" "errors" "fmt" @@ -40,6 +41,7 @@ import ( "github.com/dell/csi-powermax/v2/k8smock" "github.com/dell/csi-powermax/v2/pkg/migration" + "github.com/dell/csi-powermax/v2/pkg/symmetrix" "github.com/dell/dell-csi-extensions/common" @@ -57,7 +59,6 @@ import ( types "github.com/dell/gopowermax/v2/types/v100" csi "github.com/container-storage-interface/spec/lib/go/csi" "github.com/cucumber/godog" - "golang.org/x/net/context" "google.golang.org/grpc/metadata" csimgr "github.com/dell/dell-csi-extensions/migration" @@ -175,6 +176,7 @@ type feature struct { noNodeID bool omitAccessMode, omitVolumeCapability bool wrongCapacity, wrongStoragePool bool + largerCapacity bool useAccessTypeMount bool capability *csi.VolumeCapability capabilities []*csi.VolumeCapability @@ -213,6 +215,8 @@ type feature struct { setIOLimits bool validateVHCResp *podmon.ValidateVolumeHostConnectivityResponse arrayMigrateResponse *csimgr.ArrayMigrateResponse + podmonServer *httptest.Server + dbusNewOriginal func() (dBusConn, error) // systemd mocking } var inducedErrors struct { @@ -276,6 +280,10 @@ func (f *feature) aPowerMaxService() error { maxRemoveGroupSize = 10 f.maxRetryCount = MaxRetries enableBatchGetMaskingViewConnections = true + // Stop the previous scenario's deletion worker to prevent goroutine accumulation + if lastDeletionWorker != nil { + lastDeletionWorker.Stop() + } f.checkGoRoutines("start aPowerMaxService") // Save off the admin client and the system if f.service != nil && f.service.adminClient != nil { @@ -287,6 +295,11 @@ func (f *feature) aPowerMaxService() error { pmaxCache = make(map[string]*pmaxCachedInformation) } + // Clear snapshot license cache to ensure test isolation + for k := range symmRepCapabilities { + delete(symmRepCapabilities, k) + } + nodeCache = sync.Map{} f.err = nil f.symmetrixID = mock.DefaultSymmetrixID @@ -311,6 +324,7 @@ func (f *feature) aPowerMaxService() error { f.omitVolumeCapability = false f.useAccessTypeMount = false f.wrongCapacity = false + f.largerCapacity = false f.wrongStoragePool = false f.deleteVolumeRequest = nil f.deleteLocalVolumeRequest = nil @@ -351,6 +365,9 @@ func (f *feature) aPowerMaxService() error { f.nodeGetVolumeStatsResponse = nil f.setIOLimits = false f.validateVHCResp = nil + for k := range symmRepCapabilities { + delete(symmRepCapabilities, k) + } inducedErrors.invalidSymID = false inducedErrors.invalidStoragePool = false inducedErrors.invalidServiceLevel = false @@ -401,7 +418,7 @@ func (f *feature) aPowerMaxService() error { // get or reuse the cached service f.getService() f.service.storagePoolCacheDuration = 4 * time.Hour - f.service.SetPmaxTimeoutSeconds(3) + pmaxQueryAttempts = 1 // create the mock iscsi client f.service.iscsiClient = goiscsi.NewMockISCSI(map[string]string{}) @@ -477,6 +494,7 @@ func (f *feature) aPowerMaxService() error { f.checkGoRoutines("end aPowerMaxService") symIDs := f.service.retryableGetSymmetrixIDList() f.service.NewDeletionWorker(f.service.opts.ClusterPrefix, symIDs.SymmetrixIDs) + lastDeletionWorker = f.service.deletionWorker f.errType = "" // Configure ManifestSemver @@ -501,6 +519,7 @@ func (f *feature) getService() *service { svc.iscsiTargets = map[string][]string{} svc.nvmeTargets = new(sync.Map) svc.loggedInNVMeArrays = map[string]bool{} + svc.versionCache = newVersionCache() var opts Opts opts.User = "username" opts.Password = "password" @@ -540,11 +559,19 @@ ip6t_rpfilter 12595 1 svc.iscsiConnector = &mockISCSIGobrick{} svc.nvmetcpClient = &gonvme.MockNVMe{} svc.nvmeTCPConnector = &mockNVMeTCPConnector{} - svc.dBusConn = &mockDbusConnection{} + svc.k8sUtils = k8smock.Init() mockGobrickReset() mockgosystemdReset() disconnectVolumeRetryTime = 10 * time.Millisecond + + // Mock iscsid.service status + // dbusNewConnectionFunc is reset to its original value in the post-test hook + f.dbusNewOriginal = dbusNewConnectionFunc + dbusNewConnectionFunc = func() (dBusConn, error) { + return &mockDbusConnection{}, nil + } + f.service = svc return svc } @@ -554,6 +581,33 @@ func (f *feature) aPostELMSRArray() error { return nil } +func (f *feature) aU4P104Array() error { + f.symmetrixID = mock.DefaultEnhancedSymmetrixID104 + // Dynamically add the 10.4 array to the symmetrix package if not already present. + // This avoids adding it to the default ManagedArrays which would spawn extra + // background goroutines (deletion workers, etc.) for every test scenario. + err := symmetrix.Initialize([]string{mock.DefaultEnhancedSymmetrixID104}, f.service.adminClient) + if err != nil && !strings.Contains(err.Error(), "already added") { + return err + } + // Seed the version cache with 10.4 details for both: + // 1) admin client key and + // 2) resolved per-array client key returned by symmetrix.GetPowerMaxClient. + // This avoids dependence on HTTP /version interception timing and ensures + // deterministic 10.4 path selection for this scenario. + if f.service != nil && f.service.versionCache != nil { + versionDetails := &types.VersionDetails{Version: "10.4.0.4", APIVersion: "104"} + if f.service.adminClient != nil { + f.service.versionCache.versionEntries.Store(symmetrixKey(f.service.adminClient, mock.DefaultEnhancedSymmetrixID104), versionDetails) + } + pmaxClient, clientErr := symmetrix.GetPowerMaxClient(mock.DefaultEnhancedSymmetrixID104) + if clientErr == nil { + f.service.versionCache.versionEntries.Store(symmetrixKey(pmaxClient, mock.DefaultEnhancedSymmetrixID104), versionDetails) + } + } + return nil +} + // GetPluginInfo func (f *feature) iCallGetPluginInfo() error { header := metadata.New(map[string]string{"csi.requestid": "1"}) @@ -698,6 +752,7 @@ func (f *feature) aValidProbeResponseIsReturned() error { } func (f *feature) theErrorContains(arg1 string) error { + arg1 = strings.Trim(arg1, `"`) f.checkGoRoutines("theErrorContains") // If arg1 is none, we expect no error, any error received is unexpected if arg1 == "none" { @@ -715,7 +770,7 @@ func (f *feature) theErrorContains(arg1 string) error { return nil } } - return fmt.Errorf("Expected error to contain %s but no error", arg1) + return fmt.Errorf("Expected error to contain \"%s\" but got none", arg1) } // Allow for multiple possible matches, separated by @@. This was necessary // because Windows and Linux sometimes return different error strings for @@ -728,10 +783,11 @@ func (f *feature) theErrorContains(arg1 string) error { return nil } } - return fmt.Errorf("Expected error to contain %s but it was %s", arg1, f.err.Error()) + return fmt.Errorf("Expected error to contain \"%s\" but it was \"%s\"", arg1, f.err.Error()) } func (f *feature) thePossibleErrorContains(arg1 string) error { + arg1 = strings.Trim(arg1, `"`) if f.err == nil { return nil } @@ -1179,6 +1235,14 @@ func (f *feature) iInduceError(errtype string) error { inducedErrors.noDeviceWWNError = true case "PortGroupError": inducedErrors.portGroupError = true + case "NoVolumeSource": + inducedErrors.noVolumeSource = true + case "NonExistentVolume": + inducedErrors.nonExistentVolume = true + case "WrongCapacity": + f.wrongCapacity = true + case "WrongStoragePool": + f.wrongStoragePool = true case "GetVolumeIteratorError": mock.SafeSetInducedError(mock.InducedErrors, "GetVolumeIteratorError", true) case "GetVolumeError": @@ -1233,6 +1297,10 @@ func (f *feature) iInduceError(errtype string) error { mock.SafeSetInducedError(mock.InducedErrors, "GetInitiatorError", true) case "GetInitiatorByIDError": mock.SafeSetInducedError(mock.InducedErrors, "GetInitiatorByIDError", true) + case "CreateVolumeError": + mock.SafeSetInducedError(mock.InducedErrors, "CreateVolumeError", true) + case "PublishMaskingViewsError": + mock.SafeSetInducedError(mock.InducedErrors, "PublishMaskingViewsError", true) case "CreateSnapshotError": mock.SafeSetInducedError(mock.InducedErrors, "CreateSnapshotError", true) case "DeleteSnapshotError": @@ -1241,6 +1309,9 @@ func (f *feature) iInduceError(errtype string) error { mock.SafeSetInducedError(mock.InducedErrors, "LinkSnapshotError", true) case "SnapshotNotLicensed": mock.SafeSetInducedError(mock.InducedErrors, "SnapshotNotLicensed", true) + for k := range symmRepCapabilities { + delete(symmRepCapabilities, k) + } case "InvalidResponse": mock.SafeSetInducedError(mock.InducedErrors, "InvalidResponse", true) case "UnisphereMismatchError": @@ -1528,6 +1599,16 @@ func (f *feature) iInduceError(errtype string) error { migration.AddVolumesToRemoteSG = func(_ context.Context, _ string, _ pmax.Pmax) (bool, error) { return true, nil } + case "nonExistentVolume": + inducedErrors.nonExistentVolume = true + case "invalidVolumeID": + inducedErrors.invalidVolumeID = true + case "wrongCapacity": + f.wrongCapacity = true + case "wrongStoragePool": + f.wrongStoragePool = true + case "noVolumeSource": + inducedErrors.noVolumeSource = true case "none": return nil default: @@ -1843,6 +1924,7 @@ func (f *feature) iAddTheVolumeTo(_ string) error { } func (f *feature) iCallPublishVolumeWithTo(accessMode, nodeID string) error { + accessMode = strings.Trim(accessMode, `"`) header := metadata.New(map[string]string{"csi.requestid": "1"}) ctx := metadata.NewIncomingContext(context.Background(), header) req := f.publishVolumeRequest @@ -1894,11 +1976,6 @@ func (f *feature) aValidVolumeWithSizeOfCYL(nCYL int) error { return nil } -func (f *feature) anInvalidVolume() error { - inducedErrors.invalidVolumeID = true - return nil -} - func (f *feature) anInvalidSnapshot() error { inducedErrors.invalidSnapID = true return nil @@ -3213,8 +3290,11 @@ func (f *feature) iCallCreateVolumeFromSnapshot() error { if f.wrongCapacity { req.CapacityRange.RequiredBytes = 64 * 1024 * 1024 * 1024 } + if f.largerCapacity { + req.CapacityRange.RequiredBytes = 200 * 1024 * 1024 * 1024 + } if f.wrongStoragePool { - req.Parameters["storagepool"] = "bad storage pool" + req.Parameters[StoragePoolParam] = "bad storage pool" } var snapshotID string if inducedErrors.invalidSnapID { @@ -3240,8 +3320,11 @@ func (f *feature) iCallCreateVolumeFromVolume() error { if f.wrongCapacity { req.CapacityRange.RequiredBytes = 64 * 1024 * 1024 * 1024 } + if f.largerCapacity { + req.CapacityRange.RequiredBytes = 200 * 1024 * 1024 * 1024 + } if f.wrongStoragePool { - req.Parameters["storagepool"] = "bad storage pool" + req.Parameters[StoragePoolParam] = "bad storage pool" } var volumeID string if inducedErrors.noVolumeSource { @@ -3265,13 +3348,8 @@ func (f *feature) iCallCreateVolumeFromVolume() error { return nil } -func (f *feature) theWrongCapacity() error { - f.wrongCapacity = true - return nil -} - -func (f *feature) theWrongStoragePool() error { - f.wrongStoragePool = true +func (f *feature) aLargerCapacityThanTheSource() error { + f.largerCapacity = true return nil } @@ -3395,10 +3473,10 @@ func (f *feature) deletionWorkerProcessesWhichResultsIn(volumeName, errormsg str } // We expected an error if vol.Status.ErrorMsgs == nil { - return fmt.Errorf("Expected error %s but got none", errormsg) + return fmt.Errorf("Expected error \"%s\" but got none", errormsg) } if !hasError(vol.Status.ErrorMsgs, errormsg) { - return fmt.Errorf("Expected error to contain %s: but got: %s", errormsg, vol.Status.ErrorMsgs) + return fmt.Errorf("Expected error to contain \"%s\" but got \"%s\"", errormsg, vol.Status.ErrorMsgs) } return nil } @@ -3445,25 +3523,30 @@ func (f *feature) volumesAreBeingProcessedForDeletion(nVols int) error { if f.err != nil { return nil } - retry := 5 - // Count the number of volumes in the delWorker queue - cnt := 0 - for retryno := 0; retryno < retry; retryno++ { + minExpected := nVols - 2 + maxExpected := nVols + if minExpected < 0 { + minExpected = 0 + } + deadline := time.Now().Add(30 * time.Second) + + // Count the number of volumes in the delWorker queue, allowing time for async processing. + for { + cnt := 0 for _, dQ := range f.service.deletionWorker.DeletionQueues { dQ.Print() - cnt = cnt + len(dQ.GetDeviceList()) + cnt += len(dQ.GetDeviceList()) break } - if cnt > 0 { - break + if cnt >= minExpected && cnt <= maxExpected { + fmt.Println("Expected count reached") + return nil } - time.Sleep(time.Second * 1) - } - if cnt < (nVols-2) || cnt > nVols { - return fmt.Errorf("Expected at least %d volumes and not more than %d volumes in deletion queue but got %d", nVols-2, nVols, cnt) + if time.Now().After(deadline) { + return fmt.Errorf("Expected at least %d volumes and not more than %d volumes in deletion queue but got %d", minExpected, maxExpected, cnt) + } + time.Sleep(500 * time.Millisecond) } - fmt.Println("Expected count reached") - return nil } func (f *feature) iRequestAPortGroup() error { @@ -3485,7 +3568,6 @@ func (f *feature) aValidPortGroupIsReturned() error { } func (f *feature) iInvokeCreateOrUpdateIscsiHost(hostName string) error { - f.service.SetPmaxTimeoutSeconds(3) symID := f.symmetrixID if inducedErrors.noSymID { symID = "" @@ -3508,12 +3590,15 @@ func (f *feature) iInvokeCreateOrUpdateIscsiHost(hostName string) error { initiators = initiators[:0] } f.host, f.err = f.service.createOrUpdateIscsiHost(context.Background(), symID, hostID, initiators, f.service.adminClient) - f.initiators = f.host.Initiators + if f.err != nil || f.host == nil { + f.initiators = []string{} + } else { + f.initiators = f.host.Initiators + } return nil } func (f *feature) iInvokeCreateOrUpdateFCHost(hostName string) error { - f.service.SetPmaxTimeoutSeconds(3) symID := f.symmetrixID if inducedErrors.noSymID { symID = "" @@ -3561,7 +3646,6 @@ func (f *feature) iInvokeNodeHostSetupWithAService(mode string) error { f.service.useIscsi = false f.service.useFC = false f.service.useNVMeTCP = false - f.service.SetPmaxTimeoutSeconds(10) f.err = f.service.nodeHostSetup(context.Background(), fcInitiators, iscsiInitiators, nvmetcpinitiators, symmetrixIDs) return nil } @@ -3743,7 +3827,6 @@ func (f *feature) thereAreNoArraysLoggedIn() error { } func (f *feature) arraysAreLoggedInWithProtocol(protocol string) error { - f.service.SetPmaxTimeoutSeconds(3) s.cacheMutex.Lock() defer s.cacheMutex.Unlock() if protocol == "FC" { @@ -3759,7 +3842,6 @@ func (f *feature) arraysAreLoggedInWithProtocol(protocol string) error { } func (f *feature) iInvokeEnsureLoggedIntoEveryArray() error { - f.service.SetPmaxTimeoutSeconds(3) isSymConnFC.Clear() // Ensure none of the other test marked the array as FC f.err = f.service.ensureLoggedIntoEveryArray(context.Background(), false) @@ -4183,16 +4265,6 @@ func (f *feature) aValidDeleteSnapshotResponseIsReturned() error { return nil } -func (f *feature) aNonexistentVolume() error { - inducedErrors.nonExistentVolume = true - return nil -} - -func (f *feature) noVolumeSource() error { - inducedErrors.noVolumeSource = true - return nil -} - func (f *feature) validateSnapshotLicenseCache(count int) error { if len(symmRepCapabilities) != count { return fmt.Errorf("Expected %d array(s) in the license cache but got %d", count, len(symmRepCapabilities)) @@ -4316,10 +4388,12 @@ func (f *feature) iCallRequestAddVolumeToSGMVMv(nodeID, maskingViewName string) fmt.Printf("deviceID %s\n", deviceID) if maskingViewName != "" && maskingViewName != "default" { f.addVolumeToSGMVResponse2, f.lockChan, f.err = f.service.sgSvc.requestAddVolumeToSGMV( - context.Background(), f.sgID, maskingViewName, f.hostID, "0001", mock.DefaultSymmetrixID, mock.DefaultSymmetrixID, deviceID, "fake", accessMode) + context.Background(), f.sgID, maskingViewName, f.hostID, "0001", mock.DefaultSymmetrixID, mock.DefaultSymmetrixID, deviceID, "fake", accessMode, + ) } else { f.addVolumeToSGMVResponse1, f.lockChan, f.err = f.service.sgSvc.requestAddVolumeToSGMV( - context.Background(), f.sgID, f.mvID, f.hostID, "0001", mock.DefaultSymmetrixID, mock.DefaultSymmetrixID, deviceID, "fake", accessMode) + context.Background(), f.sgID, f.mvID, f.hostID, "0001", mock.DefaultSymmetrixID, mock.DefaultSymmetrixID, deviceID, "fake", accessMode, + ) } return nil } @@ -4869,7 +4943,7 @@ func (f *feature) iCallRDFEnabledCreateVolumeFromSnapshot(volName, namespace, mo req.CapacityRange.RequiredBytes = 64 * 1024 * 1024 * 1024 } if f.wrongStoragePool { - req.Parameters["storagepool"] = "bad storage pool" + req.Parameters[StoragePoolParam] = "bad storage pool" } var snapshotID string if inducedErrors.invalidSnapID { @@ -4905,7 +4979,7 @@ func (f *feature) iCallRDFEnabledCreateVolumeFromVolume(volName, namespace, mode req.CapacityRange.RequiredBytes = 64 * 1024 * 1024 * 1024 } if f.wrongStoragePool { - req.Parameters["storagepool"] = "bad storage pool" + req.Parameters[StoragePoolParam] = "bad storage pool" } var volumeID string if inducedErrors.noVolumeSource { @@ -5172,26 +5246,42 @@ func (f *feature) iCallValidateVolumeHostConnectivityWithAndSymID(nodeID, symID return nil } +func (f *feature) startPodmonServer(url string, handler func(http.ResponseWriter, *http.Request)) { + mux := http.NewServeMux() + // responding with some dummy response that is for the case when array is connected and LastSuccess check was just finished + mux.HandleFunc(url, handler) + // Start test server synchronously + f.podmonServer = httptest.NewServer(mux) + // Get the allocated port number + addr := f.podmonServer.Listener.Addr().String() + f.service.opts.PodmonPort = ":" + strings.Split(addr, ":")[1] + + fmt.Printf("Started fake podmon server at port %s\n", f.service.opts.PodmonPort) +} + func (f *feature) iStartNodeAPIServer() { var status ArrayConnectivityStatus status.LastAttempt = time.Now().Unix() status.LastSuccess = time.Now().Unix() input, _ := json.Marshal(status) - // responding with some dummy response that is for the case when array is connected and LastSuccess check was just finished - http.HandleFunc(ArrayStatus+"/"+f.symmetrixID, func(w http.ResponseWriter, _ *http.Request) { - w.Write(input) + f.startPodmonServer(ArrayStatus+"/"+f.symmetrixID, func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write(input) }) - - f.service.opts.PodmonPort = ":9028" - fmt.Printf("Starting server at port %s\n", f.service.opts.PodmonPort) - // http.ListenAndServe(f.service.opts.PodmonPort, nil) // #nosec G114 - go listenAndServe(f.service.opts.PodmonPort) } -func listenAndServe(port string) { - err := http.ListenAndServe(port, nil) // #nosec G114 - fmt.Println("Error with listen and serve: ", err) +func findAvailablePort() string { + // Create a temporary listener with dynamic port allocation + // to see what port is not currently in use on this machine + listener, err := net.Listen("tcp", "localhost:0") + if err != nil { + fmt.Println("Failed to get a dynamic port, fallback to port 9999") + return ":9999" + } + // Get the dynamically allocated port and close the listener immediately + addr := listener.Addr().String() + listener.Close() + return ":" + strings.Split(addr, ":")[1] } func (f *feature) iCallQueryArrayStatus(url string, statusType string) { @@ -5212,17 +5302,15 @@ func (f *feature) iCallQueryArrayStatus(url string, statusType string) { input = nil } - // responding with some dummy response that is for the case when array is connected and LastSuccess check was just finished - http.HandleFunc(url, func(w http.ResponseWriter, _ *http.Request) { + f.startPodmonServer(url, func(w http.ResponseWriter, _ *http.Request) { if inducedErrors.unexpectedResponse { w.WriteHeader(http.StatusBadRequest) } - w.Write(input) + _, _ = w.Write(input) }) - - f.service.opts.PodmonPort = ":9028" - fmt.Printf("Starting server at port %s\n", f.service.opts.PodmonPort) - go http.ListenAndServe(f.service.opts.PodmonPort, nil) // #nosec G114 + } else { + // Assign a port on which no one is listening + f.service.opts.PodmonPort = findAvailablePort() } _, f.err = f.service.QueryArrayStatus(context.TODO(), "http://localhost"+f.service.opts.PodmonPort+url) @@ -5240,7 +5328,7 @@ func (f *feature) iCallIsIOInProgress() error { func (f *feature) theValidateVolumeHostMessageContains(msg string) error { if !strings.Contains(f.validateVHCResp.Messages[0], msg) { - errMsg := fmt.Sprintf("validateVolumeHostConnectivity response is incorrect, expected: %s actual %s", msg, f.validateVHCResp.Messages[0]) + errMsg := fmt.Sprintf("validateVolumeHostConnectivity response is incorrect, expected: \"%s\" actual \"%s\"", msg, f.validateVHCResp.Messages[0]) return errors.New(errMsg) } return nil @@ -5342,16 +5430,31 @@ func (f *feature) restartDriver() error { func FeatureContext(s *godog.ScenarioContext) { f := &feature{} + + // Post-scenario tear-down/cleanup + s.After(func(ctx context.Context, _ *godog.Scenario, _ error) (context.Context, error) { + if f != nil && f.podmonServer != nil { + fmt.Println("Stopping fake podmon server") + f.podmonServer.Close() + f.podmonServer = nil + } + dbusNewConnectionFunc = f.dbusNewOriginal + return ctx, nil + }) + s.Step(`^a PowerMax service$`, f.aPowerMaxService) s.Step(`^a PostELMSR Array$`, f.aPostELMSRArray) + s.Step(`^a 104 array$`, f.aU4P104Array) s.Step(`^I call GetPluginInfo$`, f.iCallGetPluginInfo) s.Step(`^a valid GetPluginInfoResponse is returned$`, f.aValidGetPluginInfoResponseIsReturned) s.Step(`^I call GetPluginCapabilities$`, f.iCallGetPluginCapabilities) s.Step(`^a valid GetPluginCapabilitiesResponse is returned$`, f.aValidGetPluginCapabilitiesResponseIsReturned) s.Step(`^I call Probe$`, f.iCallProbe) s.Step(`^a valid ProbeResponse is returned$`, f.aValidProbeResponseIsReturned) - s.Step(`^the error contains "([^"]*)"$`, f.theErrorContains) - s.Step(`^the possible error contains "([^"]*)"$`, f.thePossibleErrorContains) + s.Step(`^the error contains "?([^"]*)"?$`, f.theErrorContains) + s.Step(`^the error contains (.+)$`, f.theErrorContains) + s.Step(`^the possible error contains "?([^"]*)"?$`, f.thePossibleErrorContains) + s.Step(`^the possible error contains (.+)$`, f.thePossibleErrorContains) s.Step(`^the Controller has no connection$`, f.theControllerHasNoConnection) s.Step(`^there is a Node Probe Lsmod error$`, f.thereIsANodeProbeLsmodError) s.Step(`^I call CreateVolume "([^"]*)"$`, f.iCallCreateVolume) @@ -5366,14 +5469,14 @@ func FeatureContext(s *godog.ScenarioContext) { s.Step(`^I specify NoStoragePool$`, f.iSpecifyNoStoragePool) s.Step(`^I call CreateVolumeSize "([^"]*)" "(\d+)"$`, f.iCallCreateVolumeSize) s.Step(`^I change the StoragePool "([^"]*)"$`, f.iChangeTheStoragePool) - s.Step(`^I induce error "([^"]*)"$`, f.iInduceError) + s.Step(`^I induce error "?([^"]*)"?$`, f.iInduceError) s.Step(`^I specify VolumeContentSource$`, f.iSpecifyVolumeContentSource) s.Step(`^I specify CreateVolumeMountRequest "([^"]*)"$`, f.iSpecifyCreateVolumeMountRequest) s.Step(`^I call PublishVolume with "([^"]*)" to "([^"]*)"$`, f.iCallPublishVolumeWithTo) + s.Step(`^I call PublishVolume with (.+) to "([^"]*)"$`, f.iCallPublishVolumeWithTo) s.Step(`^a valid PublishVolumeResponse is returned$`, f.aValidPublishVolumeResponseIsReturned) s.Step(`^a valid volume$`, f.aValidVolume) s.Step(`^a valid volume with size of (\d+) CYL$`, f.aValidVolumeWithSizeOfCYL) - s.Step(`^an invalid volume$`, f.anInvalidVolume) s.Step(`^an invalid snapshot$`, f.anInvalidSnapshot) s.Step(`^no volume$`, f.noVolume) s.Step(`^no node$`, f.noNode) @@ -5436,8 +5539,7 @@ func FeatureContext(s *godog.ScenarioContext) { s.Step(`^I call RemoveSnapshot "([^"]*)"$`, f.iCallRemoveSnapshot) s.Step(`^a valid snapshot consistency group$`, f.aValidSnapshotConsistencyGroup) s.Step(`^I call Create Volume from Snapshot$`, f.iCallCreateVolumeFromSnapshot) - s.Step(`^the wrong capacity$`, f.theWrongCapacity) - s.Step(`^the wrong storage pool$`, f.theWrongStoragePool) + s.Step(`^a larger capacity than the source$`, f.aLargerCapacityThanTheSource) s.Step(`^there are (\d+) valid snapshots of "([^"]*)" volume$`, f.thereAreValidSnapshotsOfVolume) s.Step(`^I call ListSnapshots$`, f.iCallListSnapshots) s.Step(`^I call ListSnapshots with max_entries "([^"]*)" and starting_token "([^"]*)"$`, f.iCallListSnapshotsWithMaxEntriesAndStartingToken) @@ -5517,8 +5619,6 @@ func FeatureContext(s *godog.ScenarioContext) { s.Step(`^I call TerminateSnapshot$`, f.iCallTerminateSnapshot) s.Step(`^I call UnlinkAndTerminate snapshot$`, f.iCallUnlinkAndTerminateSnapshot) s.Step(`^a valid DeleteSnapshotResponse is returned$`, f.aValidDeleteSnapshotResponseIsReturned) - s.Step(`^a non-existent volume$`, f.aNonexistentVolume) - s.Step(`^no volume source$`, f.noVolumeSource) s.Step(`^the snapshot license cache has "(\d+)" array$`, f.validateSnapshotLicenseCache) s.Step(`^I reset the license cache$`, f.iResetTheLicenseCache) s.Step(`^I call IsSnapshotSource$`, f.iCallIsSnapshotSource) diff --git a/service/version_cache.go b/service/version_cache.go new file mode 100644 index 00000000..7b0cef4d --- /dev/null +++ b/service/version_cache.go @@ -0,0 +1,84 @@ +/* +Copyright © 2021-2026 Dell Inc. or its subsidiaries. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package service + +import ( + "context" + "fmt" + "sync" + + pmax "github.com/dell/gopowermax/v2" + types "github.com/dell/gopowermax/v2/types/v100" +) + +// versionCache caches VersionDetails and Symmetrix info per Unisphere client instance. +// Cache is populated once and lives for the lifetime of the driver process (no TTL). +// Keys are formed from the client's underlying HTTP client pointer to isolate per-Unisphere +// connections, ensuring multi-array, multi-Unisphere, and backup-Unisphere safety. +// sync.Map provides concurrent-safe reads with no starvation or deadlocks. +type versionCache struct { + // versionEntries: key = fmt.Sprintf("%p", httpClient) -> *types.VersionDetails + versionEntries sync.Map + // symmetrixEntries: key = fmt.Sprintf("%p:%s", httpClient, symID) -> *types.Symmetrix + symmetrixEntries sync.Map +} + +func newVersionCache() *versionCache { + return &versionCache{} +} + +func clientKey(pmaxClient pmax.Pmax) string { + return fmt.Sprintf("%p", pmaxClient.GetHTTPClient()) +} + +func symmetrixKey(pmaxClient pmax.Pmax, symID string) string { + return fmt.Sprintf("%p:%s", pmaxClient.GetHTTPClient(), symID) +} + +// getOrFetchVersionDetails returns cached VersionDetails, or fetches and caches it. +// Errors are never cached — a subsequent call will retry the fetch. +// Version is cached per client+SymID to ensure multi-array safety when arrays share a client. +func (c *versionCache) getOrFetchVersionDetails(ctx context.Context, symID string, pmaxClient pmax.Pmax) (*types.VersionDetails, error) { + key := symmetrixKey(pmaxClient, symID) + if v, ok := c.versionEntries.Load(key); ok { + return v.(*types.VersionDetails), nil + } + details, err := pmaxClient.GetVersionDetails(ctx) + if err != nil { + return nil, err + } + // Store a copy to avoid callers mutating the cached value. + cached := *details + c.versionEntries.Store(key, &cached) + return details, nil +} + +// getOrFetchSymmetrix returns cached Symmetrix, or fetches and caches it. +// Errors are never cached — a subsequent call will retry the fetch. +func (c *versionCache) getOrFetchSymmetrix(ctx context.Context, symID string, pmaxClient pmax.Pmax) (*types.Symmetrix, error) { + key := symmetrixKey(pmaxClient, symID) + if v, ok := c.symmetrixEntries.Load(key); ok { + return v.(*types.Symmetrix), nil + } + sym, err := pmaxClient.GetSymmetrixByID(ctx, symID) + if err != nil { + return nil, err + } + // Store a copy to avoid callers mutating the cached value. + cached := *sym + c.symmetrixEntries.Store(key, &cached) + return sym, nil +} diff --git a/service/volume_creator.go b/service/volume_creator.go new file mode 100644 index 00000000..3841598d --- /dev/null +++ b/service/volume_creator.go @@ -0,0 +1,1445 @@ +package service + +import ( + "context" + "fmt" + "path" + "strconv" + "strings" + "time" + + "github.com/dell/csi-powermax/v2/pkg/file" + + pmax "github.com/dell/gopowermax/v2" + types "github.com/dell/gopowermax/v2/types/v100" + "github.com/container-storage-interface/spec/lib/go/csi" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// --------------------------------------------------------------------------- +// Struct definitions +// --------------------------------------------------------------------------- + +type u4p104VolumeCreator struct { + s U4P104ServiceDeps + pmaxClient pmax.Pmax // standard client used for idempotency fallback (GetVolumesByIdentifier) + pmaxClient104 pmax.Pmax // 10.4-capable client used for CreateVolume + symmetrixID string + reqID string + params map[string]string + symmIDFoundInAZ bool + apiVersion int +} + +type legacyVolumeCreator struct { + s *service + pmaxClient pmax.Pmax + symmetrixID string + reqID string + params map[string]string + symmIDFoundInAZ bool + apiVersion int +} + +// --------------------------------------------------------------------------- +// 10.4 volume creation constants +// --------------------------------------------------------------------------- + +const ( + volumeStorageGroupActionAdd = "Add" + createVolumesResponseSelect = "id,identifier,cap_cyl,storage_groups" +) + +// --------------------------------------------------------------------------- +// 10.4 volume creation +// --------------------------------------------------------------------------- + +func (c *u4p104VolumeCreator) Create(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { + log := log.WithContext(ctx) + var err error + if req == nil { + return nil, status.Error(codes.InvalidArgument, "create volume request cannot be nil") + } + + params := req.GetParameters() + accessibility := req.GetAccessibilityRequirements() + + volName := req.GetName() + if volName == "" { + return nil, status.Error(codes.InvalidArgument, "volume name is required") + } + + // Block capability validation — match legacy behavior + vcs := req.GetVolumeCapabilities() + if vcs != nil && accTypeIsBlock(vcs) && !c.s.isBlockEnabled() { + return nil, status.Error(codes.InvalidArgument, "Block Volume Capability is not supported") + } + + storagePoolID := c.s.resolveParameter(params, c.symmetrixID, StoragePoolParam, "") + if storagePoolID == "" { + return nil, status.Error(codes.InvalidArgument, "A valid SRP parameter is required") + } + serviceLevel := c.s.resolveParameter(params, c.symmetrixID, ServiceLevelParam, "Optimized") + if !isValidSLO(serviceLevel) { + log.Error("An invalid Service Level parameter was specified") + return nil, status.Errorf(codes.InvalidArgument, "An invalid Service Level parameter was specified") + } + storageGroupName := c.s.resolveParameter(params, c.symmetrixID, StorageGroupParam, "") + applicationPrefix := c.s.resolveParameter(params, c.symmetrixID, ApplicationPrefixParam, "") + hostLimitName := c.s.resolveParameter(params, c.symmetrixID, HostLimitNameParam, "") + hostIOLimitMBSec := c.s.resolveParameter(params, c.symmetrixID, HostIOLimitMBSecParam, "") + hostIOLimitIOSec := c.s.resolveParameter(params, c.symmetrixID, HostIOLimitIOSecParam, "") + dynamicDistribution := c.s.resolveParameter(params, c.symmetrixID, DynamicDistributionParam, "") + namespace := c.s.resolveParameter(params, "", CSIPVCNamespace, "") + + hostIOLimitInfo, err := buildHostIOLimitInfo(hostIOLimitMBSec, hostIOLimitIOSec, dynamicDistribution) + if err != nil { + return nil, err + } + + requiredCylinders, err := computeRequiredCylinders(req.GetCapacityRange()) + if err != nil { + return nil, err + } + + clusterPrefix := c.s.getClusterPrefix() + volumeIdentifier := buildVolumeIdentifier(clusterPrefix, volName, namespace) + storageGroupName = resolveStorageGroupName(storageGroupName, clusterPrefix, applicationPrefix, serviceLevel, storagePoolID, hostLimitName) + + var ( + snapshotID string + srcDevID string + contentSourceValue string + ) + contentSource := req.GetVolumeContentSource() + if contentSource != nil { + switch src := contentSource.GetType().(type) { + case *csi.VolumeContentSource_Snapshot: + snapshotID, srcDevID, contentSourceValue, err = c.handleSnapshotSource(ctx, contentSource) + if err != nil { + return nil, err + } + case *csi.VolumeContentSource_Volume: + srcDevID, contentSourceValue, err = c.handleCloneSource(ctx, src.Volume.GetVolumeId()) + if err != nil { + return nil, err + } + default: + return nil, status.Error(codes.InvalidArgument, "VolumeContentSource is missing volume and snapshot source") + } + // check snapshot is licensed + if licnErr := c.s.isSnapshotLicensed(ctx, c.symmetrixID, c.pmaxClient); licnErr != nil { + log.WithContext(ctx).Errorf("Snapshot license check failed on array %s: %v", c.symmetrixID, licnErr) + return nil, status.Error(codes.Internal, licnErr.Error()) + } + } + + // Dynamic SG: select the best existing SG or propose a new one based on volume counts. + // In 10.4 API, we don't need to explicitly create the SG - the API will create it + // automatically if it doesn't exist using the provided attributes. + if c.s.isDynamicSGEnabled() { + dynamicSGName, _, err := c.s.getDynamicSG(ctx, c.symmetrixID, storageGroupName) + if err != nil { + log.Errorf("Failed to get dynamic SG for array %s: %v", c.symmetrixID, err) + return nil, status.Errorf(codes.Internal, "failed to get dynamic storage group: %s", err.Error()) + } + log.Infof("Dynamic SG enabled: selected storage group %s (base: %s) for array %s", dynamicSGName, storageGroupName, c.symmetrixID) + storageGroupName = dynamicSGName + } + + log.Infof("Attempting 10.4 CreateVolume for volume %s (identifier: %s) on array %s, %d cylinders", + volName, volumeIdentifier, c.symmetrixID, requiredCylinders) + + // Single 10.4 API call: creates the volume, adds it to the SG (creating the SG if needed), + // sets the identifier, and validates SRP capacity — all atomically. + // Native idempotency: supplying volume.identifier alongside create_new causes the array to + // return 200 and either the existing volume object (if re-added to SG) or just the SG object + // (if fully idempotent: volume already exists and is already in the SG). + // This replaces the legacy pre-flight sequence of: + // GetStoragePoolList + GetStoragePool + GetStorageGroup/CreateStorageGroup + + // GetVolumeIDList + GetVolumeByID + CreateVolumeInStorageGroupS (5-7 calls → 1 call) + var gpmaxReq *types.CreateVolumesRequest + if snapshotID != "" { + gpmaxReq = buildCreateVolumeFromSnapshotRequest(requiredCylinders, snapshotID, storagePoolID, serviceLevel, storageGroupName, volumeIdentifier, hostIOLimitInfo, c.reqID) + } else if srcDevID != "" { + gpmaxReq = buildCloneVolumesRequest(requiredCylinders, srcDevID, storagePoolID, serviceLevel, storageGroupName, volumeIdentifier, hostIOLimitInfo, c.reqID) + } else { + gpmaxReq = buildCreateVolumesRequest(requiredCylinders, storagePoolID, serviceLevel, storageGroupName, volumeIdentifier, hostIOLimitInfo, c.reqID) + } + + // CSI specific metada for authorization metadata headers (PV name, PVC name, PVC namespace) + headerMetadata := addMetaData(params) + + gpmaxResp, err := c.pmaxClient104.CreateVolume(ctx, c.symmetrixID, *gpmaxReq, headerMetadata) + if err != nil { + log.Errorf("10.4 CreateVolume failed for volume %s on array %s: %v", volName, c.symmetrixID, err) + return nil, classifyCreateVolumeError(err) + } + + if gpmaxResp == nil || len(gpmaxResp.Results.Result) == 0 { + log.Errorf("10.4 CreateVolume response is empty for volume %s on array %s", volName, c.symmetrixID) + return nil, status.Error(codes.Internal, "create volume response is empty") + } + + result := gpmaxResp.Results.Result[0] + if result.Volume == nil { + // Fully idempotent: volume already exists and is already a member of the SG. + // The array returns only the storage_group object, not the volume object. + + if result.StorageGroup != nil && result.StorageGroup.ID != "" { + storageGroupName = result.StorageGroup.ID + } + // Fall back to GetVolumesByIdentifier to obtain the device ID and size. + log.Infof("10.4 CreateVolume: fully idempotent for volume %s on array %s, performing lookup", volumeIdentifier, c.symmetrixID) + return c.handleIdempotentVolume(ctx, volumeIdentifier, storageGroupName, serviceLevel, storagePoolID, requiredCylinders, contentSourceValue, contentSource, accessibility) + } + + devID := result.Volume.ID + if devID == "" { + log.Errorf("10.4 CreateVolume returned volume object with empty device ID for volume %s on array %s", volName, c.symmetrixID) + return nil, status.Error(codes.Internal, "create volume response contains empty device ID") + } + log.Infof("10.4 CreateVolume succeeded for volume %s on array %s with device ID %s (status: %s)", + volName, c.symmetrixID, devID, result.Status) + + // Use the array-returned identifier for VolumeId (may include a tenant prefix). + if result.Volume.Identifier != "" { + volumeIdentifier = result.Volume.Identifier + } + + // The top-level storage_group object in the response always identifies the target SG, + // regardless of how many SGs the volume belongs to. + if result.StorageGroup != nil && result.StorageGroup.ID != "" { + storageGroupName = result.StorageGroup.ID + } + + // Build VolumeId in the same format as legacy: volumeIdentifier-symID-devID + csiVolumeID := fmt.Sprintf("%s-%s-%s", volumeIdentifier, c.symmetrixID, devID) + + volumeContext := buildVolumeContext(c.s.getReplicationContextPrefix(), c.symmetrixID, serviceLevel, storagePoolID, storageGroupName, result.Volume.CapCyl, contentSourceValue) + + // Add zone labels to volume context when array was found via Availability Zone + if c.symmIDFoundInAZ { + for k, v := range c.s.getStorageArrayLabels(c.symmetrixID) { + volumeContext[k] = v + } + } + + resp := &csi.CreateVolumeResponse{ + Volume: &csi.Volume{ + VolumeId: csiVolumeID, + CapacityBytes: int64(requiredCylinders) * cylinderSizeInBytes, + VolumeContext: volumeContext, + ContentSource: contentSource, + }, + } + if accessibility != nil { + resp.Volume.AccessibleTopology = accessibility.Preferred + } + return resp, nil +} + +// classifyCreateVolumeError inspects the error returned by the 10.4 CreateVolume +// API and maps well-known error codes/messages to appropriate CSI gRPC status errors. +// Error codes from the PowerMax REST API: +// +// 0x020e0105 – size mismatch (idempotent volume or target < source) +// 0x020e0114 – source volume does not exist (clone) +// 0x020e0117 – snapshot not found (snapshot restore) +func classifyCreateVolumeError(err error) error { + if err == nil { + return nil + } + msg := err.Error() + + // Idempotent volume creation with size mismatch + if strings.Contains(msg, "does not match existing volume size") { + return status.Errorf(codes.AlreadyExists, + "A volume with the same name exists but has a different size: %s", msg) + } + + // Clone / snapshot restore: requested target smaller than source + if strings.Contains(msg, "cannot be smaller than source volume size") { + return status.Errorf(codes.InvalidArgument, + "Requested capacity is smaller than the source") + } + + // Clone source volume does not exist + if strings.Contains(msg, "Source Volume") && strings.Contains(msg, "does not exist") { + return status.Errorf(codes.InvalidArgument, + "Volume content source couldn't be found in the array: %s", msg) + } + + // Snapshot not found + if strings.Contains(msg, "No Snapshot found") { + return status.Errorf(codes.InvalidArgument, + "Snapshot not found on the array: %s", msg) + } + + // Default: wrap as Internal + return status.Errorf(codes.Internal, "Failed to create volume: %s", msg) +} + +// buildVolumeIdentifier builds the volume identifier in the same format as legacy: +// csi--[-] +func buildVolumeIdentifier(clusterPrefix, volumeName, namespace string) string { + maxLength := MaxVolIdentifierLength - len(clusterPrefix) - len(clusterPrefix) - len(CsiVolumePrefix) - 1 + shortVolumeName := truncateString(volumeName, maxLength) + if namespace == "" { + return CsiVolumePrefix + clusterPrefix + "-" + shortVolumeName + } + return CsiVolumePrefix + clusterPrefix + "-" + shortVolumeName + "-" + namespace +} + +func resolveStorageGroupName(currentSGName, clusterPrefix, applicationPrefix, serviceLevel, storagePoolID, hostLimitName string) string { + if currentSGName != "" { + return currentSGName + } + + var storageGroupName string + if applicationPrefix == "" { + storageGroupName = fmt.Sprintf("%s-%s-%s-%s-SG", CSIPrefix, clusterPrefix, serviceLevel, storagePoolID) + } else { + storageGroupName = fmt.Sprintf("%s-%s-%s-%s-%s-SG", CSIPrefix, clusterPrefix, applicationPrefix, serviceLevel, storagePoolID) + } + if hostLimitName != "" { + storageGroupName = fmt.Sprintf("%s-%s", storageGroupName, hostLimitName) + } + return storageGroupName +} + +func buildManageVolumeStorageGroupAction(storageGroupName, storagePoolID, serviceLevel string, hostIOLimitInfo *types.HostIOLimitInfo) *types.ManageVolumeStorageGroupAction { + return &types.ManageVolumeStorageGroupAction{ + Action: volumeStorageGroupActionAdd, + StorageGroup: types.VolumeStorageGroupParam{ + ID: storageGroupName, + SRP: &types.VolumeSrpParam{ + ID: storagePoolID, + }, + ServiceLevel: &types.VolumeServiceLevelParam{ + ID: serviceLevel, + }, + HostIOLimitInfo: hostIOLimitInfo, + }, + } +} + +func buildCreateVolumesRequest(requiredCylinders int, storagePoolID, serviceLevel, storageGroupName, volumeIdentifier string, hostIOLimitInfo *types.HostIOLimitInfo, reqID string) *types.CreateVolumesRequest { + return &types.CreateVolumesRequest{ + Volumes: []types.VolumeRequestParam{ + { + // volume.identifier alongside create_new enables native 10.4 idempotency: + // the array sets the identifier on creation, and uses it to recognise duplicate + // requests instead of failing. ManageIdentifier action is NOT used — combining + // volume.identifier with manage_identifier causes a 500 error. + Volume: &types.ExistingVolumeRequestParam{Identifier: volumeIdentifier}, + CreateNew: &types.CreateVolumeParam{ + CreateNewFromAttributes: &types.CreateNewFromAttributes{ + CapacityUnit: "CYL", + VolumeSize: float64(requiredCylinders), + }, + PrecheckSrpCapacity: &types.ValidationSrpAction{ + SRP: types.VolumeSrpParam{ID: storagePoolID}, + }, + }, + Actions: &types.VolumeRequestParamActions{ + ManageVolumeStorageGroup: buildManageVolumeStorageGroupAction(storageGroupName, storagePoolID, serviceLevel, hostIOLimitInfo), + }, + RequestID: reqID, + ResponseSelect: createVolumesResponseSelect, + }, + }, + ExecutionOption: types.ExecutionOptionSynchronous, + } +} + +// buildCreateVolumeFromSnapshotRequest builds the 10.4 CreateVolumes request using +// create_new_from_snapshot with new_volume_attributes for explicit size control. +// It uses the same SG management and identifier-based idempotency as the attributes path. +func buildCreateVolumeFromSnapshotRequest(requiredCylinders int, snapshotID, storagePoolID, serviceLevel, storageGroupName, volumeIdentifier string, hostIOLimitInfo *types.HostIOLimitInfo, reqID string) *types.CreateVolumesRequest { + return &types.CreateVolumesRequest{ + Volumes: []types.VolumeRequestParam{ + { + Volume: &types.ExistingVolumeRequestParam{Identifier: volumeIdentifier}, + CreateNew: &types.CreateVolumeParam{ + CreateNewFromSnapshot: &types.CreateNewFromSnapshot{ + Snapshot: types.SnapshotRequestParam{ID: snapshotID}, + NewVolumeAttributes: &types.CreateNewFromAttributes{ + CapacityUnit: "CYL", + VolumeSize: float64(requiredCylinders), + }, + }, + PrecheckSrpCapacity: &types.ValidationSrpAction{ + SRP: types.VolumeSrpParam{ID: storagePoolID}, + }, + }, + Actions: &types.VolumeRequestParamActions{ + ManageVolumeStorageGroup: buildManageVolumeStorageGroupAction(storageGroupName, storagePoolID, serviceLevel, hostIOLimitInfo), + }, + RequestID: reqID, + ResponseSelect: createVolumesResponseSelect, + }, + }, + ExecutionOption: types.ExecutionOptionSynchronous, + } +} + +// buildCloneVolumesRequest builds a 10.4 CreateVolumes request that creates a new +// volume and copies data from srcDevID. The request uses create_new_from_attributes +// with an explicit size so the target volume can be equal to or larger than the +// source. Data is copied via the manage_replication / local / CopyFrom action +// with establish_terminate. +func buildCloneVolumesRequest(requiredCylinders int, srcDevID, storagePoolID, serviceLevel, storageGroupName, volumeIdentifier string, hostIOLimitInfo *types.HostIOLimitInfo, reqID string) *types.CreateVolumesRequest { + estTerminate := true + return &types.CreateVolumesRequest{ + Volumes: []types.VolumeRequestParam{ + { + Volume: &types.ExistingVolumeRequestParam{Identifier: volumeIdentifier}, + CreateNew: &types.CreateVolumeParam{ + CreateNewFromAttributes: &types.CreateNewFromAttributes{ + CapacityUnit: "CYL", + VolumeSize: float64(requiredCylinders), + }, + PrecheckSrpCapacity: &types.ValidationSrpAction{ + SRP: types.VolumeSrpParam{ID: storagePoolID}, + }, + }, + Actions: &types.VolumeRequestParamActions{ + ManageVolumeStorageGroup: buildManageVolumeStorageGroupAction(storageGroupName, storagePoolID, serviceLevel, hostIOLimitInfo), + ManageReplication: &types.ManageReplicationAction{ + Local: &types.LocalReplicationAction{ + Action: "CopyFrom", + Volume: types.ExistingVolumeRequestParam{ + ID: srcDevID, + }, + EstablishTerminate: &estTerminate, + }, + }, + }, + RequestID: reqID, + ResponseSelect: createVolumesResponseSelect, + }, + }, + ExecutionOption: types.ExecutionOptionSynchronous, + } +} + +func buildHostIOLimitInfo(hostIOLimitMBSec, hostIOLimitIOSec, dynamicDistribution string) (*types.HostIOLimitInfo, error) { + if hostIOLimitMBSec == "" && hostIOLimitIOSec == "" && dynamicDistribution == "" { + return nil, nil + } + + hostIOLimitInfo := &types.HostIOLimitInfo{ + DynamicDistribution: dynamicDistribution, + } + + if hostIOLimitMBSec != "" { + mbSec, err := strconv.Atoi(hostIOLimitMBSec) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "invalid %s value: %s", HostIOLimitMBSecParam, hostIOLimitMBSec) + } + if mbSec < 0 { + return nil, status.Errorf(codes.InvalidArgument, "%s must be non-negative", HostIOLimitMBSecParam) + } + hostIOLimitInfo.HostIOLimitMBSec = mbSec + } + + if hostIOLimitIOSec != "" { + ioSec, err := strconv.Atoi(hostIOLimitIOSec) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "invalid %s value: %s", HostIOLimitIOSecParam, hostIOLimitIOSec) + } + if ioSec < 0 { + return nil, status.Errorf(codes.InvalidArgument, "%s must be non-negative", HostIOLimitIOSecParam) + } + hostIOLimitInfo.HostIOLimitIOSec = ioSec + } + + return hostIOLimitInfo, nil +} + +func buildVolumeContext(replicationContextPrefix, symmetrixID, serviceLevel, storagePoolID, storageGroupName string, capCyl float64, contentSource string) map[string]string { + return map[string]string{ + ServiceLevelParam: serviceLevel, + StoragePoolParam: storagePoolID, + path.Join(replicationContextPrefix, SymmetrixIDParam): symmetrixID, + CapacityGB: fmt.Sprintf("%.2f", capCyl), + ContentSource: contentSource, + StorageGroup: storageGroupName, + "CreationTime": time.Now().Format("20060102150405"), + } +} + +// handleSnapshotSource parses, validates, and performs size-check for snapshot restore. +// Returns: +// - snapshotID: the snap_id from VolumeSnapshotSource (for create_new_from_snapshot.snapshot.id) +// - srcDevID: the source volume device ID (for size validation) +// - contentSource: the parsed snapshot name (stored in VolumeContext[ContentSource]) +func (c *u4p104VolumeCreator) handleSnapshotSource(ctx context.Context, cs *csi.VolumeContentSource) (snapshotID, srcDevID, contentSource string, err error) { + if ctx.Err() != nil { + return "", "", "", status.FromContextError(ctx.Err()).Err() + } + + if cs == nil || cs.GetSnapshot() == nil { + return "", "", "", status.Error(codes.InvalidArgument, "snapshot source is required in VolumeContentSource") + } + + srcSnapID := cs.GetSnapshot().GetSnapshotId() + if srcSnapID == "" { + return "", "", "", status.Error(codes.InvalidArgument, "snapshot ID is required in VolumeContentSource") + } + + // Parse the CSI snapshot ID: format is -- + parsedSnapID, symID, devID, _, _, parseErr := c.s.parseCsiID(srcSnapID) + if parseErr != nil { + log.WithContext(ctx).Errorf("Snapshot identifier not in supported format: %s", srcSnapID) + return "", "", "", status.Error(codes.InvalidArgument, "Snapshot identifier not in supported format") + } + + // Validate the snapshot belongs to the target array + if symID != c.symmetrixID { + log.WithContext(ctx).WithFields(map[string]interface{}{ + "snapshot_array": symID, + "target_array": c.symmetrixID, + }).Error("Snapshot is on different PowerMax array") + return "", "", "", status.Error(codes.InvalidArgument, "The volume content source is in different PowerMax array") + } + + // Fetch snapshot info to obtain snap_id for 10.4 API + snapInfo, snapErr := c.pmaxClient.GetSnapshotInfo(ctx, c.symmetrixID, devID, parsedSnapID) + if snapErr != nil { + log.WithContext(ctx).WithFields(map[string]interface{}{ + "snapshot_name": parsedSnapID, + "source_device": devID, + "array": c.symmetrixID, + }).Errorf("Failed to get snapshot info: %v", snapErr) + return "", "", "", status.Errorf(codes.InvalidArgument, "Snapshot %s not found on source volume %s: %s", parsedSnapID, devID, snapErr.Error()) + } + + if snapInfo == nil || len(snapInfo.VolumeSnapshotSource) == 0 { + log.WithContext(ctx).Errorf("Snapshot %s has no source generations on volume %s", parsedSnapID, devID) + return "", "", "", status.Errorf(codes.InvalidArgument, "Snapshot %s has no source generations on volume %s", parsedSnapID, devID) + } + + // Extract snap_id from the first generation for the 10.4 API's create_new_from_snapshot.snapshot.id field + snapID := snapInfo.VolumeSnapshotSource[0].SnapID + if snapID == 0 { + log.WithContext(ctx).Errorf("Snapshot %s has invalid snap_id on volume %s", parsedSnapID, devID) + return "", "", "", status.Errorf(codes.Internal, "Snapshot %s has invalid snap_id on volume %s", parsedSnapID, devID) + } + + // Return snap_id as string for the 10.4 API, devID for tracking, and parsedSnapID for VolumeContext + return fmt.Sprintf("%d", snapID), devID, parsedSnapID, nil +} + +// handleCloneSource parses, validates, and performs size-check for volume clone. +// Returns: +// - srcDevID: the source volume device ID +// - contentSource: the full CSI volume ID (stored in VolumeContext[ContentSource]) +func (c *u4p104VolumeCreator) handleCloneSource(ctx context.Context, srcVolID string) (srcDevID string, contentSource string, err error) { + if ctx.Err() != nil { + return "", "", status.FromContextError(ctx.Err()).Err() + } + + if srcVolID == "" { + return "", "", status.Error(codes.InvalidArgument, "Source volume ID is required for cloning") + } + + _, srcSymID, parsedDevID, _, _, parseErr := c.s.parseCsiID(srcVolID) + if parseErr != nil { + log.WithContext(ctx).Errorf("Could not parse source CSI VolumeId: %s", srcVolID) + return "", "", status.Error(codes.InvalidArgument, "Source volume identifier not in supported format") + } + + if srcSymID != c.symmetrixID { + log.WithContext(ctx).WithFields(map[string]interface{}{ + "source_array": srcSymID, + "target_array": c.symmetrixID, + }).Error("Source volume is on different PowerMax array") + return "", "", status.Error(codes.InvalidArgument, "Source volume must be on the same PowerMax array for cloning") + } + + return parsedDevID, srcVolID, nil +} + +// computeRequiredCylinders converts a CSI CapacityRange into a cylinder count. +// This is pure local math — no API calls. SRP capacity is validated server-side +// by precheck_srp_capacity in the 10.4 CreateVolume request. +func computeRequiredCylinders(cr *csi.CapacityRange) (int, error) { + var minSizeBytes, maxSizeBytes int64 + if cr != nil { + minSizeBytes = cr.GetRequiredBytes() + maxSizeBytes = cr.GetLimitBytes() + } + if minSizeBytes < 0 || maxSizeBytes < 0 { + return 0, status.Errorf(codes.OutOfRange, + "bad capacity: requested volume size bytes %d and limit size bytes %d must not be negative", + minSizeBytes, maxSizeBytes) + } + if minSizeBytes == 0 { + minSizeBytes = DefaultVolumeSizeBytes + } + if minSizeBytes < MinVolumeSizeBytes { + minSizeBytes = MinVolumeSizeBytes + } + numOfCylinders := int(minSizeBytes / cylinderSizeInBytes) + if minSizeBytes%cylinderSizeInBytes > 0 { + numOfCylinders++ + } + sizeInBytes := int64(numOfCylinders) * cylinderSizeInBytes + if maxSizeBytes > 0 && sizeInBytes > maxSizeBytes { + return 0, status.Errorf(codes.OutOfRange, + "bad capacity: size in bytes %d exceeds limit size bytes %d", sizeInBytes, maxSizeBytes) + } + return numOfCylinders, nil +} + +// handleIdempotentVolume is called when the 10.4 CreateVolume API returns 200 but no volume +// object (fully idempotent: volume already exists and is already in the SG). It looks up the +// volume by identifier to obtain the device ID needed to build the CSI VolumeId. +// SG membership and size validation are omitted: the array already enforces both (a size +// mismatch returns 500 before reaching this path, and the SG was confirmed by the 200 response). +func (c *u4p104VolumeCreator) handleIdempotentVolume(ctx context.Context, volumeIdentifier, storageGroupName, serviceLevel, storagePoolID string, requiredCylinders int, volContent string, contentSource *csi.VolumeContentSource, accessibility *csi.TopologyRequirement) (*csi.CreateVolumeResponse, error) { + log := log.WithContext(ctx) + volumeList, err := c.pmaxClient.GetVolumesByIdentifier(ctx, c.symmetrixID, volumeIdentifier) + if err != nil { + log.Errorf("10.4 idempotency lookup failed for volume %s on array %s: %v", volumeIdentifier, c.symmetrixID, err) + return nil, status.Errorf(codes.Internal, "idempotency check failed: %s", err.Error()) + } + + for _, eachVol := range volumeList.Volumes { + log.Infof("10.4 idempotent volume detected %s on array %s, returning success", eachVol.ID, c.symmetrixID) + + csiVolumeID := fmt.Sprintf("%s-%s-%s", eachVol.Identifier, c.symmetrixID, eachVol.ID) + volumeContext := buildVolumeContext(c.s.getReplicationContextPrefix(), c.symmetrixID, serviceLevel, storagePoolID, storageGroupName, eachVol.CapCyl, volContent) + + // Add zone labels to volume context when array was found via Availability Zone + if c.symmIDFoundInAZ { + for k, v := range c.s.getStorageArrayLabels(c.symmetrixID) { + volumeContext[k] = v + } + } + + resp := &csi.CreateVolumeResponse{ + Volume: &csi.Volume{ + VolumeId: csiVolumeID, + CapacityBytes: int64(requiredCylinders) * cylinderSizeInBytes, + VolumeContext: volumeContext, + ContentSource: contentSource, + }, + } + if accessibility != nil { + resp.Volume.AccessibleTopology = accessibility.Preferred + } + return resp, nil + } + + return nil, status.Errorf(codes.Internal, + "volume with identifier %s reported as existing but not found on array %s", volumeIdentifier, c.symmetrixID) +} + +// --------------------------------------------------------------------------- +// Legacy volume creation (pre-10.4) +// --------------------------------------------------------------------------- + +func (c *legacyVolumeCreator) Create(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { + log := log.WithContext(ctx) + + reqID := c.reqID + params := c.params + symmetrixID := c.symmetrixID + pmaxClient := c.pmaxClient + symmIDFoundInAZ := c.symmIDFoundInAZ + version := c.apiVersion + + accessibility := req.GetAccessibilityRequirements() + + thick := params[ThickVolumesParam] + applicationPrefix := c.s.resolveParameter(params, symmetrixID, ApplicationPrefixParam, "") + + // Storage (resource) Pool. Validate it against exist Pools + storagePoolID := c.s.resolveParameter(params, symmetrixID, StoragePoolParam, "") + err := c.s.validateStoragePoolID(ctx, symmetrixID, storagePoolID, pmaxClient) + if err != nil { + log.Error(err.Error()) + return nil, status.Errorf(codes.InvalidArgument, "%s", err.Error()) + } + + // SLO is optional + serviceLevel := c.s.resolveParameter(params, symmetrixID, ServiceLevelParam, "Optimized") + found := false + for _, val := range validSLO { + if serviceLevel == val { + found = true + break + } + } + if !found { + return nil, status.Errorf(codes.InvalidArgument, "An invalid Service Level parameter was specified") + } + + storageGroupName := c.s.resolveParameter(params, symmetrixID, StorageGroupParam, "") + hostLimitName := c.s.resolveParameter(params, symmetrixID, HostLimitNameParam, "") + hostMBsec := c.s.resolveParameter(params, symmetrixID, HostIOLimitMBSecParam, "") + hostIOsec := c.s.resolveParameter(params, symmetrixID, HostIOLimitIOSecParam, "") + hostDynDistribution := c.s.resolveParameter(params, symmetrixID, DynamicDistributionParam, "") + namespace := c.s.resolveParameter(params, "", CSIPVCNamespace, "") + + // Dynamic SG check is available only from 10.1 + if c.s.opts.dynamicSGEnabled && version < 101 { + log.Errorf("Dynamic SG is enabled, but not supported for array %s with version %d. Minimum expected array version is 10.1", symmetrixID, version) + return nil, status.Errorf(codes.Internal, "Dynamic SG is enabled, but not supported for array %s with version %d. Minimum expected array version is 10.1", symmetrixID, version) + } + + // File related params + useNFS := false + nasServer := "" + allowRoot := "" + if params[NASServerName] != "" { + nasServer = params[NASServerName] + } + if params[AllowRootParam] != "" { + allowRoot = params[AllowRootParam] + } + + // Validate volume capabilities + vcs := req.GetVolumeCapabilities() + if vcs != nil { + isBlock := accTypeIsBlock(vcs) + if isBlock && !c.s.opts.EnableBlock { + return nil, status.Error(codes.InvalidArgument, "Block Volume Capability is not supported") + } + useNFS = accTypeIsNFS(vcs) + if isBlock && useNFS { + return nil, status.Errorf(codes.InvalidArgument, "NFS with Block is not supported") + } + } + + // Remote Replication based paramsMes + var replicationEnabled string + var remoteSymID string + var localRDFGrpNo string + var remoteRDFGrpNo string + var remoteServiceLevel string + var remoteSRPID string + var repMode string + var bias string + + if params[path.Join(c.s.opts.ReplicationPrefix, RepEnabledParam)] == "true" { + if c.s.opts.IsVsphereEnabled { + return nil, status.Errorf(codes.Unavailable, "Replication on a vSphere volume is not supported") + } + if useNFS { + return nil, status.Errorf(codes.Unavailable, "Replication on a NFS volume is not supported") + } + replicationEnabled = params[path.Join(c.s.opts.ReplicationPrefix, RepEnabledParam)] + // remote symmetrix ID and rdf group name are mandatory params when replication is enabled + remoteSymID = params[path.Join(c.s.opts.ReplicationPrefix, RemoteSymIDParam)] + // check if storage class contains SRDG details + if params[path.Join(c.s.opts.ReplicationPrefix, LocalRDFGroupParam)] != "" { + localRDFGrpNo = params[path.Join(c.s.opts.ReplicationPrefix, LocalRDFGroupParam)] + } + if params[path.Join(c.s.opts.ReplicationPrefix, RemoteRDFGroupParam)] != "" { + remoteRDFGrpNo = params[path.Join(c.s.opts.ReplicationPrefix, RemoteRDFGroupParam)] + } + repMode = params[path.Join(c.s.opts.ReplicationPrefix, ReplicationModeParam)] + + if symmIDFoundInAZ && repMode == Metro { + return nil, status.Errorf(codes.InvalidArgument, "The use of Availability Zones with Metro volumes is not supported") + } + + remoteServiceLevel = params[path.Join(c.s.opts.ReplicationPrefix, RemoteServiceLevelParam)] + remoteSRPID = params[path.Join(c.s.opts.ReplicationPrefix, RemoteSRPParam)] + bias = params[path.Join(c.s.opts.ReplicationPrefix, BiasParam)] + + // Get Local and remote RDFg Numbers from a rest call + // Create RDFg for a namespace if it doens't exist? + // Create RDFg when the volume gets added first time for a replication sssn + if localRDFGrpNo == "" && remoteRDFGrpNo == "" { + localRDFGrpNo, remoteRDFGrpNo, err = c.s.GetOrCreateRDFGroup(ctx, symmetrixID, remoteSymID, repMode, namespace, pmaxClient) + if err != nil { + return nil, status.Errorf(codes.NotFound, "Received error get/create RDFG, err: %s", err.Error()) + } + if localRDFGrpNo == "" || remoteRDFGrpNo == "" { + return nil, status.Errorf(codes.Unavailable, "Can not fetch RDF Group for volume creation get/create RDFG") + } + log.Debugf("RDF group for given array pair and RDF mode: local(%s), remote(%s)", localRDFGrpNo, remoteRDFGrpNo) + } + if repMode == Metro { + return c.s.createMetroVolume(ctx, req, reqID, storagePoolID, symmetrixID, storageGroupName, serviceLevel, thick, remoteSymID, localRDFGrpNo, remoteRDFGrpNo, remoteServiceLevel, remoteSRPID, namespace, applicationPrefix, bias, hostLimitName, hostMBsec, hostIOsec, hostDynDistribution) + } + if repMode != Async && repMode != Sync { + log.Errorf("Unsupported Replication Mode: (%s)", repMode) + return nil, status.Errorf(codes.InvalidArgument, "Unsupported Replication Mode: (%s)", repMode) + } + } + + // Get the required capacity + cr := req.GetCapacityRange() + requiredCylinders, err := c.s.validateVolSize(ctx, cr, symmetrixID, storagePoolID, pmaxClient) + if err != nil { + return nil, err + } + + var srcVolID, srcSnapID string + var symID, SrcDevID, snapID string + var srcVol *types.Volume + var volContent string + // When content source is specified, the size of the new volume + // is determined based on the size of the source volume in the + // snapshot. The size of the new volume to be created should be + // greater than or equal to the size of snapshot source + contentSource := req.GetVolumeContentSource() + if contentSource != nil { + if useNFS { + return nil, status.Errorf(codes.Unavailable, "Cloning on a NFS volume is not supported") + } + switch req.GetVolumeContentSource().GetType().(type) { + case *csi.VolumeContentSource_Volume: + srcVolID = req.GetVolumeContentSource().GetVolume().GetVolumeId() + if srcVolID != "" { + _, symID, SrcDevID, _, _, err = c.s.parseCsiID(srcVolID) + if err != nil { + // We couldn't comprehend the identifier. + log.Error("Could not parse CSI VolumeId: " + srcVolID) + return nil, status.Error(codes.InvalidArgument, "Source volume identifier not in supported format") + } + volContent = srcVolID + } + break + case *csi.VolumeContentSource_Snapshot: + srcSnapID = req.GetVolumeContentSource().GetSnapshot().GetSnapshotId() + if srcSnapID != "" { + snapID, symID, SrcDevID, _, _, err = c.s.parseCsiID(srcSnapID) + if err != nil { + // We couldn't comprehend the identifier. + log.Error("Snapshot identifier not in supported format: " + srcSnapID) + return nil, status.Error(codes.InvalidArgument, "Snapshot identifier not in supported format") + } + volContent = snapID + } + break + default: + return nil, status.Error(codes.InvalidArgument, "VolumeContentSource is missing volume and snapshot source") + } + // check snapshot is licensed + if err := c.s.IsSnapshotLicensed(ctx, symID, pmaxClient); err != nil { + log.Error("Error - " + err.Error()) + return nil, status.Error(codes.Internal, err.Error()) + } + } + + if SrcDevID != "" && symID != "" { + if symID != symmetrixID { + log.Error("The volume content source is in different PowerMax array") + return nil, status.Errorf(codes.InvalidArgument, "The volume content source is in different PowerMax array") + } + srcVol, err = pmaxClient.GetVolumeByID(ctx, symmetrixID, SrcDevID) + if err != nil { + log.Error("Volume content source volume couldn't be found in the array: " + err.Error()) + return nil, status.Errorf(codes.InvalidArgument, "Volume content source volume couldn't be found in the array: %s", err.Error()) + } + // reset the volume size to match with source + if requiredCylinders < srcVol.CapacityCYL { + log.Error("Capacity specified is smaller than the source") + return nil, status.Error(codes.InvalidArgument, "Requested capacity is smaller than the source") + } + } + + // Get the volume name + volumeName := req.GetName() + if volumeName == "" { + log.Error("Name cannot be empty") + return nil, status.Error(codes.InvalidArgument, + "Name cannot be empty") + } + + // Get the Volume prefix from environment + volumePrefix := c.s.getClusterPrefix() + maxLength := MaxVolIdentifierLength - len(volumePrefix) - len(c.s.getClusterPrefix()) - len(CsiVolumePrefix) - 1 + // First get the short volume name + shortVolumeName := truncateString(volumeName, maxLength) + // Form the volume identifier using short volume name and namespace + var namespaceSuffix string + if namespace != "" { + namespaceSuffix = "-" + namespace + } + volumeIdentifier := fmt.Sprintf("%s%s-%s%s", CsiVolumePrefix, c.s.getClusterPrefix(), shortVolumeName, namespaceSuffix) + + if useNFS { + // calculate size in MiB + reqSizeInMiB := (cr.GetRequiredBytes() + MiBSizeInBytes - 1) / MiBSizeInBytes + return file.CreateFileSystem(ctx, reqID, accessibility, params, symmetrixID, storagePoolID, serviceLevel, nasServer, volumeIdentifier, allowRoot, reqSizeInMiB, pmaxClient) + } + // Storage Group is required to be derived from the parameters (such as service level and storage resource pool which are supplied in parameters) + // Storage Group Name can optionally be supplied in the parameters (for testing) to over-ride the default. + if storageGroupName == "" { + if applicationPrefix == "" { + storageGroupName = fmt.Sprintf("%s-%s-%s-%s-SG", CSIPrefix, c.s.getClusterPrefix(), + serviceLevel, storagePoolID) + } else { + storageGroupName = fmt.Sprintf("%s-%s-%s-%s-%s-SG", CSIPrefix, c.s.getClusterPrefix(), + applicationPrefix, serviceLevel, storagePoolID) + } + if hostLimitName != "" { + storageGroupName = fmt.Sprintf("%s-%s", storageGroupName, hostLimitName) + } + } + + var dynamicSGName string + var needCreation bool + if c.s.opts.dynamicSGEnabled { + if dynamicSGName, needCreation, err = getDynamicSG(ctx, symmetrixID, storageGroupName, c.s); err != nil { + log.Error("failed to get dynamic SG: " + err.Error()) + return nil, status.Error(codes.Internal, err.Error()) + } + log.Infof("####### dynamic storage group name: %s, base SG name: %s, needCreation: %v", dynamicSGName, storageGroupName, needCreation) + storageGroupName = dynamicSGName + } + + // localProtectionGroupID refers to name of Storage Group which has protected local volumes + // remoteProtectionGroupID refers to name of Storage Group which has protected remote volumes + var localProtectionGroupID string + var remoteProtectionGroupID string + if replicationEnabled == "true" { + localProtectionGroupID = buildProtectionGroupID(namespace, localRDFGrpNo, repMode) + remoteProtectionGroupID = buildProtectionGroupID(namespace, remoteRDFGrpNo, repMode) + } + + // log all parameters used in CreateVolume call + fields := map[string]interface{}{ + "SymmetrixID": symmetrixID, + "SRP": storagePoolID, + "Accessibility": accessibility, + "ApplicationPrefix": applicationPrefix, + "volumeIdentifier": volumeIdentifier, + "requiredCylinders": requiredCylinders, + "storageGroupName": storageGroupName, + "CSIRequestID": reqID, + "SourceVolume": srcVolID, + "SourceSnapshot": srcSnapID, + "ReplicationEnabled": replicationEnabled, + "RemoteSymID": remoteSymID, + "LocalRDFGroup": localRDFGrpNo, + "RemoteRDFGroup": remoteRDFGrpNo, + "SRDFMode": repMode, + "PVCNamespace": namespace, + "LocalProtectionGroupID": localProtectionGroupID, + "RemoteProtectionGroupID": remoteProtectionGroupID, + HeaderPersistentVolumeName: params[CSIPersistentVolumeName], + HeaderPersistentVolumeClaimName: params[CSIPersistentVolumeClaimName], + HeaderPersistentVolumeClaimNamespace: params[CSIPVCNamespace], + HostIOLimitMBSec: hostMBsec, + HostIOLimitIOSec: hostIOsec, + DynamicDistribution: hostDynDistribution, + } + log.WithFields(fields).Info("Executing CreateVolume with following fields") + + // isSGUnprotected is set to true only if SG has a replica, eg if the SG is new + isSGUnprotected := false + if replicationEnabled == "true" { + sg, err := c.s.getOrCreateProtectedStorageGroup(ctx, symmetrixID, localProtectionGroupID, namespace, localRDFGrpNo, repMode, reqID, pmaxClient) + if err != nil { + return nil, status.Errorf(codes.Internal, "Error in getOrCreateProtectedStorageGroup: (%s)", err.Error()) + } + if sg != nil && sg.Rdf == true { + // Check the direction of SG + // Creation of replicated volume is allowed in an SG of type R1 + err := c.s.VerifyProtectedGroupDirection(ctx, symmetrixID, localProtectionGroupID, localRDFGrpNo, pmaxClient) + if err != nil { + return nil, err + } + } else { + isSGUnprotected = true + } + } + + // Check existence of the Storage Group and create if necessary. + if !c.s.opts.dynamicSGEnabled { + sg, err := pmaxClient.GetStorageGroup(ctx, symmetrixID, storageGroupName) + if err != nil || sg == nil { + log.Debug(fmt.Sprintf("Unable to find storage group: %s", storageGroupName)) + needCreation = true + } + } + + if needCreation { + hostLimitsParam := &types.SetHostIOLimitsParam{ + HostIOLimitMBSec: hostMBsec, + HostIOLimitIOSec: hostIOsec, + DynamicDistribution: hostDynDistribution, + } + optionalPayload := make(map[string]interface{}) + optionalPayload[HostLimits] = hostLimitsParam + if *hostLimitsParam == (types.SetHostIOLimitsParam{}) { + optionalPayload = nil + } + _, err := pmaxClient.CreateStorageGroup(ctx, symmetrixID, storageGroupName, storagePoolID, + serviceLevel, thick == "true", optionalPayload) + if err != nil { + log.Error("Error creating storage group: " + err.Error()) + return nil, status.Errorf(codes.Internal, "Error creating storage group: %s", err.Error()) + } + } + alreadyExists := false + isLocalVolumePresent := false + + var vol *types.Volume + var volumeList *types.Volumev1 + isV4 := c.s.isV4OrAbove(ctx, symmetrixID, pmaxClient) + + if version >= APIVersion103 && isV4 { + log.Debug("API version is greater than or equal to 103. Using enhanced API") + // Idempotency test. We will read the volume and check for: + // 1. Existence of a volume with matching volume name + // 2. Matching cylinderSize + // 3. Is a member of the storage group + // 4. Check if snapshot/volume target + log.Debug("Calling GetVolumeIDList for idempotency test") + volumeList, err = pmaxClient.GetVolumesByIdentifier(ctx, symmetrixID, volumeIdentifier) + if err != nil { + log.Error("Error getting the volumes for idempotence check: " + err.Error()) + return nil, status.Errorf(codes.Internal, "Error getting the volumes for idempotence check: %s", err.Error()) + } + + // isLocalVolumePresent restrict CreateVolumeInProtectedSG call if the volume is present in local SG but not in remote SG + // isLocalVolumePresent := false + // Look up the volume(s), if any, returned for the idempotency check to see if there are any matches + // We ignore any volume not in the desired storage group (even though they have the same name). + for _, eachVol := range volumeList.Volumes { + if len(eachVol.StorageGroups) < 1 { + log.Error("Idempotence check: StorageGroupIDList is empty for (%s): " + eachVol.ID) + return nil, status.Errorf(codes.Internal, "Idempotence check: StorageGroupIDList is empty for (%s)", eachVol.ID) + } + matchesStorageGroup := false + for _, sgid := range eachVol.StorageGroups { + if strings.Contains(sgid.StorageGroupID, storageGroupName) { + matchesStorageGroup = true + storageGroupName = sgid.StorageGroupID + } + } + + // with Authorization, a tenant prefix is applied to the volume identifier on the array + // csi-CSM-pmax-69298b3d3d-namespace -> tn1-csi-CSM-pmax-69298b3d3d-namespace + // since we don't know the tenant prefix, the volume identifier on the array is checked to contain the standard volume identifier + if matchesStorageGroup && (eachVol.Identifier == volumeIdentifier || strings.Contains(eachVol.Identifier, volumeIdentifier)) { + // A volume with the same name exists and has the same size + if eachVol.CapCyl != float64(requiredCylinders) { + log.Error("A volume with the same name exists but has a different size than required.") + alreadyExists = true + continue + } + var remoteVolumeID string + if replicationEnabled == "true" { + remoteVolumeID, _, err = c.s.GetRemoteVolumeID(ctx, symmetrixID, localRDFGrpNo, eachVol.ID, pmaxClient) + if err != nil && !strings.Contains(err.Error(), "The device must be an RDF device") { + return nil, status.Errorf(codes.Internal, "Failed to fetch rdf pair information for (%s) - Error (%s)", eachVol.ID, err.Error()) + } + if remoteVolumeID == "" { + // Missing corresponding Remote Volume Name for existing local volume + // The SG is unprotected as Local volume and Local SG exists but missing corresponding SRDF info + // If the SG was protected, there must exist a corresponding remote replica volume + log.Debugf("Local Volume already exist, skipping creation (%s)", eachVol.ID) + isLocalVolumePresent = true + vol = &types.Volume{} + vol.VolumeID = eachVol.ID + vol.CapacityGB = eachVol.CapCyl + vol.VolumeIdentifier = eachVol.Identifier + var sgIDs []string + for _, sg := range eachVol.StorageGroups { + if sg.StorageGroupID != "" { + sgIDs = append(sgIDs, sg.StorageGroupID) + } + } + vol.StorageGroupIDList = sgIDs + continue + } + } + if volContent != "" { + if replicationEnabled == "true" { + if srcSnapID != "" { + err = c.s.LinkSRDFVolToSnapshot(ctx, reqID, symID, srcVol.VolumeID, snapID, localProtectionGroupID, localRDFGrpNo, vol, bias, false, pmaxClient) + if err != nil { + return nil, err + } + } else if srcVolID != "" { + isV4 := c.s.isV4OrAbove(ctx, symID, pmaxClient) + if isV4 { + err = c.s.LinkSRDFCloneVolume(ctx, reqID, symID, srcVol, vol, localProtectionGroupID, localRDFGrpNo, "false", pmaxClient) + if err != nil { + return nil, status.Errorf(codes.Internal, "Failed to create SRDF volume from volume (%s)", err.Error()) + } + } else { + tmpSnapID := fmt.Sprintf("%s%s-%d", TempSnap, c.s.getClusterPrefix(), time.Now().Nanosecond()) + err = c.s.LinkSRDFVolToVolume(ctx, reqID, symID, srcVol, vol, tmpSnapID, localProtectionGroupID, localRDFGrpNo, "false", false, pmaxClient) + if err != nil { + return nil, status.Errorf(codes.Internal, "Failed to create SRDF volume from volume (%s)", err.Error()) + } + } + } + } else { // replication is not enabled + if srcSnapID != "" { + err = c.s.UnlinkTargets(ctx, symID, SrcDevID, pmaxClient) + if err != nil { + return nil, status.Errorf(codes.Internal, "Failed unlink existing target from snapshot (%s)", err.Error()) + } + err = c.s.LinkVolumeToSnapshot(ctx, symID, srcVol.VolumeID, eachVol.ID, snapID, reqID, false, pmaxClient) + if err != nil { + return nil, status.Errorf(codes.Internal, "Failed to create volume from snapshot (%s)", err.Error()) + } + } else if srcVolID != "" && eachVol.ID != "" { + isV4 := c.s.isV4OrAbove(ctx, symID, pmaxClient) + if isV4 { + replicaRequest := types.ReplicationRequest{ + ReplicationPair: []types.ReplicationPair{ + { + SourceVolumeName: srcVol.VolumeID, + TargetVolumeName: eachVol.ID, + }, + }, + Establish: true, + EstablishTerminate: true, + } + err = pmaxClient.CloneVolumeFromVolume(ctx, symID, replicaRequest) + if err != nil { + return nil, status.Errorf(codes.Internal, "Failed to create volume from volume (%s)", err.Error()) + } + } else { + tmpSnapID := fmt.Sprintf("%s%s-%d", TempSnap, c.s.getClusterPrefix(), time.Now().Nanosecond()) + err = c.s.LinkVolumeToVolume(ctx, symID, srcVol, eachVol.ID, tmpSnapID, reqID, false, pmaxClient) + if err != nil { + return nil, status.Errorf(codes.Internal, "Failed to create volume from volume (%s)", err.Error()) + } + } + } + } + } + + log.WithFields(fields).Info("Idempotent volume detected, returning success") + eachVol.ID = fmt.Sprintf("%s-%s-%s", eachVol.Identifier, symmetrixID, eachVol.ID) + volResp := c.s.buildCSIVolume(&eachVol) + // Set the volume context + attributes := map[string]string{ + ServiceLevelParam: serviceLevel, + StoragePoolParam: storagePoolID, + path.Join(c.s.opts.ReplicationContextPrefix, SymmetrixIDParam): symmetrixID, + CapacityGB: fmt.Sprintf("%.2f", eachVol.CapCyl), + ContentSource: volContent, + StorageGroup: storageGroupName, + // Format the time output + "CreationTime": time.Now().Format("20060102150405"), + } + if replicationEnabled == "true" { + addReplicationParamsToVolumeAttributes(attributes, c.s.opts.ReplicationContextPrefix, remoteSymID, repMode, remoteVolumeID, localRDFGrpNo, remoteRDFGrpNo) + } + volResp.VolumeContext = attributes + csiResp := &csi.CreateVolumeResponse{ + Volume: volResp, + } + volResp.ContentSource = contentSource + if accessibility != nil { + volResp.AccessibleTopology = accessibility.Preferred + } + return csiResp, nil + } + } + } else { + // Idempotency test. We will read the volume and check for: + // 1. Existence of a volume with matching volume name + // 2. Matching cylinderSize + // 3. Is a member of the storage group + // 4. Check if snapshot/volume target + log.Debug("Calling GetVolumeIDList for idempotency test") + // For now an exact match + volumeIDList, err := pmaxClient.GetVolumeIDList(ctx, symmetrixID, volumeIdentifier, false) + if err != nil { + log.Error("Error looking up volume for idempotence check: " + err.Error()) + return nil, status.Errorf(codes.Internal, "Error looking up volume for idempotence check: %s", err.Error()) + } + // isLocalVolumePresent restrict CreateVolumeInProtectedSG call if the volume is present in local SG but not in remote SG + // isLocalVolumePresent := false + // Look up the volume(s), if any, returned for the idempotency check to see if there are any matches + // We ignore any volume not in the desired storage group (even though they have the same name). + for _, volumeID := range volumeIDList { + // Fetch the volume + log.WithFields(fields).Info("Calling GetVolumeByID for idempotence check") + vol, err = pmaxClient.GetVolumeByID(ctx, symmetrixID, volumeID) + if err != nil { + log.Error("Error fetching volume for idempotence check: " + err.Error()) + return nil, status.Errorf(codes.Internal, "Error fetching volume for idempotence check: %s", err.Error()) + } + if len(vol.StorageGroupIDList) < 1 { + log.Error("Idempotence check: StorageGroupIDList is empty for (%s): " + volumeID) + return nil, status.Errorf(codes.Internal, "Idempotence check: StorageGroupIDList is empty for (%s)", volumeID) + } + matchesStorageGroup := false + for _, sgid := range vol.StorageGroupIDList { + if strings.Contains(sgid, storageGroupName) { + matchesStorageGroup = true + storageGroupName = sgid + } + } + + // with Authorization, a tenant prefix is applied to the volume identifier on the array + // csi-CSM-pmax-69298b3d3d-namespace -> tn1-csi-CSM-pmax-69298b3d3d-namespace + // since we don't know the tenant prefix, the volume identifier on the array is checked to contain the standard volume identifier + if matchesStorageGroup && (vol.VolumeIdentifier == volumeIdentifier || strings.Contains(vol.VolumeIdentifier, volumeIdentifier)) { + // A volume with the same name exists and has the same size + if vol.CapacityCYL != requiredCylinders { + log.Error("A volume with the same name exists but has a different size than required.") + alreadyExists = true + continue + } + var remoteVolumeID string + if replicationEnabled == "true" { + remoteVolumeID, _, err = c.s.GetRemoteVolumeID(ctx, symmetrixID, localRDFGrpNo, vol.VolumeID, pmaxClient) + if err != nil && !strings.Contains(err.Error(), "The device must be an RDF device") { + return nil, status.Errorf(codes.Internal, "Failed to fetch rdf pair information for (%s) - Error (%s)", vol.VolumeID, err.Error()) + } + if remoteVolumeID == "" { + // Missing corresponding Remote Volume Name for existing local volume + // The SG is unprotected as Local volume and Local SG exists but missing corresponding SRDF info + // If the SG was protected, there must exist a corresponding remote replica volume + log.Debugf("Local Volume already exist, skipping creation (%s)", vol.VolumeID) + isLocalVolumePresent = true + continue + } + } + if volContent != "" { + if replicationEnabled == "true" { + if srcSnapID != "" { + err = c.s.LinkSRDFVolToSnapshot(ctx, reqID, symID, srcVol.VolumeID, snapID, localProtectionGroupID, localRDFGrpNo, vol, bias, false, pmaxClient) + if err != nil { + return nil, err + } + } else if srcVolID != "" { + if isV4 { + err = c.s.LinkSRDFCloneVolume(ctx, reqID, symID, srcVol, vol, localProtectionGroupID, localRDFGrpNo, "false", pmaxClient) + if err != nil { + return nil, status.Errorf(codes.Internal, "Failed to create SRDF volume from volume (%s)", err.Error()) + } + } else { + tmpSnapID := fmt.Sprintf("%s%s-%d", TempSnap, c.s.getClusterPrefix(), time.Now().Nanosecond()) + err = c.s.LinkSRDFVolToVolume(ctx, reqID, symID, srcVol, vol, tmpSnapID, localProtectionGroupID, localRDFGrpNo, "false", false, pmaxClient) + if err != nil { + return nil, status.Errorf(codes.Internal, "Failed to create SRDF volume from volume (%s)", err.Error()) + } + } + } + } else { // replication is not enabled + if srcSnapID != "" { + err = c.s.UnlinkTargets(ctx, symID, SrcDevID, pmaxClient) + if err != nil { + return nil, status.Errorf(codes.Internal, "Failed unlink existing target from snapshot (%s)", err.Error()) + } + err = c.s.LinkVolumeToSnapshot(ctx, symID, srcVol.VolumeID, vol.VolumeID, snapID, reqID, false, pmaxClient) + if err != nil { + return nil, status.Errorf(codes.Internal, "Failed to create volume from snapshot (%s)", err.Error()) + } + } else if srcVolID != "" { + isV4 := c.s.isV4OrAbove(ctx, symID, pmaxClient) + if isV4 { + replicaRequest := types.ReplicationRequest{ + ReplicationPair: []types.ReplicationPair{ + { + SourceVolumeName: srcVol.VolumeID, + TargetVolumeName: vol.VolumeID, + }, + }, + Establish: true, + EstablishTerminate: true, + } + err = pmaxClient.CloneVolumeFromVolume(ctx, symID, replicaRequest) + if err != nil { + return nil, status.Errorf(codes.Internal, "Failed to create volume from volume (%s)", err.Error()) + } + } else { + tmpSnapID := fmt.Sprintf("%s%s-%d", TempSnap, c.s.getClusterPrefix(), time.Now().Nanosecond()) + err = c.s.LinkVolumeToVolume(ctx, symID, srcVol, vol.VolumeID, tmpSnapID, reqID, false, pmaxClient) + if err != nil { + return nil, status.Errorf(codes.Internal, "Failed to create volume from volume (%s)", err.Error()) + } + } + } + } + } + + log.WithFields(fields).Info("Idempotent volume detected, returning success") + vol.VolumeID = fmt.Sprintf("%s-%s-%s", vol.VolumeIdentifier, symmetrixID, vol.VolumeID) + volResp := c.s.getCSIVolume(vol) + // Set the volume context + attributes := map[string]string{ + ServiceLevelParam: serviceLevel, + StoragePoolParam: storagePoolID, + path.Join(c.s.opts.ReplicationContextPrefix, SymmetrixIDParam): symmetrixID, + CapacityGB: fmt.Sprintf("%.2f", vol.CapacityGB), + ContentSource: volContent, + StorageGroup: storageGroupName, + // Format the time output + "CreationTime": time.Now().Format("20060102150405"), + } + + if symmIDFoundInAZ { + c.s.addZoneLabelsToVolumeAttributes(attributes, symmetrixID) + } + + if replicationEnabled == "true" { + addReplicationParamsToVolumeAttributes(attributes, c.s.opts.ReplicationContextPrefix, remoteSymID, repMode, remoteVolumeID, localRDFGrpNo, remoteRDFGrpNo) + } + volResp.VolumeContext = attributes + csiResp := &csi.CreateVolumeResponse{ + Volume: volResp, + } + volResp.ContentSource = contentSource + if accessibility != nil { + volResp.AccessibleTopology = accessibility.Preferred + } + return csiResp, nil + } + } + } + if alreadyExists { + log.Error("A volume with the same name " + volumeName + "exists but has a different size than requested. Use a different name.") + return nil, status.Errorf(codes.AlreadyExists, "A volume with the same name %s exists but has a different size than requested. Use a different name.", volumeName) + } + + // CSI specific metada for authorization + headerMetadata := addMetaData(params) + + // Let's create the volume + if !isLocalVolumePresent { + vol, err = pmaxClient.CreateVolumeInStorageGroupS(ctx, symmetrixID, storageGroupName, volumeIdentifier, requiredCylinders, nil, headerMetadata) + if err != nil { + log.Error(fmt.Sprintf("Could not create volume: %s: %s", volumeName, err.Error())) + return nil, status.Errorf(codes.Internal, "Could not create volume: %s: %s", volumeName, err.Error()) + } + } + + if replicationEnabled == "true" { + log.Debugf("RDF: Found Rdf enabled") + // remote storage group name is kept same as local storage group name + // Check if volume is already added in SG, else add it + protectedSGID := c.s.GetProtectedStorageGroupID(vol.StorageGroupIDList, localRDFGrpNo+"-"+repMode) + if protectedSGID == "" { + // Volume is not present in Protected Storage Group, Add + err = c.s.addVolumesToProtectedStorageGroup(ctx, reqID, symmetrixID, localProtectionGroupID, remoteSymID, remoteProtectionGroupID, false, vol.VolumeID, pmaxClient) + if err != nil { + return nil, err + } + } + if isSGUnprotected { + // If the required SG is still unprotected, protect the local SG with RDF info + // If valid RDF group is supplied this will create a remote SG, a RDF pair and add the vol in respective SG created + // Remote storage group name is kept same as local storage group name + err := c.s.ProtectStorageGroup(ctx, symmetrixID, remoteSymID, localProtectionGroupID, remoteProtectionGroupID, "", localRDFGrpNo, repMode, vol.VolumeID, reqID, false, pmaxClient) + if err != nil { + log.Errorf("Proceeding to remove volume from protected storage group as rollback") + // Remove volume from protected storage group as a rollback + // The device could be just a TDEV and can make RDF unmanageable due to slow u4p response + _, er := pmaxClient.RemoveVolumesFromStorageGroup(ctx, symmetrixID, localProtectionGroupID, true, vol.VolumeID) + if er != nil { + log.Errorf("Error removing volume %s from protected SG %s with error: %s", vol.VolumeID, localProtectionGroupID, er.Error()) + } + return nil, err + } + } + } + + // If volume content source is specified, initiate no_copy to newly created volume + if contentSource != nil { + if srcVolID != "" { + if replicationEnabled == "true" { + isV4 := c.s.isV4OrAbove(ctx, symID, pmaxClient) + if isV4 { + err = c.s.LinkSRDFCloneVolume(ctx, reqID, symID, srcVol, vol, localProtectionGroupID, localRDFGrpNo, "false", pmaxClient) + if err != nil { + return nil, status.Errorf(codes.Internal, "Failed to create SRDF volume from volume (%s)", err.Error()) + } + } else { + tmpSnapID := fmt.Sprintf("%s%s-%d", TempSnap, c.s.getClusterPrefix(), time.Now().Nanosecond()) + err = c.s.LinkSRDFVolToVolume(ctx, reqID, symID, srcVol, vol, tmpSnapID, localProtectionGroupID, localRDFGrpNo, "false", false, pmaxClient) + if err != nil { + return nil, status.Errorf(codes.Internal, "Failed to create SRDF volume from volume (%s)", err.Error()) + } + } + } else { + isV4 := c.s.isV4OrAbove(ctx, symID, pmaxClient) + if isV4 { + replicaRequest := types.ReplicationRequest{ + ReplicationPair: []types.ReplicationPair{ + { + SourceVolumeName: srcVol.VolumeID, + TargetVolumeName: vol.VolumeID, + }, + }, + Establish: true, + EstablishTerminate: true, + } + err = pmaxClient.CloneVolumeFromVolume(ctx, symID, replicaRequest) + if err != nil { + return nil, status.Errorf(codes.Internal, "Failed to create volume from volume (%s)", err.Error()) + } + } else { + tmpSnapID := fmt.Sprintf("%s%s-%d", TempSnap, c.s.getClusterPrefix(), time.Now().Nanosecond()) + err = c.s.LinkVolumeToVolume(ctx, symID, srcVol, vol.VolumeID, tmpSnapID, reqID, false, pmaxClient) + if err != nil { + return nil, status.Errorf(codes.Internal, "Failed to create volume from volume (%s)", err.Error()) + } + } + } + } else if srcSnapID != "" { + if replicationEnabled == "true" { + err = c.s.LinkSRDFVolToSnapshot(ctx, reqID, symID, srcVol.VolumeID, snapID, localProtectionGroupID, localRDFGrpNo, vol, bias, false, pmaxClient) + if err != nil { + return nil, err + } + } else { + // Unlink all previous targets from this snapshot if the link is in defined state + err = c.s.UnlinkTargets(ctx, symID, SrcDevID, pmaxClient) + if err != nil { + return nil, status.Errorf(codes.Internal, "Failed unlink existing target from snapshot (%s)", err.Error()) + } + err = c.s.LinkVolumeToSnapshot(ctx, symID, srcVol.VolumeID, vol.VolumeID, snapID, reqID, false, pmaxClient) + if err != nil { + return nil, status.Errorf(codes.Internal, "Failed to create volume from snapshot (%s)", err.Error()) + } + } + } + } + + // Formulate the return response + volID := vol.VolumeID + vol.VolumeID = fmt.Sprintf("%s-%s-%s", vol.VolumeIdentifier, symmetrixID, vol.VolumeID) + volResp := c.s.getCSIVolume(vol) + volResp.ContentSource = contentSource + // Set the volume context + attributes := map[string]string{ + ServiceLevelParam: serviceLevel, + StoragePoolParam: storagePoolID, + path.Join(c.s.opts.ReplicationContextPrefix, SymmetrixIDParam): symmetrixID, + CapacityGB: fmt.Sprintf("%.2f", vol.CapacityGB), + ContentSource: volContent, + StorageGroup: storageGroupName, + // Format the time output + "CreationTime": time.Now().Format("20060102150405"), + } + if replicationEnabled == "true" { + remoteVolumeID, _, err := c.s.GetRemoteVolumeID(ctx, symmetrixID, localRDFGrpNo, volID, pmaxClient) + if err != nil { + return nil, status.Errorf(codes.Internal, "Failed to fetch rdf pair information for (%s) - Error (%s)", vol.VolumeID, err.Error()) + } + addReplicationParamsToVolumeAttributes(attributes, c.s.opts.ReplicationContextPrefix, remoteSymID, repMode, remoteVolumeID, localRDFGrpNo, remoteRDFGrpNo) + } + + volResp.VolumeContext = attributes + if accessibility != nil { + volResp.AccessibleTopology = accessibility.Preferred + } + csiResp := &csi.CreateVolumeResponse{ + Volume: volResp, + } + fields[storageGroupName] = storageGroupName + log.WithFields(fields).Infof("Created volume with ID: %s", volResp.VolumeId) + return csiResp, nil +} diff --git a/service/volume_creator_test.go b/service/volume_creator_test.go new file mode 100644 index 00000000..29f024a4 --- /dev/null +++ b/service/volume_creator_test.go @@ -0,0 +1,2971 @@ +package service + +import ( + "context" + "errors" + "fmt" + "net/http" + "testing" + + "github.com/dell/csi-powermax/v2/pkg/symmetrix/mocks" + pmax "github.com/dell/gopowermax/v2" + types "github.com/dell/gopowermax/v2/types/v100" + "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" +) + +func TestU4P104VolumeCreator_DynamicSGEnabled_SelectsExistingSG(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := mocks.NewMockPmaxClient(ctrl) + symmetrixID := "000197900049" + baseSGName := "csi--Optimized-SRP_1-SG" + deviceID := "00DYN" + volumeIdentifier := "csi--dyn-vol" + + // Mock CreateVolume + mockClient.EXPECT().CreateVolume(gomock.Any(), symmetrixID, gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, _ string, req types.CreateVolumesRequest, _ ...http.Header) (*types.CreateVolumesResponse, error) { + // Verify the SG name is the one with lowest volume count + assert.Equal(t, baseSGName, req.Volumes[0].Actions.ManageVolumeStorageGroup.StorageGroup.ID) + return &types.CreateVolumesResponse{ + Summary: types.ResponseSummary{Total: 1, Succeeded: 1}, + Results: types.CreateVolumesResults{ + Result: []types.CreateVolumeResponseItem{ + { + Status: "success", + Volume: &types.VolumeRefResponse{ + ID: deviceID, + Identifier: volumeIdentifier, + }, + StorageGroup: &types.StorageGroupRefResponse{ID: baseSGName}, + }, + }, + }, + }, nil + }).Times(1) + + // Use mock service deps with getDynamicSG returning the base SG name (existing SG) + mockSvc := &mockU4P104ServiceDeps{ + dynamicSGEnabled: true, + dynamicSGName: baseSGName, + dynamicSGCreated: false, + } + creator := &u4p104VolumeCreator{ + s: mockSvc, + pmaxClient: mockClient, + pmaxClient104: mockClient, + symmetrixID: symmetrixID, + } + + req := &csi.CreateVolumeRequest{ + Name: "dyn-vol", + CapacityRange: &csi.CapacityRange{ + RequiredBytes: 1073741824, + }, + Parameters: map[string]string{ + StoragePoolParam: "SRP_1", + ServiceLevelParam: "Optimized", + }, + } + + resp, err := creator.Create(context.Background(), req) + assert.NoError(t, err) + assert.NotNil(t, resp) + assert.Equal(t, baseSGName, resp.Volume.VolumeContext[StorageGroup]) +} + +func TestBuildHostIOLimitInfo_EmptyReturnsNil(t *testing.T) { + hostIOLimitInfo, err := buildHostIOLimitInfo("", "", "") + assert.NoError(t, err) + assert.Nil(t, hostIOLimitInfo) +} + +func TestBuildHostIOLimitInfo_ParsesValues(t *testing.T) { + hostIOLimitInfo, err := buildHostIOLimitInfo("1000", "5000", "Always") + assert.NoError(t, err) + assert.NotNil(t, hostIOLimitInfo) + assert.Equal(t, 1000, hostIOLimitInfo.HostIOLimitMBSec) + assert.Equal(t, 5000, hostIOLimitInfo.HostIOLimitIOSec) + assert.Equal(t, "Always", hostIOLimitInfo.DynamicDistribution) +} + +func TestBuildHostIOLimitInfo_InvalidValues(t *testing.T) { + _, err := buildHostIOLimitInfo("invalid", "", "") + assert.Error(t, err) + assert.Contains(t, err.Error(), HostIOLimitMBSecParam) + + _, err = buildHostIOLimitInfo("", "invalid", "") + assert.Error(t, err) + assert.Contains(t, err.Error(), HostIOLimitIOSecParam) + + _, err = buildHostIOLimitInfo("-1", "", "") + assert.Error(t, err) + assert.Contains(t, err.Error(), "non-negative") + + _, err = buildHostIOLimitInfo("", "-1", "") + assert.Error(t, err) + assert.Contains(t, err.Error(), "non-negative") +} + +func TestU4P104VolumeCreator_ApplicationPrefixInSGName(t *testing.T) { + // ApplicationPrefix is supported: it changes the SG name format to + // csi-----SG (matching legacy lines 254-256). + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := mocks.NewMockPmaxClient(ctrl) + symmetrixID := "000197900049" + deviceID := "00APP" + volumeIdentifier := "csi--app-vol" + expectedSGName := "csi--myapp-Optimized-SRP_1-SG" + + mockClient.EXPECT().CreateVolume(gomock.Any(), symmetrixID, gomock.Any(), gomock.Any()). + Return(&types.CreateVolumesResponse{ + Summary: types.ResponseSummary{Total: 1, Succeeded: 1}, + Results: types.CreateVolumesResults{ + Result: []types.CreateVolumeResponseItem{ + { + Status: "success", + Volume: &types.VolumeRefResponse{ + ID: deviceID, + Identifier: volumeIdentifier, + }, + StorageGroup: &types.StorageGroupRefResponse{ID: expectedSGName}, + }, + }, + }, + }, nil).Times(1) + + creator := &u4p104VolumeCreator{ + s: &service{}, + pmaxClient: mockClient, + pmaxClient104: mockClient, + symmetrixID: symmetrixID, + } + + req := &csi.CreateVolumeRequest{ + Name: "app-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 1073741824}, + Parameters: map[string]string{ + StoragePoolParam: "SRP_1", + ServiceLevelParam: "Optimized", + ApplicationPrefixParam: "myapp", + }, + } + + resp, err := creator.Create(context.Background(), req) + assert.NoError(t, err) + assert.NotNil(t, resp) + // SG name in context must include the application prefix + assert.Equal(t, expectedSGName, resp.Volume.VolumeContext[StorageGroup]) +} + +func TestU4P104VolumeCreator_HostLimitNameInSGName(t *testing.T) { + // HostLimitName is a naming-only param: it appends a suffix to the SG name + // (matching legacy controller.go lines 364-366). + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := mocks.NewMockPmaxClient(ctrl) + symmetrixID := "000197900049" + deviceID := "00HL1" + volumeIdentifier := "csi--hl-vol" + expectedSGName := "csi--Optimized-SRP_1-SG-gold" + + mockClient.EXPECT().CreateVolume(gomock.Any(), symmetrixID, gomock.Any(), gomock.Any()). + Return(&types.CreateVolumesResponse{ + Summary: types.ResponseSummary{Total: 1, Succeeded: 1}, + Results: types.CreateVolumesResults{ + Result: []types.CreateVolumeResponseItem{ + { + Status: "success", + Volume: &types.VolumeRefResponse{ + ID: deviceID, + Identifier: volumeIdentifier, + }, + StorageGroup: &types.StorageGroupRefResponse{ID: expectedSGName}, + }, + }, + }, + }, nil).Times(1) + + creator := &u4p104VolumeCreator{ + s: &service{}, + pmaxClient: mockClient, + pmaxClient104: mockClient, + symmetrixID: symmetrixID, + } + + req := &csi.CreateVolumeRequest{ + Name: "hl-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 1073741824}, + Parameters: map[string]string{ + StoragePoolParam: "SRP_1", + ServiceLevelParam: "Optimized", + HostLimitNameParam: "gold", + }, + } + + resp, err := creator.Create(context.Background(), req) + assert.NoError(t, err) + assert.NotNil(t, resp) + // SG name in context must include the host limit name suffix + assert.Equal(t, expectedSGName, resp.Volume.VolumeContext[StorageGroup]) +} + +func TestU4P104VolumeCreator_HostIOLimitInfoInCreateRequest(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := mocks.NewMockPmaxClient(ctrl) + symmetrixID := "000197900049" + deviceID := "00IOL" + volumeIdentifier := "csi--io-vol" + + mockClient.EXPECT().CreateVolume(gomock.Any(), symmetrixID, gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, _ string, req types.CreateVolumesRequest, _ ...http.Header) (*types.CreateVolumesResponse, error) { + if assert.Len(t, req.Volumes, 1) { + actions := req.Volumes[0].Actions + if assert.NotNil(t, actions) && assert.NotNil(t, actions.ManageVolumeStorageGroup) { + hostIOLimitInfo := actions.ManageVolumeStorageGroup.StorageGroup.HostIOLimitInfo + if assert.NotNil(t, hostIOLimitInfo) { + assert.Equal(t, 1000, hostIOLimitInfo.HostIOLimitMBSec) + assert.Equal(t, 5000, hostIOLimitInfo.HostIOLimitIOSec) + assert.Equal(t, "Always", hostIOLimitInfo.DynamicDistribution) + } + } + } + + return &types.CreateVolumesResponse{ + Summary: types.ResponseSummary{Total: 1, Succeeded: 1}, + Results: types.CreateVolumesResults{ + Result: []types.CreateVolumeResponseItem{ + { + Status: "success", + Volume: &types.VolumeRefResponse{ + ID: deviceID, + Identifier: volumeIdentifier, + }, + }, + }, + }, + }, nil + }).Times(1) + + creator := &u4p104VolumeCreator{ + s: &service{}, + pmaxClient: mockClient, + pmaxClient104: mockClient, + symmetrixID: symmetrixID, + } + + req := &csi.CreateVolumeRequest{ + Name: "io-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 1073741824}, + Parameters: map[string]string{ + StoragePoolParam: "SRP_1", + ServiceLevelParam: "Optimized", + HostIOLimitMBSecParam: "1000", + HostIOLimitIOSecParam: "5000", + DynamicDistributionParam: "Always", + }, + } + + resp, err := creator.Create(context.Background(), req) + assert.NoError(t, err) + assert.NotNil(t, resp) +} + +func TestU4P104VolumeCreator_StorageGroupParamOverride(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := mocks.NewMockPmaxClient(ctrl) + symmetrixID := "000197900049" + deviceID := "00SG1" + volumeIdentifier := "csi--sg-override-vol" + expectedSGName := "custom-test-sg" + + mockClient.EXPECT().CreateVolume(gomock.Any(), symmetrixID, gomock.Any(), gomock.Any()). + Return(&types.CreateVolumesResponse{ + Summary: types.ResponseSummary{Total: 1, Succeeded: 1}, + Results: types.CreateVolumesResults{ + Result: []types.CreateVolumeResponseItem{ + { + Status: "success", + Volume: &types.VolumeRefResponse{ + ID: deviceID, + Identifier: volumeIdentifier, + }, + }, + }, + }, + }, nil).Times(1) + + creator := &u4p104VolumeCreator{ + s: &service{}, + pmaxClient: mockClient, + pmaxClient104: mockClient, + symmetrixID: symmetrixID, + } + + req := &csi.CreateVolumeRequest{ + Name: "sg-override-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 1073741824}, + Parameters: map[string]string{ + StoragePoolParam: "SRP_1", + ServiceLevelParam: "Optimized", + StorageGroupParam: expectedSGName, + ApplicationPrefixParam: "myapp", + HostLimitNameParam: "gold", + }, + } + + resp, err := creator.Create(context.Background(), req) + assert.NoError(t, err) + assert.NotNil(t, resp) + assert.Equal(t, expectedSGName, resp.Volume.VolumeContext[StorageGroup]) +} + +func TestU4P104VolumeCreator_CreateSuccess(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := mocks.NewMockPmaxClient(ctrl) + mockService := &service{} + + symmetrixID := "000197900049" + volumeName := "test-volume" + storagePoolID := "SRP_1" + serviceLevel := "Optimized" + deviceID := "00ABC" + // With empty clusterPrefix: volumeIdentifier = "csi--test-volume" + volumeIdentifier := "csi--" + volumeName + + // Optimized path: single 10.4 CreateVolume call — no pre-flight GetStoragePoolList, + // GetStoragePool, GetVolumeIDList, or GetVolumeByID calls. + mockClient.EXPECT().CreateVolume(gomock.Any(), symmetrixID, gomock.Any(), gomock.Any()). + Return(&types.CreateVolumesResponse{ + Summary: types.ResponseSummary{Total: 1, Succeeded: 1}, + Results: types.CreateVolumesResults{ + Result: []types.CreateVolumeResponseItem{ + { + Status: "success", + Volume: &types.VolumeRefResponse{ + ID: deviceID, + Identifier: volumeIdentifier, + }, + }, + }, + }, + }, nil).Times(1) + + creator := &u4p104VolumeCreator{ + s: mockService, + pmaxClient: mockClient, + pmaxClient104: mockClient, + symmetrixID: symmetrixID, + params: map[string]string{StoragePoolParam: storagePoolID, ServiceLevelParam: serviceLevel}, + } + + req := &csi.CreateVolumeRequest{ + Name: volumeName, + CapacityRange: &csi.CapacityRange{ + RequiredBytes: 1073741824, + }, + Parameters: map[string]string{ + StoragePoolParam: storagePoolID, + ServiceLevelParam: serviceLevel, + }, + } + + resp, err := creator.Create(context.Background(), req) + + assert.NoError(t, err) + assert.NotNil(t, resp) + // VolumeId format: volumeIdentifier-symID-devID (matches legacy createCSIVolumeID format) + assert.Equal(t, volumeIdentifier+"-"+symmetrixID+"-"+deviceID, resp.Volume.VolumeId) + assert.Equal(t, serviceLevel, resp.Volume.VolumeContext[ServiceLevelParam]) + assert.Equal(t, storagePoolID, resp.Volume.VolumeContext[StoragePoolParam]) + // With empty ReplicationContextPrefix on &service{}, key is path.Join("", SymmetrixIDParam) = SymmetrixIDParam + assert.Equal(t, symmetrixID, resp.Volume.VolumeContext[SymmetrixIDParam]) +} + +func TestU4P104VolumeCreator_CreateIdempotent(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := mocks.NewMockPmaxClient(ctrl) + mockService := &service{} + + symmetrixID := "000197900049" + volumeName := "existing-volume" + storagePoolID := "SRP_1" + serviceLevel := "Optimized" + deviceID := "00DEF" + // ceil(1073741824 / 1966080) = 547 + capacityCYL := 547 + // With empty clusterPrefix: volumeIdentifier = "csi--existing-volume" + volumeIdentifier := "csi--" + volumeName + storageGroupName := "csi--" + serviceLevel + "-" + storagePoolID + "-SG" + + // Fully idempotent: volume already exists and is already in the SG. + // The array returns 200 with only the storage_group object (no volume object). + // The driver falls back to GetVolumesByIdentifier to obtain device ID and size. + mockClient.EXPECT().CreateVolume(gomock.Any(), symmetrixID, gomock.Any(), gomock.Any()). + Return(&types.CreateVolumesResponse{ + Summary: types.ResponseSummary{Total: 1, Succeeded: 1}, + Results: types.CreateVolumesResults{ + Result: []types.CreateVolumeResponseItem{ + {Status: "success", Volume: nil}, + }, + }, + }, nil).Times(1) + mockClient.EXPECT().GetVolumesByIdentifier(gomock.Any(), symmetrixID, volumeIdentifier). + Return(&types.Volumev1{ + Volumes: []types.VolumeEnhanced{ + { + ID: deviceID, + Identifier: volumeIdentifier, + CapCyl: float64(capacityCYL), + StorageGroups: []types.StorageGroupID{ + {StorageGroupID: storageGroupName}, + }, + }, + }, + }, nil).Times(1) + + creator := &u4p104VolumeCreator{ + s: mockService, + pmaxClient: mockClient, + pmaxClient104: mockClient, + symmetrixID: symmetrixID, + params: map[string]string{StoragePoolParam: storagePoolID, ServiceLevelParam: serviceLevel}, + } + + req := &csi.CreateVolumeRequest{ + Name: volumeName, + CapacityRange: &csi.CapacityRange{ + RequiredBytes: 1073741824, + }, + Parameters: map[string]string{ + StoragePoolParam: storagePoolID, + ServiceLevelParam: serviceLevel, + }, + } + + resp, err := creator.Create(context.Background(), req) + + assert.NoError(t, err) + assert.NotNil(t, resp) + // VolumeId format: volumeIdentifier-symID-devID + assert.Equal(t, volumeIdentifier+"-"+symmetrixID+"-"+deviceID, resp.Volume.VolumeId) + // SG name resolved from GetVolumesByIdentifier response + assert.Equal(t, storageGroupName, resp.Volume.VolumeContext[StorageGroup]) +} + +func TestU4P104VolumeCreator_IdempotentSizeMismatch(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := mocks.NewMockPmaxClient(ctrl) + mockService := &service{} + + symmetrixID := "000197900049" + volumeName := "existing-volume" + storagePoolID := "SRP_1" + serviceLevel := "Optimized" + + // The array enforces size consistency: when volume.identifier matches an existing volume + // but the requested size differs, the array returns a 500 error directly from CreateVolume. + // classifyCreateVolumeError wraps this as AlreadyExists. + mockClient.EXPECT().CreateVolume(gomock.Any(), symmetrixID, gomock.Any(), gomock.Any()). + Return(nil, errors.New("create volumes failed: 0x020e0105: defined volume size [1092]Cyls does not match existing volume size [546]Cyls")).Times(1) + + creator := &u4p104VolumeCreator{ + s: mockService, + pmaxClient: mockClient, + pmaxClient104: mockClient, + symmetrixID: symmetrixID, + params: map[string]string{StoragePoolParam: storagePoolID, ServiceLevelParam: serviceLevel}, + } + + req := &csi.CreateVolumeRequest{ + Name: volumeName, + CapacityRange: &csi.CapacityRange{ + RequiredBytes: 2147483648, // 2 GiB — mismatches existing 1 GiB volume + }, + Parameters: map[string]string{ + StoragePoolParam: storagePoolID, + ServiceLevelParam: serviceLevel, + }, + } + + resp, err := creator.Create(context.Background(), req) + + assert.Error(t, err) + assert.Nil(t, resp) + assert.Contains(t, err.Error(), "A volume with the same name exists but has a different size") + assert.Contains(t, err.Error(), "AlreadyExists") +} + +func TestU4P104VolumeCreator_DuplicateIdentifiers(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := mocks.NewMockPmaxClient(ctrl) + mockService := &service{} + + symmetrixID := "000197900049" + + // The array returns 500 when multiple volumes share the same identifier. + mockClient.EXPECT().CreateVolume(gomock.Any(), symmetrixID, gomock.Any(), gomock.Any()). + Return(nil, errors.New("Multiple volumes found with same identifier or id")).Times(1) + + creator := &u4p104VolumeCreator{ + s: mockService, + pmaxClient: mockClient, + pmaxClient104: mockClient, + symmetrixID: symmetrixID, + params: map[string]string{StoragePoolParam: "SRP_1", ServiceLevelParam: "Optimized"}, + } + + req := &csi.CreateVolumeRequest{ + Name: "dup-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 1073741824}, + Parameters: map[string]string{StoragePoolParam: "SRP_1", ServiceLevelParam: "Optimized"}, + } + + resp, err := creator.Create(context.Background(), req) + + assert.Error(t, err) + assert.Nil(t, resp) + assert.Contains(t, err.Error(), "Multiple volumes found") +} + +func TestU4P104VolumeCreator_PartiallyIdempotent(t *testing.T) { + // Volume existed but was removed from SG. Array re-adds it and returns the volume object. + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := mocks.NewMockPmaxClient(ctrl) + mockService := &service{} + + symmetrixID := "000197900049" + volumeName := "re-added-volume" + storagePoolID := "SRP_1" + serviceLevel := "Optimized" + deviceID := "001FD" + volumeIdentifier := "csi--" + volumeName + storageGroupName := "csi--" + serviceLevel + "-" + storagePoolID + "-SG" + + mockClient.EXPECT().CreateVolume(gomock.Any(), symmetrixID, gomock.Any(), gomock.Any()). + Return(&types.CreateVolumesResponse{ + Summary: types.ResponseSummary{Total: 1, Succeeded: 1}, + Results: types.CreateVolumesResults{ + Result: []types.CreateVolumeResponseItem{ + { + Status: "success", + Volume: &types.VolumeRefResponse{ + ID: deviceID, + Identifier: volumeIdentifier, + CapCyl: 546.0, + StorageGroups: []types.StorageGroupID{ + {StorageGroupID: storageGroupName}, + }, + }, + }, + }, + }, + }, nil).Times(1) + + creator := &u4p104VolumeCreator{ + s: mockService, + pmaxClient: mockClient, + pmaxClient104: mockClient, + symmetrixID: symmetrixID, + params: map[string]string{StoragePoolParam: storagePoolID, ServiceLevelParam: serviceLevel}, + } + + req := &csi.CreateVolumeRequest{ + Name: volumeName, + CapacityRange: &csi.CapacityRange{RequiredBytes: 1073741824}, + Parameters: map[string]string{StoragePoolParam: storagePoolID, ServiceLevelParam: serviceLevel}, + } + + resp, err := creator.Create(context.Background(), req) + + assert.NoError(t, err) + assert.NotNil(t, resp) + assert.Equal(t, volumeIdentifier+"-"+symmetrixID+"-"+deviceID, resp.Volume.VolumeId) + // SG resolved from response + assert.Equal(t, storageGroupName, resp.Volume.VolumeContext[StorageGroup]) +} + +func TestU4P104VolumeCreator_APIFailure(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := mocks.NewMockPmaxClient(ctrl) + mockService := &service{} + + symmetrixID := "000197900049" + volumeName := "test-volume" + storagePoolID := "SRP_1" + + // Optimized path: single 10.4 call, no pre-flight mocks needed. + mockClient.EXPECT().CreateVolume(gomock.Any(), symmetrixID, gomock.Any(), gomock.Any()). + Return(nil, errors.New("array connection failed")).Times(1) + + creator := &u4p104VolumeCreator{ + s: mockService, + pmaxClient: mockClient, + pmaxClient104: mockClient, + symmetrixID: symmetrixID, + params: map[string]string{StoragePoolParam: storagePoolID}, + } + + req := &csi.CreateVolumeRequest{ + Name: volumeName, + CapacityRange: &csi.CapacityRange{ + RequiredBytes: 1073741824, + }, + Parameters: map[string]string{StoragePoolParam: storagePoolID}, + } + + resp, err := creator.Create(context.Background(), req) + + assert.Error(t, err) + assert.Nil(t, resp) + assert.Contains(t, err.Error(), "array connection failed") +} + +func TestU4P104VolumeCreator_EmptyResponse(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := mocks.NewMockPmaxClient(ctrl) + mockService := &service{} + + symmetrixID := "000197900049" + volumeName := "test-volume" + storagePoolID := "SRP_1" + + // Optimized path: single 10.4 call, no pre-flight mocks needed. + mockClient.EXPECT().CreateVolume(gomock.Any(), symmetrixID, gomock.Any(), gomock.Any()). + Return(&types.CreateVolumesResponse{ + Results: types.CreateVolumesResults{ + Result: []types.CreateVolumeResponseItem{}, + }, + }, nil).Times(1) + + creator := &u4p104VolumeCreator{ + s: mockService, + pmaxClient: mockClient, + pmaxClient104: mockClient, + symmetrixID: symmetrixID, + params: map[string]string{StoragePoolParam: storagePoolID}, + } + + req := &csi.CreateVolumeRequest{ + Name: volumeName, + CapacityRange: &csi.CapacityRange{ + RequiredBytes: 1073741824, + }, + Parameters: map[string]string{StoragePoolParam: storagePoolID}, + } + + resp, err := creator.Create(context.Background(), req) + + assert.Error(t, err) + assert.Nil(t, resp) + assert.Contains(t, err.Error(), "create volume response is empty") +} + +func TestU4P104VolumeCreator_VolumeContextValidation(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := mocks.NewMockPmaxClient(ctrl) + mockService := &service{} + + symmetrixID := "000197900049" + volumeName := "test-volume" + storagePoolID := "SRP_1" + serviceLevel := "Optimized" + deviceID := "00GHI" + volumeIdentifier := "csi--" + volumeName + + // Optimized path: single 10.4 call, no pre-flight mocks needed. + mockClient.EXPECT().CreateVolume(gomock.Any(), symmetrixID, gomock.Any(), gomock.Any()). + Return(&types.CreateVolumesResponse{ + Summary: types.ResponseSummary{Total: 1, Succeeded: 1}, + Results: types.CreateVolumesResults{ + Result: []types.CreateVolumeResponseItem{ + { + Status: "success", + Volume: &types.VolumeRefResponse{ + ID: deviceID, + Identifier: volumeIdentifier, + }, + }, + }, + }, + }, nil).Times(1) + + creator := &u4p104VolumeCreator{ + s: mockService, + pmaxClient: mockClient, + pmaxClient104: mockClient, + symmetrixID: symmetrixID, + params: map[string]string{StoragePoolParam: storagePoolID, ServiceLevelParam: serviceLevel}, + } + + req := &csi.CreateVolumeRequest{ + Name: volumeName, + CapacityRange: &csi.CapacityRange{ + RequiredBytes: 1073741824, + }, + Parameters: map[string]string{ + StoragePoolParam: storagePoolID, + ServiceLevelParam: serviceLevel, + }, + } + + resp, err := creator.Create(context.Background(), req) + + assert.NoError(t, err) + assert.NotNil(t, resp) + assert.NotNil(t, resp.Volume.VolumeContext) + // Legacy context keys: ServiceLevelParam, StoragePoolParam, replication/symmetrixID, + // CapacityGB, ContentSource, StorageGroup, CreationTime + assert.Equal(t, serviceLevel, resp.Volume.VolumeContext[ServiceLevelParam]) + assert.Equal(t, storagePoolID, resp.Volume.VolumeContext[StoragePoolParam]) + // With empty ReplicationContextPrefix on &service{}, key is path.Join("", SymmetrixIDParam) = SymmetrixIDParam + assert.Equal(t, symmetrixID, resp.Volume.VolumeContext[SymmetrixIDParam]) +} + +func TestU4P104VolumeCreator_NilRequest(t *testing.T) { + creator := &u4p104VolumeCreator{s: &service{}} + resp, err := creator.Create(context.Background(), nil) + assert.Nil(t, resp) + assert.Error(t, err) + assert.Contains(t, err.Error(), "cannot be nil") +} + +func TestU4P104VolumeCreator_EmptyVolumeName(t *testing.T) { + creator := &u4p104VolumeCreator{s: &service{}} + req := &csi.CreateVolumeRequest{ + Name: "", + CapacityRange: &csi.CapacityRange{RequiredBytes: 1073741824}, + Parameters: map[string]string{StoragePoolParam: "SRP_1"}, + } + resp, err := creator.Create(context.Background(), req) + assert.Nil(t, resp) + assert.Error(t, err) + assert.Contains(t, err.Error(), "volume name is required") +} + +func TestU4P104VolumeCreator_EmptySRP(t *testing.T) { + creator := &u4p104VolumeCreator{s: &service{}} + req := &csi.CreateVolumeRequest{ + Name: "test-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 1073741824}, + Parameters: map[string]string{}, + } + resp, err := creator.Create(context.Background(), req) + assert.Nil(t, resp) + assert.Error(t, err) + assert.Contains(t, err.Error(), "SRP parameter is required") +} + +func TestBuildCloneVolumesRequest(t *testing.T) { + req := buildCloneVolumesRequest(547, "00SRC", "SRP_1", "Optimized", "csi-my-SG", "csi-my-clone", nil, "req-99") + + assert.Equal(t, types.ExecutionOptionSynchronous, req.ExecutionOption) + assert.Len(t, req.Volumes, 1) + + vol := req.Volumes[0] + assert.Equal(t, "req-99", vol.RequestID) + // volume.identifier set for native idempotency + assert.NotNil(t, vol.Volume) + assert.Equal(t, "csi-my-clone", vol.Volume.Identifier) + // create_new: must use create_new_from_attributes with explicit CYL size + assert.NotNil(t, vol.CreateNew) + assert.NotNil(t, vol.CreateNew.CreateNewFromAttributes) + assert.Equal(t, "CYL", vol.CreateNew.CreateNewFromAttributes.CapacityUnit) + assert.Equal(t, float64(547), vol.CreateNew.CreateNewFromAttributes.VolumeSize) + // precheck_srp_capacity set + assert.NotNil(t, vol.CreateNew.PrecheckSrpCapacity) + assert.Equal(t, "SRP_1", vol.CreateNew.PrecheckSrpCapacity.SRP.ID) + // actions: manage_volume_storage_group + manage_replication + assert.NotNil(t, vol.Actions) + assert.NotNil(t, vol.Actions.ManageVolumeStorageGroup) + assert.Equal(t, "Add", vol.Actions.ManageVolumeStorageGroup.Action) + assert.Equal(t, "csi-my-SG", vol.Actions.ManageVolumeStorageGroup.StorageGroup.ID) + // manage_replication: local CopyFrom with establish_terminate + assert.NotNil(t, vol.Actions.ManageReplication) + assert.NotNil(t, vol.Actions.ManageReplication.Local) + assert.Equal(t, "CopyFrom", vol.Actions.ManageReplication.Local.Action) + assert.Equal(t, "00SRC", vol.Actions.ManageReplication.Local.Volume.ID) + assert.NotNil(t, vol.Actions.ManageReplication.Local.EstablishTerminate) + assert.True(t, *vol.Actions.ManageReplication.Local.EstablishTerminate) + // manage_identifier must NOT be set + assert.Nil(t, vol.Actions.ManageIdentifier) + // response_select + assert.Equal(t, "id,identifier,cap_cyl,storage_groups", vol.ResponseSelect) +} + +func TestU4P104VolumeCreator_CloneSuccess(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := mocks.NewMockPmaxClient(ctrl) + + symmetrixID := "000197900049" + srcDevID := "00SRC" + srcCSIVolumeID := "csi--source-vol-" + symmetrixID + "-" + srcDevID + targetDevID := "00TGT" + volumeIdentifier := "csi--clone-vol" + storagePoolID := "SRP_1" + serviceLevel := "Optimized" + + // Mock: 10.4 CreateVolume — must receive clone request (create_new_from_attributes + CopyFrom) + mockClient.EXPECT().CreateVolume(gomock.Any(), symmetrixID, gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, _ string, req types.CreateVolumesRequest, _ ...http.Header) (*types.CreateVolumesResponse, error) { + vol := req.Volumes[0] + // Verify: create_new_from_attributes set with CYL size from the request (547 cyl = 1 GiB) + assert.NotNil(t, vol.CreateNew.CreateNewFromAttributes) + assert.Equal(t, "CYL", vol.CreateNew.CreateNewFromAttributes.CapacityUnit) + assert.Equal(t, float64(547), vol.CreateNew.CreateNewFromAttributes.VolumeSize) + // Verify: NO create_new_from_snapshot (this is a clone, not snapshot restore) + assert.Nil(t, vol.CreateNew.CreateNewFromSnapshot) + // Verify: manage_replication CopyFrom with establish_terminate + assert.NotNil(t, vol.Actions.ManageReplication) + assert.NotNil(t, vol.Actions.ManageReplication.Local) + assert.Equal(t, "CopyFrom", vol.Actions.ManageReplication.Local.Action) + assert.Equal(t, srcDevID, vol.Actions.ManageReplication.Local.Volume.ID) + assert.NotNil(t, vol.Actions.ManageReplication.Local.EstablishTerminate) + assert.True(t, *vol.Actions.ManageReplication.Local.EstablishTerminate) + + return &types.CreateVolumesResponse{ + Summary: types.ResponseSummary{Total: 1, Succeeded: 1}, + Results: types.CreateVolumesResults{ + Result: []types.CreateVolumeResponseItem{ + { + Status: "success", + Volume: &types.VolumeRefResponse{ + ID: targetDevID, + Identifier: volumeIdentifier, + }, + }, + }, + }, + }, nil + }).Times(1) + + deps := &snapshotDeps{service: &service{}, licenseErr: nil} + creator := &u4p104VolumeCreator{ + s: deps, + pmaxClient: mockClient, + pmaxClient104: mockClient, + symmetrixID: symmetrixID, + params: map[string]string{StoragePoolParam: storagePoolID, ServiceLevelParam: serviceLevel}, + } + + req := &csi.CreateVolumeRequest{ + Name: "clone-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 1073741824}, + Parameters: map[string]string{StoragePoolParam: storagePoolID, ServiceLevelParam: serviceLevel}, + VolumeContentSource: &csi.VolumeContentSource{ + Type: &csi.VolumeContentSource_Volume{ + Volume: &csi.VolumeContentSource_VolumeSource{ + VolumeId: srcCSIVolumeID, + }, + }, + }, + } + + resp, err := creator.Create(context.Background(), req) + assert.NoError(t, err) + assert.NotNil(t, resp) + assert.Equal(t, volumeIdentifier+"-"+symmetrixID+"-"+targetDevID, resp.Volume.VolumeId) + // ContentSource in context should reference the source CSI volume ID + assert.Equal(t, srcCSIVolumeID, resp.Volume.VolumeContext[ContentSource]) + // CSI ContentSource must be set + assert.NotNil(t, resp.Volume.ContentSource) +} + +func TestU4P104VolumeCreator_CloneEmptySourceID(t *testing.T) { + creator := &u4p104VolumeCreator{s: &service{}} + req := &csi.CreateVolumeRequest{ + Name: "clone-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 1073741824}, + Parameters: map[string]string{StoragePoolParam: "SRP_1"}, + VolumeContentSource: &csi.VolumeContentSource{ + Type: &csi.VolumeContentSource_Volume{ + Volume: &csi.VolumeContentSource_VolumeSource{ + VolumeId: "", + }, + }, + }, + } + resp, err := creator.Create(context.Background(), req) + assert.Nil(t, resp) + assert.Error(t, err) + assert.Contains(t, err.Error(), "Source volume ID is required") +} + +func TestU4P104VolumeCreator_CloneBadSourceFormat(t *testing.T) { + creator := &u4p104VolumeCreator{s: &service{}} + req := &csi.CreateVolumeRequest{ + Name: "clone-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 1073741824}, + Parameters: map[string]string{StoragePoolParam: "SRP_1"}, + VolumeContentSource: &csi.VolumeContentSource{ + Type: &csi.VolumeContentSource_Volume{ + Volume: &csi.VolumeContentSource_VolumeSource{ + VolumeId: "bad-format", + }, + }, + }, + } + resp, err := creator.Create(context.Background(), req) + assert.Nil(t, resp) + assert.Error(t, err) + assert.Contains(t, err.Error(), "not in supported format") +} + +func TestU4P104VolumeCreator_CloneCrossArrayFails(t *testing.T) { + creator := &u4p104VolumeCreator{ + s: &service{}, + symmetrixID: "000197900049", + } + // Source is on a different array + req := &csi.CreateVolumeRequest{ + Name: "clone-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 1073741824}, + Parameters: map[string]string{StoragePoolParam: "SRP_1"}, + VolumeContentSource: &csi.VolumeContentSource{ + Type: &csi.VolumeContentSource_Volume{ + Volume: &csi.VolumeContentSource_VolumeSource{ + VolumeId: "csi--src-vol-000197900099-00SRC", + }, + }, + }, + } + resp, err := creator.Create(context.Background(), req) + assert.Nil(t, resp) + assert.Error(t, err) + assert.Contains(t, err.Error(), "same PowerMax array") +} + +func TestU4P104VolumeCreator_CloneLargerCapacitySucceeds(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := mocks.NewMockPmaxClient(ctrl) + symmetrixID := "000197900049" + srcDevID := "00SRC" + targetDevID := "00TGT" + volumeIdentifier := "csi--clone-vol" + + // CreateVolume should be called with the larger requested size (1094 cyl) from the request + mockClient.EXPECT().CreateVolume(gomock.Any(), symmetrixID, gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, _ string, req types.CreateVolumesRequest, _ ...http.Header) (*types.CreateVolumesResponse, error) { + // Verify: create_new_from_attributes with 1093 cylinders (2 GiB) + assert.NotNil(t, req.Volumes[0].CreateNew.CreateNewFromAttributes) + assert.Equal(t, "CYL", req.Volumes[0].CreateNew.CreateNewFromAttributes.CapacityUnit) + assert.Equal(t, float64(1093), req.Volumes[0].CreateNew.CreateNewFromAttributes.VolumeSize) + // Verify: CopyFrom the source + assert.NotNil(t, req.Volumes[0].Actions.ManageReplication) + assert.Equal(t, "CopyFrom", req.Volumes[0].Actions.ManageReplication.Local.Action) + assert.Equal(t, srcDevID, req.Volumes[0].Actions.ManageReplication.Local.Volume.ID) + + return &types.CreateVolumesResponse{ + Summary: types.ResponseSummary{Total: 1, Succeeded: 1}, + Results: types.CreateVolumesResults{ + Result: []types.CreateVolumeResponseItem{ + { + Status: "success", + Volume: &types.VolumeRefResponse{ + ID: targetDevID, + Identifier: volumeIdentifier, + }, + }, + }, + }, + }, nil + }).Times(1) + + deps := &snapshotDeps{service: &service{}, licenseErr: nil} + creator := &u4p104VolumeCreator{ + s: deps, + pmaxClient: mockClient, + pmaxClient104: mockClient, + symmetrixID: symmetrixID, + params: map[string]string{StoragePoolParam: "SRP_1", ServiceLevelParam: "Optimized"}, + } + + // Request 2 GiB (1094 cylinders) — larger than source + req := &csi.CreateVolumeRequest{ + Name: "clone-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 2147483648}, + Parameters: map[string]string{StoragePoolParam: "SRP_1", ServiceLevelParam: "Optimized"}, + VolumeContentSource: &csi.VolumeContentSource{ + Type: &csi.VolumeContentSource_Volume{ + Volume: &csi.VolumeContentSource_VolumeSource{ + VolumeId: "csi--src-vol-" + symmetrixID + "-00SRC", + }, + }, + }, + } + + resp, err := creator.Create(context.Background(), req) + assert.NoError(t, err) + assert.NotNil(t, resp) + assert.Contains(t, resp.Volume.VolumeId, targetDevID) +} + +func TestU4P104VolumeCreator_IdempotentGetVolumesByIdentifierFailure(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := mocks.NewMockPmaxClient(ctrl) + symmetrixID := "000197900049" + + // CreateVolume returns nil volume (fully idempotent), then GetVolumesByIdentifier fails. + mockClient.EXPECT().CreateVolume(gomock.Any(), symmetrixID, gomock.Any(), gomock.Any()). + Return(&types.CreateVolumesResponse{ + Summary: types.ResponseSummary{Total: 1, Succeeded: 1}, + Results: types.CreateVolumesResults{ + Result: []types.CreateVolumeResponseItem{ + {Status: "success", Volume: nil}, + }, + }, + }, nil).Times(1) + mockClient.EXPECT().GetVolumesByIdentifier(gomock.Any(), symmetrixID, gomock.Any()). + Return(nil, errors.New("connection timeout")).Times(1) + + creator := &u4p104VolumeCreator{ + s: &service{}, + pmaxClient: mockClient, + pmaxClient104: mockClient, + symmetrixID: symmetrixID, + params: map[string]string{StoragePoolParam: "SRP_1", ServiceLevelParam: "Optimized"}, + } + + req := &csi.CreateVolumeRequest{ + Name: "test-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 1073741824}, + Parameters: map[string]string{StoragePoolParam: "SRP_1", ServiceLevelParam: "Optimized"}, + } + + resp, err := creator.Create(context.Background(), req) + assert.Error(t, err) + assert.Nil(t, resp) + assert.Contains(t, err.Error(), "idempotency check failed") +} + +func TestU4P104VolumeCreator_IdempotentEmptyVolumeList(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := mocks.NewMockPmaxClient(ctrl) + symmetrixID := "000197900049" + + // CreateVolume returns nil volume (fully idempotent), but GetVolumesByIdentifier returns empty. + mockClient.EXPECT().CreateVolume(gomock.Any(), symmetrixID, gomock.Any(), gomock.Any()). + Return(&types.CreateVolumesResponse{ + Summary: types.ResponseSummary{Total: 1, Succeeded: 1}, + Results: types.CreateVolumesResults{ + Result: []types.CreateVolumeResponseItem{ + {Status: "success", Volume: nil}, + }, + }, + }, nil).Times(1) + mockClient.EXPECT().GetVolumesByIdentifier(gomock.Any(), symmetrixID, gomock.Any()). + Return(&types.Volumev1{Volumes: []types.VolumeEnhanced{}}, nil).Times(1) + + creator := &u4p104VolumeCreator{ + s: &service{}, + pmaxClient: mockClient, + pmaxClient104: mockClient, + symmetrixID: symmetrixID, + params: map[string]string{StoragePoolParam: "SRP_1", ServiceLevelParam: "Optimized"}, + } + + req := &csi.CreateVolumeRequest{ + Name: "ghost-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 1073741824}, + Parameters: map[string]string{StoragePoolParam: "SRP_1", ServiceLevelParam: "Optimized"}, + } + + resp, err := creator.Create(context.Background(), req) + assert.Error(t, err) + assert.Nil(t, resp) + assert.Contains(t, err.Error(), "reported as existing but not found") +} + +func TestU4P104VolumeCreator_IdempotentCapacityBytes(t *testing.T) { + // Verify that CapacityBytes is correctly computed in the idempotent fallback path. + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := mocks.NewMockPmaxClient(ctrl) + mockService := &service{} + + symmetrixID := "000197900049" + deviceID := "00DEF" + volumeIdentifier := "csi--cap-vol" + // 1 GiB → ceil(1073741824 / 1966080) = 547 cylinders + expectedCylinders := 547 + expectedCapacityBytes := int64(expectedCylinders) * 1966080 + + mockClient.EXPECT().CreateVolume(gomock.Any(), symmetrixID, gomock.Any(), gomock.Any()). + Return(&types.CreateVolumesResponse{ + Summary: types.ResponseSummary{Total: 1, Succeeded: 1}, + Results: types.CreateVolumesResults{ + Result: []types.CreateVolumeResponseItem{ + {Status: "success", Volume: nil}, + }, + }, + }, nil).Times(1) + mockClient.EXPECT().GetVolumesByIdentifier(gomock.Any(), symmetrixID, volumeIdentifier). + Return(&types.Volumev1{ + Volumes: []types.VolumeEnhanced{ + { + ID: deviceID, + Identifier: volumeIdentifier, + CapCyl: float64(expectedCylinders), + }, + }, + }, nil).Times(1) + + creator := &u4p104VolumeCreator{ + s: mockService, + pmaxClient: mockClient, + pmaxClient104: mockClient, + symmetrixID: symmetrixID, + params: map[string]string{StoragePoolParam: "SRP_1", ServiceLevelParam: "Optimized"}, + } + + req := &csi.CreateVolumeRequest{ + Name: "cap-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 1073741824}, + Parameters: map[string]string{StoragePoolParam: "SRP_1", ServiceLevelParam: "Optimized"}, + } + + resp, err := creator.Create(context.Background(), req) + assert.NoError(t, err) + assert.NotNil(t, resp) + assert.Equal(t, expectedCapacityBytes, resp.Volume.CapacityBytes) +} + +func TestU4P104VolumeCreator_NamespaceInIdentifier(t *testing.T) { + // Verify namespace is appended to volumeIdentifier when CSIPVCNamespace is present. + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := mocks.NewMockPmaxClient(ctrl) + mockService := &service{} + + symmetrixID := "000197900049" + deviceID := "00XYZ" + volumeName := "ns-vol" + namespace := "prod-ns" + // With empty clusterPrefix: "csi--ns-vol-prod-ns" + expectedIdentifier := "csi--" + volumeName + "-" + namespace + + mockClient.EXPECT().CreateVolume(gomock.Any(), symmetrixID, gomock.Any(), gomock.Any()). + Return(&types.CreateVolumesResponse{ + Summary: types.ResponseSummary{Total: 1, Succeeded: 1}, + Results: types.CreateVolumesResults{ + Result: []types.CreateVolumeResponseItem{ + { + Status: "success", + Volume: &types.VolumeRefResponse{ + ID: deviceID, + Identifier: expectedIdentifier, + }, + }, + }, + }, + }, nil).Times(1) + + creator := &u4p104VolumeCreator{ + s: mockService, + pmaxClient: mockClient, + pmaxClient104: mockClient, + symmetrixID: symmetrixID, + params: map[string]string{StoragePoolParam: "SRP_1", ServiceLevelParam: "Optimized"}, + } + + req := &csi.CreateVolumeRequest{ + Name: volumeName, + CapacityRange: &csi.CapacityRange{RequiredBytes: 1073741824}, + Parameters: map[string]string{ + StoragePoolParam: "SRP_1", + ServiceLevelParam: "Optimized", + CSIPVCNamespace: namespace, + }, + } + + resp, err := creator.Create(context.Background(), req) + assert.NoError(t, err) + assert.NotNil(t, resp) + assert.Contains(t, resp.Volume.VolumeId, expectedIdentifier) +} + +func TestU4P104VolumeCreator_EmptyDeviceID(t *testing.T) { + // If the array returns a volume object with an empty device ID, the driver must + // return an error rather than silently building a broken VolumeId. + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := mocks.NewMockPmaxClient(ctrl) + symmetrixID := "000197900049" + + mockClient.EXPECT().CreateVolume(gomock.Any(), symmetrixID, gomock.Any(), gomock.Any()). + Return(&types.CreateVolumesResponse{ + Summary: types.ResponseSummary{Total: 1, Succeeded: 1}, + Results: types.CreateVolumesResults{ + Result: []types.CreateVolumeResponseItem{ + { + Status: "success", + Volume: &types.VolumeRefResponse{ + ID: "", // empty — should trigger an error + Identifier: "csi--empty-id-vol", + }, + }, + }, + }, + }, nil).Times(1) + + creator := &u4p104VolumeCreator{ + s: &service{}, + pmaxClient: mockClient, + pmaxClient104: mockClient, + symmetrixID: symmetrixID, + } + + req := &csi.CreateVolumeRequest{ + Name: "empty-id-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 1073741824}, + Parameters: map[string]string{StoragePoolParam: "SRP_1", ServiceLevelParam: "Optimized"}, + } + + resp, err := creator.Create(context.Background(), req) + assert.Error(t, err) + assert.Nil(t, resp) + assert.Contains(t, err.Error(), "empty device ID") +} + +func TestU4P104VolumeCreator_MultiSGResolvesTargetSG(t *testing.T) { + // Volume already belongs to Bronze SG. Request adds it to Diamond SG. + // The response volume.storage_groups lists both, but the top-level storage_group + // object correctly identifies the target SG (Diamond). We must use that. + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := mocks.NewMockPmaxClient(ctrl) + mockService := &service{} + + symmetrixID := "000197900049" + deviceID := "001FA" + volumeIdentifier := "csi--multi-sg-vol" + targetSG := "csi--Diamond-SRP_1-SG" + otherSG := "csi--Bronze-SRP_1-SG" + + mockClient.EXPECT().CreateVolume(gomock.Any(), symmetrixID, gomock.Any(), gomock.Any()). + Return(&types.CreateVolumesResponse{ + Summary: types.ResponseSummary{Total: 1, Succeeded: 1}, + Results: types.CreateVolumesResults{ + Result: []types.CreateVolumeResponseItem{ + { + Volume: &types.VolumeRefResponse{ + ID: deviceID, + Identifier: volumeIdentifier, + CapCyl: 546.0, + StorageGroups: []types.StorageGroupID{ + {StorageGroupID: otherSG}, + {StorageGroupID: targetSG}, + }, + }, + StorageGroup: &types.StorageGroupRefResponse{ + ID: targetSG, + NumOfVolumes: 5, + }, + Status: "success", + }, + }, + }, + }, nil).Times(1) + + creator := &u4p104VolumeCreator{ + s: mockService, + pmaxClient: mockClient, + pmaxClient104: mockClient, + symmetrixID: symmetrixID, + } + + req := &csi.CreateVolumeRequest{ + Name: "multi-sg-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 1073741824}, + Parameters: map[string]string{StoragePoolParam: "SRP_1", ServiceLevelParam: "Diamond"}, + } + + resp, err := creator.Create(context.Background(), req) + assert.NoError(t, err) + assert.NotNil(t, resp) + // Must resolve to the target SG (Diamond), not the first SG in the list (Bronze) + assert.Equal(t, targetSG, resp.Volume.VolumeContext[StorageGroup]) +} + +func TestU4P104VolumeCreator_TenantPrefixInIdentifier(t *testing.T) { + // When the array applies a tenant prefix to the volume identifier, + // the VolumeId must use the array-returned identifier, not the locally-built one. + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := mocks.NewMockPmaxClient(ctrl) + mockService := &service{} + + symmetrixID := "000197900049" + deviceID := "00TPX" + localIdentifier := "csi--tenant-vol" + // Array returns identifier with tenant prefix + arrayIdentifier := "tn1-csi--tenant-vol" + + mockClient.EXPECT().CreateVolume(gomock.Any(), symmetrixID, gomock.Any(), gomock.Any()). + Return(&types.CreateVolumesResponse{ + Summary: types.ResponseSummary{Total: 1, Succeeded: 1}, + Results: types.CreateVolumesResults{ + Result: []types.CreateVolumeResponseItem{ + { + Status: "success", + Volume: &types.VolumeRefResponse{ + ID: deviceID, + Identifier: arrayIdentifier, + StorageGroups: []types.StorageGroupID{ + {StorageGroupID: "tn1-csi--Optimized-SRP_1-SG"}, + }, + }, + StorageGroup: &types.StorageGroupRefResponse{ + ID: "tn1-csi--Optimized-SRP_1-SG", + }, + }, + }, + }, + }, nil).Times(1) + + creator := &u4p104VolumeCreator{ + s: mockService, + pmaxClient: mockClient, + pmaxClient104: mockClient, + symmetrixID: symmetrixID, + } + + req := &csi.CreateVolumeRequest{ + Name: "tenant-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 1073741824}, + Parameters: map[string]string{StoragePoolParam: "SRP_1", ServiceLevelParam: "Optimized"}, + } + + resp, err := creator.Create(context.Background(), req) + assert.NoError(t, err) + assert.NotNil(t, resp) + // VolumeId must use array-returned identifier (with tenant prefix), not local one + expectedVolumeID := arrayIdentifier + "-" + symmetrixID + "-" + deviceID + assert.Equal(t, expectedVolumeID, resp.Volume.VolumeId) + // Confirm the tenant prefix is present (not the local identifier without prefix) + assert.True(t, len(resp.Volume.VolumeId) > len(localIdentifier+"-"+symmetrixID+"-"+deviceID), + "VolumeId should be longer due to tenant prefix") + // SG must come from the response too + assert.Equal(t, "tn1-csi--Optimized-SRP_1-SG", resp.Volume.VolumeContext[StorageGroup]) +} + +func TestU4P104VolumeCreator_IdempotentSGFromCreateVolumeResponse(t *testing.T) { + // Fully idempotent path: CreateVolume returns nil volume but a StorageGroup object. + // The SG name must come from result.StorageGroup.ID in the CreateVolume response, + // not the locally-constructed name (supports tenant-prefixed SG names). + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := mocks.NewMockPmaxClient(ctrl) + symmetrixID := "000197900049" + deviceID := "00SGR" + volumeIdentifier := "csi--sg-vol" + // Array returns a tenant-prefixed SG name + arraySGName := "tn1-csi--Optimized-SRP_1-SG" + + mockClient.EXPECT().CreateVolume(gomock.Any(), symmetrixID, gomock.Any(), gomock.Any()). + Return(&types.CreateVolumesResponse{ + Summary: types.ResponseSummary{Total: 1, Succeeded: 1}, + Results: types.CreateVolumesResults{ + Result: []types.CreateVolumeResponseItem{ + { + Status: "success", + Volume: nil, + StorageGroup: &types.StorageGroupRefResponse{ + ID: arraySGName, + }, + }, + }, + }, + }, nil).Times(1) + mockClient.EXPECT().GetVolumesByIdentifier(gomock.Any(), symmetrixID, volumeIdentifier). + Return(&types.Volumev1{ + Volumes: []types.VolumeEnhanced{ + { + ID: deviceID, + Identifier: volumeIdentifier, + CapCyl: 547, + }, + }, + }, nil).Times(1) + + creator := &u4p104VolumeCreator{ + s: &service{}, + pmaxClient: mockClient, + pmaxClient104: mockClient, + symmetrixID: symmetrixID, + } + + req := &csi.CreateVolumeRequest{ + Name: "sg-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 1073741824}, + Parameters: map[string]string{StoragePoolParam: "SRP_1", ServiceLevelParam: "Optimized"}, + } + + resp, err := creator.Create(context.Background(), req) + assert.NoError(t, err) + assert.NotNil(t, resp) + // SG must come from result.StorageGroup.ID, not the locally-built "csi--Optimized-SRP_1-SG" + assert.Equal(t, arraySGName, resp.Volume.VolumeContext[StorageGroup]) +} + +func TestComputeRequiredCylinders(t *testing.T) { + tests := []struct { + name string + requiredBytes int64 + limitBytes int64 + expectedCyl int + expectError bool + errorContains string + }{ + { + name: "nil CapacityRange uses default", + // DefaultVolumeSizeBytes=1073741824 → ceil(1073741824/1966080) = 547 + expectedCyl: 547, + }, + { + name: "1 GiB request", + requiredBytes: 1073741824, + expectedCyl: 547, + }, + { + name: "exact cylinder boundary above minimum", + requiredBytes: 51118080, // MinVolumeSizeBytes = 26 * 1966080 + expectedCyl: 26, + }, + { + name: "partial cylinder above minimum rounds up", + requiredBytes: 51118081, + expectedCyl: 27, + }, + { + name: "zero bytes uses default", + // DefaultVolumeSizeBytes=1073741824 → 547 cylinders + expectedCyl: 547, + }, + { + name: "below minimum uses minimum", + requiredBytes: 1, + // MinVolumeSizeBytes=51118080 → ceil(51118080/1966080) = 26 + expectedCyl: 26, + }, + { + name: "negative required bytes", + requiredBytes: -1, + expectError: true, + errorContains: "must not be negative", + }, + { + name: "negative limit bytes", + requiredBytes: 1073741824, + limitBytes: -1, + expectError: true, + errorContains: "must not be negative", + }, + { + name: "exceeds limit", + requiredBytes: 1073741824, + limitBytes: 1073741824, // aligned size (547 cyl * 1966080) > 1073741824 + expectError: true, + errorContains: "exceeds limit", + }, + { + name: "within limit", + requiredBytes: 1073741824, + limitBytes: 1075445760, // exactly 547 * 1966080 + expectedCyl: 547, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var cr *csi.CapacityRange + if tt.requiredBytes != 0 || tt.limitBytes != 0 { + cr = &csi.CapacityRange{ + RequiredBytes: tt.requiredBytes, + LimitBytes: tt.limitBytes, + } + } + cyl, err := computeRequiredCylinders(cr) + if tt.expectError { + assert.Error(t, err) + assert.Contains(t, err.Error(), tt.errorContains) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.expectedCyl, cyl) + } + }) + } +} + +func TestBuildCreateVolumesRequest(t *testing.T) { + req := buildCreateVolumesRequest(547, "SRP_1", "Optimized", "csi-my-SG", "csi-my-vol", nil, "req-42") + + assert.Equal(t, "req-42", req.Volumes[0].RequestID) + assert.Equal(t, types.ExecutionOptionSynchronous, req.ExecutionOption) + assert.Len(t, req.Volumes, 1) + + vol := req.Volumes[0] + // volume.identifier set for native idempotency + assert.NotNil(t, vol.Volume) + assert.Equal(t, "csi-my-vol", vol.Volume.Identifier) + // create_new set + assert.NotNil(t, vol.CreateNew) + assert.NotNil(t, vol.CreateNew.CreateNewFromAttributes) + assert.Equal(t, "CYL", vol.CreateNew.CreateNewFromAttributes.CapacityUnit) + assert.Equal(t, float64(547), vol.CreateNew.CreateNewFromAttributes.VolumeSize) + // precheck_srp_capacity set + assert.NotNil(t, vol.CreateNew.PrecheckSrpCapacity) + assert.Equal(t, "SRP_1", vol.CreateNew.PrecheckSrpCapacity.SRP.ID) + // actions: manage_volume_storage_group set, manage_identifier NOT set + assert.NotNil(t, vol.Actions) + assert.NotNil(t, vol.Actions.ManageVolumeStorageGroup) + assert.Equal(t, "Add", vol.Actions.ManageVolumeStorageGroup.Action) + assert.Equal(t, "csi-my-SG", vol.Actions.ManageVolumeStorageGroup.StorageGroup.ID) + assert.NotNil(t, vol.Actions.ManageVolumeStorageGroup.StorageGroup.SRP) + assert.Equal(t, "SRP_1", vol.Actions.ManageVolumeStorageGroup.StorageGroup.SRP.ID) + assert.NotNil(t, vol.Actions.ManageVolumeStorageGroup.StorageGroup.ServiceLevel) + assert.Equal(t, "Optimized", vol.Actions.ManageVolumeStorageGroup.StorageGroup.ServiceLevel.ID) + assert.Nil(t, vol.Actions.ManageVolumeStorageGroup.StorageGroup.HostIOLimitInfo) + assert.Nil(t, vol.Actions.ManageIdentifier) // must NOT be set — causes 500 + // response_select at per-volume level + assert.Equal(t, "id,identifier,cap_cyl,storage_groups", vol.ResponseSelect) +} + +func TestBuildCreateVolumesRequest_WithHostIOLimitInfo(t *testing.T) { + hostIOLimitInfo := &types.HostIOLimitInfo{ + HostIOLimitMBSec: 1000, + HostIOLimitIOSec: 5000, + DynamicDistribution: "Always", + } + req := buildCreateVolumesRequest(547, "SRP_1", "Optimized", "csi-my-SG", "csi-my-vol", hostIOLimitInfo, "req-42") + + assert.Len(t, req.Volumes, 1) + vol := req.Volumes[0] + assert.NotNil(t, vol.Actions) + assert.NotNil(t, vol.Actions.ManageVolumeStorageGroup) + assert.NotNil(t, vol.Actions.ManageVolumeStorageGroup.StorageGroup.HostIOLimitInfo) + assert.Equal(t, 1000, vol.Actions.ManageVolumeStorageGroup.StorageGroup.HostIOLimitInfo.HostIOLimitMBSec) + assert.Equal(t, 5000, vol.Actions.ManageVolumeStorageGroup.StorageGroup.HostIOLimitInfo.HostIOLimitIOSec) + assert.Equal(t, "Always", vol.Actions.ManageVolumeStorageGroup.StorageGroup.HostIOLimitInfo.DynamicDistribution) +} + +func TestBuildVolumeContext(t *testing.T) { + ctx := buildVolumeContext("", "000197900049", "Optimized", "SRP_1", "csi-test-sg", 547, "") + assert.Equal(t, "Optimized", ctx[ServiceLevelParam]) + assert.Equal(t, "SRP_1", ctx[StoragePoolParam]) + assert.Equal(t, "000197900049", ctx[SymmetrixIDParam]) + assert.Equal(t, "547.00", ctx[CapacityGB]) + assert.Equal(t, "", ctx[ContentSource]) + assert.Equal(t, "csi-test-sg", ctx[StorageGroup]) + assert.NotEmpty(t, ctx["CreationTime"]) +} + +func TestBuildVolumeContext_WithContentSource(t *testing.T) { + csiSnapID := "snap1-000197900049-00ABC" + ctx := buildVolumeContext("", "000197900049", "Optimized", "SRP_1", "csi-test-sg", 547, csiSnapID) + assert.Equal(t, csiSnapID, ctx[ContentSource]) +} + +// --------------------------------------------------------------------------- +// Stage 8 — Snapshot restore unit tests +// --------------------------------------------------------------------------- + +// snapshotDeps is a test-local U4P104ServiceDeps that controls snapshot license outcomes +// without requiring a real Unisphere connection. +type snapshotDeps struct { + *service + licenseErr error +} + +func (d *snapshotDeps) isSnapshotLicensed(_ context.Context, _ string, _ pmax.Pmax) error { + return d.licenseErr +} + +func newSnapshotCreator(ctrl *gomock.Controller, symmetrixID string, licenseErr error) (*u4p104VolumeCreator, *mocks.MockPmaxClient) { + mc := mocks.NewMockPmaxClient(ctrl) + deps := &snapshotDeps{service: &service{}, licenseErr: licenseErr} + return &u4p104VolumeCreator{ + s: deps, + pmaxClient: mc, + pmaxClient104: mc, + symmetrixID: symmetrixID, + }, mc +} + +// mockSnapshotID is the fake snap_id returned by GetSnapshotInfo in unit tests. +const mockSnapshotID = int64(100661523201) + +// mockSnapshotInfo builds a VolumeSnapshot with a single source generation carrying +// mockSnapshotID, matching the 10.4 API expectation. +func mockSnapshotInfo(snapName string) *types.VolumeSnapshot { + return &types.VolumeSnapshot{ + SnapshotName: snapName, + VolumeSnapshotSource: []types.VolumeSnapshotSource{ + {SnapshotName: snapName, SnapID: mockSnapshotID, TimeStamp: "12:22:16 Wed, 25 Mar 2026 +0000"}, + }, + } +} + +// csiSnapshotID builds a CSI snapshot ID in the format expected by parseCsiID: +// -- +func csiSnapshotID(snapName, symID, devID string) string { + return snapName + "-" + symID + "-" + devID +} + +func TestU4P104VolumeCreator_CreateFromSnapshot_Success(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + symmetrixID := "000197900049" + srcDevID := "00SRC" + snapName := "mysnap" + snapCSIID := csiSnapshotID(snapName, symmetrixID, srcDevID) + newDevID := "00NEW" + storagePoolID := "SRP_1" + serviceLevel := "Optimized" + sgName := "csi--Optimized-SRP_1-SG" + + creator, mc := newSnapshotCreator(ctrl, symmetrixID, nil) + + // Pre-flight: GetSnapshotInfo to obtain snap_id for 10.4 API + mc.EXPECT().GetSnapshotInfo(gomock.Any(), symmetrixID, srcDevID, snapName). + Return(mockSnapshotInfo(snapName), nil).Times(1) + + // Single 10.4 create call — verify the request body + mc.EXPECT().CreateVolume(gomock.Any(), symmetrixID, gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, _ string, req types.CreateVolumesRequest, _ ...http.Header) (*types.CreateVolumesResponse, error) { + vol := req.Volumes[0] + // Verify: create_new_from_snapshot with correct snap_id from GetSnapshotInfo + assert.NotNil(t, vol.CreateNew.CreateNewFromSnapshot) + assert.Equal(t, fmt.Sprintf("%d", mockSnapshotID), vol.CreateNew.CreateNewFromSnapshot.Snapshot.ID) + // Verify: new_volume_attributes set with CYL size matching the request + assert.NotNil(t, vol.CreateNew.CreateNewFromSnapshot.NewVolumeAttributes) + assert.Equal(t, "CYL", vol.CreateNew.CreateNewFromSnapshot.NewVolumeAttributes.CapacityUnit) + assert.Equal(t, float64(547), vol.CreateNew.CreateNewFromSnapshot.NewVolumeAttributes.VolumeSize) + // Verify: top-level create_new_from_attributes is NOT set (only nested inside snapshot) + assert.Nil(t, vol.CreateNew.CreateNewFromAttributes) + // Verify: manage_volume_storage_group present + assert.NotNil(t, vol.Actions.ManageVolumeStorageGroup) + assert.Equal(t, "Add", vol.Actions.ManageVolumeStorageGroup.Action) + // Verify: NO manage_replication (snapshots don't use CopyFrom) + assert.Nil(t, vol.Actions.ManageReplication) + + return &types.CreateVolumesResponse{ + Summary: types.ResponseSummary{Total: 1, Succeeded: 1}, + Results: types.CreateVolumesResults{ + Result: []types.CreateVolumeResponseItem{ + { + Status: "success", + Volume: &types.VolumeRefResponse{ + ID: newDevID, + Identifier: "csi--snap-vol", + }, + StorageGroup: &types.StorageGroupRefResponse{ID: sgName}, + }, + }, + }, + }, nil + }).Times(1) + + req := &csi.CreateVolumeRequest{ + Name: "snap-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 1073741824}, + Parameters: map[string]string{StoragePoolParam: storagePoolID, ServiceLevelParam: serviceLevel}, + VolumeContentSource: &csi.VolumeContentSource{ + Type: &csi.VolumeContentSource_Snapshot{ + Snapshot: &csi.VolumeContentSource_SnapshotSource{SnapshotId: snapCSIID}, + }, + }, + } + + resp, err := creator.Create(context.Background(), req) + assert.NoError(t, err) + assert.NotNil(t, resp) + assert.Contains(t, resp.Volume.VolumeId, newDevID) + assert.Contains(t, resp.Volume.VolumeId, symmetrixID) + // ContentSource must be set to the parsed snapshot name (legacy parity) + assert.Equal(t, snapName, resp.Volume.VolumeContext[ContentSource]) + // StorageGroup must come from the response + assert.Equal(t, sgName, resp.Volume.VolumeContext[StorageGroup]) +} + +func TestU4P104VolumeCreator_CreateFromSnapshot_IdempotentFallback(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + symmetrixID := "000197900049" + srcDevID := "00SRC" + snapName := "mysnap" + snapCSIID := csiSnapshotID(snapName, symmetrixID, srcDevID) + existingDevID := "00EXI" + volumeIdentifier := "csi--snap-idem-vol" + sgName := "csi--Optimized-SRP_1-SG" + + creator, mc := newSnapshotCreator(ctrl, symmetrixID, nil) + + // Pre-flight: GetSnapshotInfo to obtain snap_id + mc.EXPECT().GetSnapshotInfo(gomock.Any(), symmetrixID, srcDevID, "mysnap"). + Return(mockSnapshotInfo("mysnap"), nil).Times(1) + + // CreateVolume returns nil volume (fully idempotent) + mc.EXPECT().CreateVolume(gomock.Any(), symmetrixID, gomock.Any(), gomock.Any()). + Return(&types.CreateVolumesResponse{ + Summary: types.ResponseSummary{Total: 1, Succeeded: 1}, + Results: types.CreateVolumesResults{ + Result: []types.CreateVolumeResponseItem{ + {Status: "success", Volume: nil, StorageGroup: &types.StorageGroupRefResponse{ID: sgName}}, + }, + }, + }, nil).Times(1) + + // Idempotent fallback lookup + mc.EXPECT().GetVolumesByIdentifier(gomock.Any(), symmetrixID, gomock.Any()). + Return(&types.Volumev1{ + Volumes: []types.VolumeEnhanced{ + {ID: existingDevID, Identifier: volumeIdentifier, CapCyl: 547}, + }, + }, nil).Times(1) + + req := &csi.CreateVolumeRequest{ + Name: "snap-idem-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 1073741824}, + Parameters: map[string]string{StoragePoolParam: "SRP_1", ServiceLevelParam: "Optimized"}, + VolumeContentSource: &csi.VolumeContentSource{ + Type: &csi.VolumeContentSource_Snapshot{ + Snapshot: &csi.VolumeContentSource_SnapshotSource{SnapshotId: snapCSIID}, + }, + }, + } + + resp, err := creator.Create(context.Background(), req) + assert.NoError(t, err) + assert.NotNil(t, resp) + assert.Contains(t, resp.Volume.VolumeId, existingDevID) + // ContentSource preserved through idempotent path (parsed snapshot name, not full CSI ID) + assert.Equal(t, snapName, resp.Volume.VolumeContext[ContentSource]) + // SG from CreateVolume response, not locally-computed + assert.Equal(t, sgName, resp.Volume.VolumeContext[StorageGroup]) +} + +func TestU4P104VolumeCreator_CreateFromSnapshot_InvalidSnapshotID(t *testing.T) { + creator := &u4p104VolumeCreator{s: &service{}, symmetrixID: "000197900049"} + + req := &csi.CreateVolumeRequest{ + Name: "bad-snap-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 1073741824}, + Parameters: map[string]string{StoragePoolParam: "SRP_1"}, + VolumeContentSource: &csi.VolumeContentSource{ + Type: &csi.VolumeContentSource_Snapshot{ + Snapshot: &csi.VolumeContentSource_SnapshotSource{SnapshotId: "bad"}, + }, + }, + } + + resp, err := creator.Create(context.Background(), req) + assert.Nil(t, resp) + assert.Error(t, err) + assert.Contains(t, err.Error(), "Snapshot identifier not in supported format") +} + +func TestU4P104VolumeCreator_CreateFromSnapshot_SourceArrayMismatch(t *testing.T) { + // Snapshot CSI ID references a different array than the creator's symmetrixID + creator := &u4p104VolumeCreator{s: &service{}, symmetrixID: "000197900049"} + + otherArray := "000197900046" + snapCSIID := csiSnapshotID("mysnap", otherArray, "00SRC") + + req := &csi.CreateVolumeRequest{ + Name: "mismatch-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 1073741824}, + Parameters: map[string]string{StoragePoolParam: "SRP_1"}, + VolumeContentSource: &csi.VolumeContentSource{ + Type: &csi.VolumeContentSource_Snapshot{ + Snapshot: &csi.VolumeContentSource_SnapshotSource{SnapshotId: snapCSIID}, + }, + }, + } + + resp, err := creator.Create(context.Background(), req) + assert.Nil(t, resp) + assert.Error(t, err) + assert.Contains(t, err.Error(), "The volume content source is in different PowerMax array") +} + +func TestU4P104VolumeCreator_CreateFromSnapshot_SnapshotLicenseFailure(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + symmetrixID := "000197900049" + snapCSIID := csiSnapshotID("mysnap", symmetrixID, "00SRC") + + // License check returns error + creator, mc := newSnapshotCreator(ctrl, symmetrixID, errors.New("PowerMax array (000197900049) doesn't have Snapshot license")) + + // handleSnapshotSource calls GetSnapshotInfo before the license check + mc.EXPECT().GetSnapshotInfo(gomock.Any(), symmetrixID, "00SRC", "mysnap"). + Return(mockSnapshotInfo("mysnap"), nil).Times(1) + + req := &csi.CreateVolumeRequest{ + Name: "unlicensed-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 1073741824}, + Parameters: map[string]string{StoragePoolParam: "SRP_1"}, + VolumeContentSource: &csi.VolumeContentSource{ + Type: &csi.VolumeContentSource_Snapshot{ + Snapshot: &csi.VolumeContentSource_SnapshotSource{SnapshotId: snapCSIID}, + }, + }, + } + + resp, err := creator.Create(context.Background(), req) + assert.Nil(t, resp) + assert.Error(t, err) + assert.Contains(t, err.Error(), "Snapshot license") +} + +func TestU4P104VolumeCreator_CreateFromSnapshot_CreateAPIError(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + symmetrixID := "000197900049" + srcDevID := "00SRC" + snapCSIID := csiSnapshotID("mysnap", symmetrixID, srcDevID) + + creator, mc := newSnapshotCreator(ctrl, symmetrixID, nil) + + mc.EXPECT().GetSnapshotInfo(gomock.Any(), symmetrixID, srcDevID, "mysnap"). + Return(mockSnapshotInfo("mysnap"), nil).Times(1) + mc.EXPECT().CreateVolume(gomock.Any(), symmetrixID, gomock.Any(), gomock.Any()). + Return(nil, errors.New("backend error")).Times(1) + + req := &csi.CreateVolumeRequest{ + Name: "api-err-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 1073741824}, + Parameters: map[string]string{StoragePoolParam: "SRP_1", ServiceLevelParam: "Optimized"}, + VolumeContentSource: &csi.VolumeContentSource{ + Type: &csi.VolumeContentSource_Snapshot{ + Snapshot: &csi.VolumeContentSource_SnapshotSource{SnapshotId: snapCSIID}, + }, + }, + } + + resp, err := creator.Create(context.Background(), req) + assert.Nil(t, resp) + assert.Error(t, err) + assert.Contains(t, err.Error(), "backend error") +} + +func TestU4P104VolumeCreator_CreateFromSnapshot_EmptyCreateResponse(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + symmetrixID := "000197900049" + srcDevID := "00SRC" + snapCSIID := csiSnapshotID("mysnap", symmetrixID, srcDevID) + + creator, mc := newSnapshotCreator(ctrl, symmetrixID, nil) + + mc.EXPECT().GetSnapshotInfo(gomock.Any(), symmetrixID, srcDevID, "mysnap"). + Return(mockSnapshotInfo("mysnap"), nil).Times(1) + mc.EXPECT().CreateVolume(gomock.Any(), symmetrixID, gomock.Any(), gomock.Any()). + Return(&types.CreateVolumesResponse{Results: types.CreateVolumesResults{}}, nil).Times(1) + + req := &csi.CreateVolumeRequest{ + Name: "empty-resp-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 1073741824}, + Parameters: map[string]string{StoragePoolParam: "SRP_1", ServiceLevelParam: "Optimized"}, + VolumeContentSource: &csi.VolumeContentSource{ + Type: &csi.VolumeContentSource_Snapshot{ + Snapshot: &csi.VolumeContentSource_SnapshotSource{SnapshotId: snapCSIID}, + }, + }, + } + + resp, err := creator.Create(context.Background(), req) + assert.Nil(t, resp) + assert.Error(t, err) + assert.Contains(t, err.Error(), "create volume response is empty") +} + +func TestU4P104VolumeCreator_CreateFromSnapshot_SizeLargerSucceeds(t *testing.T) { + // When requested size > snapshot source, the 10.4 API now supports explicit + // size via new_volume_attributes, so the request should succeed. + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + symmetrixID := "000197900049" + srcDevID := "00SRC" + snapName := "mysnap" + snapCSIID := csiSnapshotID(snapName, symmetrixID, srcDevID) + newDevID := "00LRG" + volumeIdentifier := "csi--larger-vol" + + creator, mc := newSnapshotCreator(ctrl, symmetrixID, nil) + + // Pre-flight: GetSnapshotInfo to obtain snap_id + mc.EXPECT().GetSnapshotInfo(gomock.Any(), symmetrixID, srcDevID, snapName). + Return(mockSnapshotInfo(snapName), nil).Times(1) + + // CreateVolume should be called with larger size in new_volume_attributes (size from request) + mc.EXPECT().CreateVolume(gomock.Any(), symmetrixID, gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, _ string, req types.CreateVolumesRequest, _ ...http.Header) (*types.CreateVolumesResponse, error) { + // Verify: new_volume_attributes set with 1093 cylinders + assert.NotNil(t, req.Volumes[0].CreateNew.CreateNewFromSnapshot) + assert.NotNil(t, req.Volumes[0].CreateNew.CreateNewFromSnapshot.NewVolumeAttributes) + assert.Equal(t, "CYL", req.Volumes[0].CreateNew.CreateNewFromSnapshot.NewVolumeAttributes.CapacityUnit) + assert.Equal(t, float64(1093), req.Volumes[0].CreateNew.CreateNewFromSnapshot.NewVolumeAttributes.VolumeSize) + + return &types.CreateVolumesResponse{ + Summary: types.ResponseSummary{Total: 1, Succeeded: 1}, + Results: types.CreateVolumesResults{ + Result: []types.CreateVolumeResponseItem{ + { + Status: "success", + Volume: &types.VolumeRefResponse{ + ID: newDevID, + Identifier: volumeIdentifier, + }, + }, + }, + }, + }, nil + }).Times(1) + + req := &csi.CreateVolumeRequest{ + Name: "larger-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 2 * 1073741824}, // 1094 cyl > 547 + Parameters: map[string]string{StoragePoolParam: "SRP_1", ServiceLevelParam: "Optimized"}, + VolumeContentSource: &csi.VolumeContentSource{ + Type: &csi.VolumeContentSource_Snapshot{ + Snapshot: &csi.VolumeContentSource_SnapshotSource{SnapshotId: snapCSIID}, + }, + }, + } + + resp, err := creator.Create(context.Background(), req) + assert.NoError(t, err) + assert.NotNil(t, resp) + assert.Contains(t, resp.Volume.VolumeId, newDevID) +} + +func TestU4P104VolumeCreator_CreateFromSnapshot_GetSnapshotInfoFailure(t *testing.T) { + // When GetSnapshotInfo fails, the driver should return an error indicating + // the snapshot was not found on the source volume. + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + symmetrixID := "000197900049" + srcDevID := "00SRC" + snapCSIID := csiSnapshotID("mysnap", symmetrixID, srcDevID) + + creator, mc := newSnapshotCreator(ctrl, symmetrixID, nil) + + mc.EXPECT().GetSnapshotInfo(gomock.Any(), symmetrixID, srcDevID, "mysnap"). + Return(nil, errors.New("snapshot not found")).Times(1) + + req := &csi.CreateVolumeRequest{ + Name: "snap-info-err-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 1073741824}, + Parameters: map[string]string{StoragePoolParam: "SRP_1", ServiceLevelParam: "Optimized"}, + VolumeContentSource: &csi.VolumeContentSource{ + Type: &csi.VolumeContentSource_Snapshot{ + Snapshot: &csi.VolumeContentSource_SnapshotSource{SnapshotId: snapCSIID}, + }, + }, + } + + resp, err := creator.Create(context.Background(), req) + assert.Nil(t, resp) + assert.Error(t, err) + assert.Contains(t, err.Error(), "Snapshot mysnap not found on source volume 00SRC") +} + +// mockU4P104ServiceDeps is a mock implementation of U4P104ServiceDeps for testing +type mockU4P104ServiceDeps struct { + dynamicSGEnabled bool + clusterPrefix string + blockEnabled bool + arrayLabels map[string]string + // getDynamicSG mock return values + dynamicSGName string + dynamicSGCreated bool + dynamicSGErr error +} + +func (m *mockU4P104ServiceDeps) resolveParameter(params map[string]string, _, key, defaultVal string) string { + if val, ok := params[key]; ok { + return val + } + return defaultVal +} + +func (m *mockU4P104ServiceDeps) isDynamicSGEnabled() bool { + return m.dynamicSGEnabled +} + +func (m *mockU4P104ServiceDeps) getReplicationPrefix() string { + return "" +} + +func (m *mockU4P104ServiceDeps) getReplicationContextPrefix() string { + return "" +} + +func (m *mockU4P104ServiceDeps) getClusterPrefix() string { + return m.clusterPrefix +} + +func (m *mockU4P104ServiceDeps) parseCsiID(_ string) (string, string, string, string, string, error) { + return "", "", "", "", "", nil +} + +func (m *mockU4P104ServiceDeps) isSnapshotLicensed(_ context.Context, _ string, _ pmax.Pmax) error { + return nil +} + +func (m *mockU4P104ServiceDeps) getDynamicSG(_ context.Context, _, baseSGName string) (string, bool, error) { + if m.dynamicSGErr != nil { + return "", false, m.dynamicSGErr + } + if m.dynamicSGName != "" { + return m.dynamicSGName, m.dynamicSGCreated, nil + } + // Default: return baseSGName as existing SG + return baseSGName, false, nil +} + +func (m *mockU4P104ServiceDeps) getStorageArrayLabels(_ string) map[string]string { + return m.arrayLabels +} + +func (m *mockU4P104ServiceDeps) isBlockEnabled() bool { + return m.blockEnabled +} + +func TestU4P104VolumeCreator_DynamicSGEnabled_ProposesNewSG(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := mocks.NewMockPmaxClient(ctrl) + symmetrixID := "000197900049" + baseSGName := "csi--Optimized-SRP_1-SG" + newSGName := baseSGName + "--1" + deviceID := "00NEW" + volumeIdentifier := "csi--new-sg-vol" + + // Mock CreateVolume - 10.4 API will create the new SG automatically + mockClient.EXPECT().CreateVolume(gomock.Any(), symmetrixID, gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, _ string, req types.CreateVolumesRequest, _ ...http.Header) (*types.CreateVolumesResponse, error) { + // Verify the new SG name is used + assert.Equal(t, newSGName, req.Volumes[0].Actions.ManageVolumeStorageGroup.StorageGroup.ID) + // Verify SRP and ServiceLevel are passed for SG creation + assert.Equal(t, "SRP_1", req.Volumes[0].Actions.ManageVolumeStorageGroup.StorageGroup.SRP.ID) + assert.Equal(t, "Optimized", req.Volumes[0].Actions.ManageVolumeStorageGroup.StorageGroup.ServiceLevel.ID) + return &types.CreateVolumesResponse{ + Summary: types.ResponseSummary{Total: 1, Succeeded: 1}, + Results: types.CreateVolumesResults{ + Result: []types.CreateVolumeResponseItem{ + { + Status: "success", + Volume: &types.VolumeRefResponse{ + ID: deviceID, + Identifier: volumeIdentifier, + }, + StorageGroup: &types.StorageGroupRefResponse{ID: newSGName}, + }, + }, + }, + }, nil + }).Times(1) + + // Use mock service deps with getDynamicSG returning the new SG name + mockSvc := &mockU4P104ServiceDeps{ + dynamicSGEnabled: true, + dynamicSGName: newSGName, + dynamicSGCreated: true, + } + creator := &u4p104VolumeCreator{ + s: mockSvc, + pmaxClient: mockClient, + pmaxClient104: mockClient, + symmetrixID: symmetrixID, + } + + req := &csi.CreateVolumeRequest{ + Name: "new-sg-vol", + CapacityRange: &csi.CapacityRange{ + RequiredBytes: 1073741824, + }, + Parameters: map[string]string{ + StoragePoolParam: "SRP_1", + ServiceLevelParam: "Optimized", + }, + } + + resp, err := creator.Create(context.Background(), req) + assert.NoError(t, err) + assert.NotNil(t, resp) + assert.Equal(t, newSGName, resp.Volume.VolumeContext[StorageGroup]) +} + +func TestU4P104VolumeCreator_DynamicSGEnabled_GetDynamicSGError(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := mocks.NewMockPmaxClient(ctrl) + symmetrixID := "000197900049" + + // Use mock service deps with getDynamicSG returning an error + mockSvc := &mockU4P104ServiceDeps{ + dynamicSGEnabled: true, + dynamicSGErr: errors.New("array unreachable"), + } + creator := &u4p104VolumeCreator{ + s: mockSvc, + pmaxClient: mockClient, + pmaxClient104: mockClient, + symmetrixID: symmetrixID, + } + + req := &csi.CreateVolumeRequest{ + Name: "fail-vol", + CapacityRange: &csi.CapacityRange{ + RequiredBytes: 1073741824, + }, + Parameters: map[string]string{ + StoragePoolParam: "SRP_1", + ServiceLevelParam: "Optimized", + }, + } + + resp, err := creator.Create(context.Background(), req) + assert.Error(t, err) + assert.Nil(t, resp) + assert.Contains(t, err.Error(), "failed to get dynamic storage group") +} + +func TestU4P104VolumeCreator_DynamicSGEnabled_WithHostIOLimits(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := mocks.NewMockPmaxClient(ctrl) + symmetrixID := "000197900049" + baseSGName := "csi--Optimized-SRP_1-SG" + deviceID := "00IOL" + volumeIdentifier := "csi--io-limit-vol" + + mockClient.EXPECT().CreateVolume(gomock.Any(), symmetrixID, gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, _ string, req types.CreateVolumesRequest, _ ...http.Header) (*types.CreateVolumesResponse, error) { + // Verify host IO limits are passed + hostIOLimitInfo := req.Volumes[0].Actions.ManageVolumeStorageGroup.StorageGroup.HostIOLimitInfo + assert.NotNil(t, hostIOLimitInfo) + assert.Equal(t, 1000, hostIOLimitInfo.HostIOLimitMBSec) + assert.Equal(t, 5000, hostIOLimitInfo.HostIOLimitIOSec) + assert.Equal(t, "Always", hostIOLimitInfo.DynamicDistribution) + return &types.CreateVolumesResponse{ + Summary: types.ResponseSummary{Total: 1, Succeeded: 1}, + Results: types.CreateVolumesResults{ + Result: []types.CreateVolumeResponseItem{ + { + Status: "success", + Volume: &types.VolumeRefResponse{ + ID: deviceID, + Identifier: volumeIdentifier, + }, + StorageGroup: &types.StorageGroupRefResponse{ID: baseSGName}, + }, + }, + }, + }, nil + }).Times(1) + + // Use mock service deps with getDynamicSG returning the base SG name + mockSvc := &mockU4P104ServiceDeps{ + dynamicSGEnabled: true, + dynamicSGName: baseSGName, + dynamicSGCreated: false, + } + creator := &u4p104VolumeCreator{ + s: mockSvc, + pmaxClient: mockClient, + pmaxClient104: mockClient, + symmetrixID: symmetrixID, + } + + req := &csi.CreateVolumeRequest{ + Name: "io-limit-vol", + CapacityRange: &csi.CapacityRange{ + RequiredBytes: 1073741824, + }, + Parameters: map[string]string{ + StoragePoolParam: "SRP_1", + ServiceLevelParam: "Optimized", + HostIOLimitMBSecParam: "1000", + HostIOLimitIOSecParam: "5000", + DynamicDistributionParam: "Always", + }, + } + + resp, err := creator.Create(context.Background(), req) + assert.NoError(t, err) + assert.NotNil(t, resp) +} + +func TestU4P104VolumeCreator_CreateFromSnapshot_ResponseParity(t *testing.T) { + // Verify VolumeId format and VolumeContext keys/values match legacy conventions. + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + symmetrixID := "000197900049" + srcDevID := "00SRC" + snapName := "mysnap" + snapCSIID := csiSnapshotID(snapName, symmetrixID, srcDevID) + newDevID := "00PAR" + volIdentifier := "csi--parity-vol" + sgName := "csi--Optimized-SRP_1-SG" + + creator, mc := newSnapshotCreator(ctrl, symmetrixID, nil) + + mc.EXPECT().GetSnapshotInfo(gomock.Any(), symmetrixID, srcDevID, snapName). + Return(mockSnapshotInfo(snapName), nil).Times(1) + + mc.EXPECT().CreateVolume(gomock.Any(), symmetrixID, gomock.Any(), gomock.Any()). + Return(&types.CreateVolumesResponse{ + Summary: types.ResponseSummary{Total: 1, Succeeded: 1}, + Results: types.CreateVolumesResults{ + Result: []types.CreateVolumeResponseItem{ + { + Status: "success", + Volume: &types.VolumeRefResponse{ + ID: newDevID, + Identifier: volIdentifier, + }, + StorageGroup: &types.StorageGroupRefResponse{ID: sgName}, + }, + }, + }, + }, nil).Times(1) + + req := &csi.CreateVolumeRequest{ + Name: "parity-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 1073741824}, + Parameters: map[string]string{StoragePoolParam: "SRP_1", ServiceLevelParam: "Optimized"}, + VolumeContentSource: &csi.VolumeContentSource{ + Type: &csi.VolumeContentSource_Snapshot{ + Snapshot: &csi.VolumeContentSource_SnapshotSource{SnapshotId: snapCSIID}, + }, + }, + } + + resp, err := creator.Create(context.Background(), req) + assert.NoError(t, err) + assert.NotNil(t, resp) + + // VolumeId format: -- + expectedVolumeID := volIdentifier + "-" + symmetrixID + "-" + newDevID + assert.Equal(t, expectedVolumeID, resp.Volume.VolumeId) + + // VolumeContext required keys + ctx := resp.Volume.VolumeContext + assert.Equal(t, "Optimized", ctx[ServiceLevelParam]) + assert.Equal(t, "SRP_1", ctx[StoragePoolParam]) + assert.Equal(t, symmetrixID, ctx[SymmetrixIDParam]) + assert.Equal(t, sgName, ctx[StorageGroup]) + assert.Equal(t, snapName, ctx[ContentSource]) + assert.NotEmpty(t, ctx[CapacityGB]) + assert.NotEmpty(t, ctx["CreationTime"]) +} + +func TestBuildCreateVolumesRequestFromSnapshot(t *testing.T) { + req := buildCreateVolumeFromSnapshotRequest(547, "mysnap", "SRP_1", "Optimized", "csi-my-SG", "csi-my-vol", nil, "req-snap-1") + + assert.Equal(t, types.ExecutionOptionSynchronous, req.ExecutionOption) + assert.Len(t, req.Volumes, 1) + + vol := req.Volumes[0] + assert.Equal(t, "req-snap-1", vol.RequestID) + + // volume.identifier set for idempotency + assert.NotNil(t, vol.Volume) + assert.Equal(t, "csi-my-vol", vol.Volume.Identifier) + + // create_new_from_snapshot set with new_volume_attributes; create_new_from_attributes NOT set at top level + assert.NotNil(t, vol.CreateNew) + assert.NotNil(t, vol.CreateNew.CreateNewFromSnapshot) + assert.Equal(t, "mysnap", vol.CreateNew.CreateNewFromSnapshot.Snapshot.ID) + assert.NotNil(t, vol.CreateNew.CreateNewFromSnapshot.NewVolumeAttributes) + assert.Equal(t, "CYL", vol.CreateNew.CreateNewFromSnapshot.NewVolumeAttributes.CapacityUnit) + assert.Equal(t, float64(547), vol.CreateNew.CreateNewFromSnapshot.NewVolumeAttributes.VolumeSize) + assert.Nil(t, vol.CreateNew.CreateNewFromAttributes) + + // precheck_srp_capacity set + assert.NotNil(t, vol.CreateNew.PrecheckSrpCapacity) + assert.Equal(t, "SRP_1", vol.CreateNew.PrecheckSrpCapacity.SRP.ID) + + // manage_volume_storage_group set + assert.NotNil(t, vol.Actions) + assert.NotNil(t, vol.Actions.ManageVolumeStorageGroup) + assert.Equal(t, "Add", vol.Actions.ManageVolumeStorageGroup.Action) + assert.Equal(t, "csi-my-SG", vol.Actions.ManageVolumeStorageGroup.StorageGroup.ID) + + // manage_identifier NOT set (causes 500 when combined with volume.identifier) + assert.Nil(t, vol.Actions.ManageIdentifier) + + // response_select at per-volume level + assert.Equal(t, "id,identifier,cap_cyl,storage_groups", vol.ResponseSelect) +} + +// --------------------------------------------------------------------------- +// Gap 1 — AccessibleTopology tests +// --------------------------------------------------------------------------- + +func TestU4P104VolumeCreator_AccessibleTopologySet(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := mocks.NewMockPmaxClient(ctrl) + symmetrixID := "000197900049" + deviceID := "00TOP" + volumeIdentifier := "csi--topo-vol" + + mockClient.EXPECT().CreateVolume(gomock.Any(), symmetrixID, gomock.Any(), gomock.Any()). + Return(&types.CreateVolumesResponse{ + Summary: types.ResponseSummary{Total: 1, Succeeded: 1}, + Results: types.CreateVolumesResults{ + Result: []types.CreateVolumeResponseItem{ + {Status: "success", Volume: &types.VolumeRefResponse{ID: deviceID, Identifier: volumeIdentifier}}, + }, + }, + }, nil).Times(1) + + creator := &u4p104VolumeCreator{ + s: &service{}, + pmaxClient: mockClient, + pmaxClient104: mockClient, + symmetrixID: symmetrixID, + } + + req := &csi.CreateVolumeRequest{ + Name: "topo-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 1073741824}, + Parameters: map[string]string{StoragePoolParam: "SRP_1", ServiceLevelParam: "Optimized"}, + AccessibilityRequirements: &csi.TopologyRequirement{ + Preferred: []*csi.Topology{ + {Segments: map[string]string{"topology.kubernetes.io/zone": "zone-a"}}, + }, + }, + } + + resp, err := creator.Create(context.Background(), req) + assert.NoError(t, err) + assert.NotNil(t, resp) + assert.NotNil(t, resp.Volume.AccessibleTopology) + assert.Len(t, resp.Volume.AccessibleTopology, 1) + assert.Equal(t, "zone-a", resp.Volume.AccessibleTopology[0].Segments["topology.kubernetes.io/zone"]) +} + +func TestU4P104VolumeCreator_AccessibleTopologyNilWhenNoRequirements(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := mocks.NewMockPmaxClient(ctrl) + symmetrixID := "000197900049" + + mockClient.EXPECT().CreateVolume(gomock.Any(), symmetrixID, gomock.Any(), gomock.Any()). + Return(&types.CreateVolumesResponse{ + Summary: types.ResponseSummary{Total: 1, Succeeded: 1}, + Results: types.CreateVolumesResults{ + Result: []types.CreateVolumeResponseItem{ + {Status: "success", Volume: &types.VolumeRefResponse{ID: "00NTR", Identifier: "csi--no-topo"}}, + }, + }, + }, nil).Times(1) + + creator := &u4p104VolumeCreator{ + s: &service{}, + pmaxClient: mockClient, + pmaxClient104: mockClient, + symmetrixID: symmetrixID, + } + + req := &csi.CreateVolumeRequest{ + Name: "no-topo", + CapacityRange: &csi.CapacityRange{RequiredBytes: 1073741824}, + Parameters: map[string]string{StoragePoolParam: "SRP_1", ServiceLevelParam: "Optimized"}, + // No AccessibilityRequirements + } + + resp, err := creator.Create(context.Background(), req) + assert.NoError(t, err) + assert.NotNil(t, resp) + assert.Nil(t, resp.Volume.AccessibleTopology) +} + +func TestU4P104VolumeCreator_AccessibleTopologyOnIdempotentPath(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := mocks.NewMockPmaxClient(ctrl) + symmetrixID := "000197900049" + volumeIdentifier := "csi--idem-topo" + + mockClient.EXPECT().CreateVolume(gomock.Any(), symmetrixID, gomock.Any(), gomock.Any()). + Return(&types.CreateVolumesResponse{ + Summary: types.ResponseSummary{Total: 1, Succeeded: 1}, + Results: types.CreateVolumesResults{ + Result: []types.CreateVolumeResponseItem{ + {Status: "success", Volume: nil}, + }, + }, + }, nil).Times(1) + mockClient.EXPECT().GetVolumesByIdentifier(gomock.Any(), symmetrixID, volumeIdentifier). + Return(&types.Volumev1{ + Volumes: []types.VolumeEnhanced{ + {ID: "00IDE", Identifier: volumeIdentifier, CapCyl: 547}, + }, + }, nil).Times(1) + + creator := &u4p104VolumeCreator{ + s: &service{}, + pmaxClient: mockClient, + pmaxClient104: mockClient, + symmetrixID: symmetrixID, + } + + req := &csi.CreateVolumeRequest{ + Name: "idem-topo", + CapacityRange: &csi.CapacityRange{RequiredBytes: 1073741824}, + Parameters: map[string]string{StoragePoolParam: "SRP_1", ServiceLevelParam: "Optimized"}, + AccessibilityRequirements: &csi.TopologyRequirement{ + Preferred: []*csi.Topology{ + {Segments: map[string]string{"topology.kubernetes.io/zone": "zone-b"}}, + }, + }, + } + + resp, err := creator.Create(context.Background(), req) + assert.NoError(t, err) + assert.NotNil(t, resp) + assert.NotNil(t, resp.Volume.AccessibleTopology) + assert.Equal(t, "zone-b", resp.Volume.AccessibleTopology[0].Segments["topology.kubernetes.io/zone"]) +} + +// --------------------------------------------------------------------------- +// Gap 5 — SLO validation tests +// --------------------------------------------------------------------------- + +func TestU4P104VolumeCreator_InvalidSLORejected(t *testing.T) { + creator := &u4p104VolumeCreator{s: &service{}} + req := &csi.CreateVolumeRequest{ + Name: "bad-slo-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 1073741824}, + Parameters: map[string]string{StoragePoolParam: "SRP_1", ServiceLevelParam: "Invalid"}, + } + resp, err := creator.Create(context.Background(), req) + assert.Nil(t, resp) + assert.Error(t, err) + assert.Contains(t, err.Error(), "An invalid Service Level parameter was specified") +} + +func TestU4P104VolumeCreator_ValidSLOAccepted(t *testing.T) { + validSLOs := []string{"Diamond", "Platinum", "Gold", "Silver", "Bronze", "Optimized", "None"} + for _, slo := range validSLOs { + t.Run(slo, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockClient := mocks.NewMockPmaxClient(ctrl) + symmetrixID := "000197900049" + + mockClient.EXPECT().CreateVolume(gomock.Any(), symmetrixID, gomock.Any(), gomock.Any()). + Return(&types.CreateVolumesResponse{ + Summary: types.ResponseSummary{Total: 1, Succeeded: 1}, + Results: types.CreateVolumesResults{ + Result: []types.CreateVolumeResponseItem{ + {Status: "success", Volume: &types.VolumeRefResponse{ID: "00SLO", Identifier: "csi--slo-vol"}}, + }, + }, + }, nil).Times(1) + + creator := &u4p104VolumeCreator{ + s: &service{}, + pmaxClient: mockClient, + pmaxClient104: mockClient, + symmetrixID: symmetrixID, + } + + req := &csi.CreateVolumeRequest{ + Name: "slo-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 1073741824}, + Parameters: map[string]string{StoragePoolParam: "SRP_1", ServiceLevelParam: slo}, + } + + resp, err := creator.Create(context.Background(), req) + assert.NoError(t, err) + assert.NotNil(t, resp) + }) + } +} + +func TestIsValidSLO(t *testing.T) { + assert.True(t, isValidSLO("Diamond")) + assert.True(t, isValidSLO("Optimized")) + assert.True(t, isValidSLO("None")) + assert.False(t, isValidSLO("Invalid")) + assert.False(t, isValidSLO("diamond")) + assert.False(t, isValidSLO("")) +} + +// --------------------------------------------------------------------------- +// Gap 6 — Zone labels tests +// --------------------------------------------------------------------------- + +func TestU4P104VolumeCreator_ZoneLabelsAddedWhenInAZ(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := mocks.NewMockPmaxClient(ctrl) + symmetrixID := "000197900049" + + mockClient.EXPECT().CreateVolume(gomock.Any(), symmetrixID, gomock.Any(), gomock.Any()). + Return(&types.CreateVolumesResponse{ + Summary: types.ResponseSummary{Total: 1, Succeeded: 1}, + Results: types.CreateVolumesResults{ + Result: []types.CreateVolumeResponseItem{ + {Status: "success", Volume: &types.VolumeRefResponse{ID: "00ZL1", Identifier: "csi--zone-vol"}}, + }, + }, + }, nil).Times(1) + + svc := &service{} + svc.opts.StorageArrays = map[string]StorageArrayConfig{ + symmetrixID: {Labels: map[string]interface{}{"topology.kubernetes.io/zone": "us-east-1a"}}, + } + + creator := &u4p104VolumeCreator{ + s: svc, + pmaxClient: mockClient, + pmaxClient104: mockClient, + symmetrixID: symmetrixID, + symmIDFoundInAZ: true, + } + + req := &csi.CreateVolumeRequest{ + Name: "zone-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 1073741824}, + Parameters: map[string]string{StoragePoolParam: "SRP_1", ServiceLevelParam: "Optimized"}, + } + + resp, err := creator.Create(context.Background(), req) + assert.NoError(t, err) + assert.NotNil(t, resp) + assert.Equal(t, "us-east-1a", resp.Volume.VolumeContext["topology.kubernetes.io/zone"]) +} + +func TestU4P104VolumeCreator_NoZoneLabelsWhenNotInAZ(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := mocks.NewMockPmaxClient(ctrl) + symmetrixID := "000197900049" + + mockClient.EXPECT().CreateVolume(gomock.Any(), symmetrixID, gomock.Any(), gomock.Any()). + Return(&types.CreateVolumesResponse{ + Summary: types.ResponseSummary{Total: 1, Succeeded: 1}, + Results: types.CreateVolumesResults{ + Result: []types.CreateVolumeResponseItem{ + {Status: "success", Volume: &types.VolumeRefResponse{ID: "00NZL", Identifier: "csi--no-zone"}}, + }, + }, + }, nil).Times(1) + + creator := &u4p104VolumeCreator{ + s: &service{}, + pmaxClient: mockClient, + pmaxClient104: mockClient, + symmetrixID: symmetrixID, + symmIDFoundInAZ: false, + } + + req := &csi.CreateVolumeRequest{ + Name: "no-zone", + CapacityRange: &csi.CapacityRange{RequiredBytes: 1073741824}, + Parameters: map[string]string{StoragePoolParam: "SRP_1", ServiceLevelParam: "Optimized"}, + } + + resp, err := creator.Create(context.Background(), req) + assert.NoError(t, err) + assert.NotNil(t, resp) + _, hasZoneLabel := resp.Volume.VolumeContext["topology.kubernetes.io/zone"] + assert.False(t, hasZoneLabel) +} + +// --------------------------------------------------------------------------- +// Gap 7 — Block capability validation tests +// --------------------------------------------------------------------------- + +func TestU4P104VolumeCreator_BlockCapabilityRejectedWhenDisabled(t *testing.T) { + svc := &service{} + svc.opts.EnableBlock = false + + creator := &u4p104VolumeCreator{s: svc} + + block := new(csi.VolumeCapability_BlockVolume) + accessType := new(csi.VolumeCapability_Block) + accessType.Block = block + capability := &csi.VolumeCapability{ + AccessType: accessType, + AccessMode: &csi.VolumeCapability_AccessMode{Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER}, + } + + req := &csi.CreateVolumeRequest{ + Name: "block-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 1073741824}, + Parameters: map[string]string{StoragePoolParam: "SRP_1", ServiceLevelParam: "Optimized"}, + VolumeCapabilities: []*csi.VolumeCapability{capability}, + } + + resp, err := creator.Create(context.Background(), req) + assert.Nil(t, resp) + assert.Error(t, err) + assert.Contains(t, err.Error(), "Block Volume Capability is not supported") +} + +func TestU4P104VolumeCreator_BlockCapabilityAcceptedWhenEnabled(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := mocks.NewMockPmaxClient(ctrl) + symmetrixID := "000197900049" + + mockClient.EXPECT().CreateVolume(gomock.Any(), symmetrixID, gomock.Any(), gomock.Any()). + Return(&types.CreateVolumesResponse{ + Summary: types.ResponseSummary{Total: 1, Succeeded: 1}, + Results: types.CreateVolumesResults{ + Result: []types.CreateVolumeResponseItem{ + {Status: "success", Volume: &types.VolumeRefResponse{ID: "00BLK", Identifier: "csi--block-vol"}}, + }, + }, + }, nil).Times(1) + + svc := &service{} + svc.opts.EnableBlock = true + + creator := &u4p104VolumeCreator{ + s: svc, + pmaxClient: mockClient, + pmaxClient104: mockClient, + symmetrixID: symmetrixID, + } + + block := new(csi.VolumeCapability_BlockVolume) + accessType := new(csi.VolumeCapability_Block) + accessType.Block = block + capability := &csi.VolumeCapability{ + AccessType: accessType, + AccessMode: &csi.VolumeCapability_AccessMode{Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER}, + } + + req := &csi.CreateVolumeRequest{ + Name: "block-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 1073741824}, + Parameters: map[string]string{StoragePoolParam: "SRP_1", ServiceLevelParam: "Optimized"}, + VolumeCapabilities: []*csi.VolumeCapability{capability}, + } + + resp, err := creator.Create(context.Background(), req) + assert.NoError(t, err) + assert.NotNil(t, resp) +} + +func TestU4P104VolumeCreator_MountCapabilityAllowedWhenBlockDisabled(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := mocks.NewMockPmaxClient(ctrl) + symmetrixID := "000197900049" + + mockClient.EXPECT().CreateVolume(gomock.Any(), symmetrixID, gomock.Any(), gomock.Any()). + Return(&types.CreateVolumesResponse{ + Summary: types.ResponseSummary{Total: 1, Succeeded: 1}, + Results: types.CreateVolumesResults{ + Result: []types.CreateVolumeResponseItem{ + {Status: "success", Volume: &types.VolumeRefResponse{ID: "00MNT", Identifier: "csi--mount-vol"}}, + }, + }, + }, nil).Times(1) + + svc := &service{} + svc.opts.EnableBlock = false // block disabled, but mount should still work + + creator := &u4p104VolumeCreator{ + s: svc, + pmaxClient: mockClient, + pmaxClient104: mockClient, + symmetrixID: symmetrixID, + } + + capability := &csi.VolumeCapability{ + AccessType: &csi.VolumeCapability_Mount{Mount: &csi.VolumeCapability_MountVolume{FsType: "ext4"}}, + AccessMode: &csi.VolumeCapability_AccessMode{Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER}, + } + + req := &csi.CreateVolumeRequest{ + Name: "mount-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 1073741824}, + Parameters: map[string]string{StoragePoolParam: "SRP_1", ServiceLevelParam: "Optimized"}, + VolumeCapabilities: []*csi.VolumeCapability{capability}, + } + + resp, err := creator.Create(context.Background(), req) + assert.NoError(t, err) + assert.NotNil(t, resp) +} + +// --------------------------------------------------------------------------- +// classifyCreateVolumeError tests +// --------------------------------------------------------------------------- + +func TestClassifyCreateVolumeError_Nil(t *testing.T) { + assert.Nil(t, classifyCreateVolumeError(nil)) +} + +func TestClassifyCreateVolumeError_SizeMismatch(t *testing.T) { + err := fmt.Errorf("create volumes failed: 0x020e0105: defined volume size [2185]Cyls does not match existing volume size [4369]Cyls") + result := classifyCreateVolumeError(err) + assert.Error(t, result) + assert.Contains(t, result.Error(), "A volume with the same name exists but has a different size") + assert.Contains(t, result.Error(), "AlreadyExists") +} + +func TestClassifyCreateVolumeError_TargetSmallerThanSource(t *testing.T) { + err := fmt.Errorf("create volumes failed: 0x020e0105: Target volume size [2185]Cyls cannot be smaller than source volume size [4369]Cyls") + result := classifyCreateVolumeError(err) + assert.Error(t, result) + assert.Contains(t, result.Error(), "Requested capacity is smaller than the source") + assert.Contains(t, result.Error(), "InvalidArgument") +} + +func TestClassifyCreateVolumeError_SourceVolumeNotFound(t *testing.T) { + err := fmt.Errorf("create volumes failed: 0x020e0114: Source Volumewith identifier [csi-test-vol-1111] does not exist") + result := classifyCreateVolumeError(err) + assert.Error(t, result) + assert.Contains(t, result.Error(), "Volume content source couldn't be found in the array") + assert.Contains(t, result.Error(), "InvalidArgument") +} + +func TestClassifyCreateVolumeError_SnapshotNotFound(t *testing.T) { + err := fmt.Errorf("create volumes failed: 0x020e0117: No Snapshot found with id [101850091802]") + result := classifyCreateVolumeError(err) + assert.Error(t, result) + assert.Contains(t, result.Error(), "Snapshot not found on the array") + assert.Contains(t, result.Error(), "InvalidArgument") +} + +func TestClassifyCreateVolumeError_GenericError(t *testing.T) { + err := fmt.Errorf("some unknown API error") + result := classifyCreateVolumeError(err) + assert.Error(t, result) + assert.Contains(t, result.Error(), "Failed to create volume") + assert.Contains(t, result.Error(), "Internal") +} + +// --------------------------------------------------------------------------- +// 10.4 CreateVolume error scenario integration tests +// --------------------------------------------------------------------------- + +func TestU4P104VolumeCreator_IdempotentSizeMismatch_AlreadyExistsCode(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := mocks.NewMockPmaxClient(ctrl) + symmetrixID := "000197900049" + + // Simulate API returning size mismatch error for idempotent volume + mockClient.EXPECT().CreateVolume(gomock.Any(), symmetrixID, gomock.Any(), gomock.Any()). + Return(nil, fmt.Errorf("create volumes failed: 0x020e0105: defined volume size [547]Cyls does not match existing volume size [1093]Cyls")). + Times(1) + + creator := &u4p104VolumeCreator{ + s: &service{}, + pmaxClient: mockClient, + pmaxClient104: mockClient, + symmetrixID: symmetrixID, + params: map[string]string{StoragePoolParam: "SRP_1", ServiceLevelParam: "Optimized"}, + } + + req := &csi.CreateVolumeRequest{ + Name: "size-mismatch-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 1073741824}, + Parameters: map[string]string{StoragePoolParam: "SRP_1", ServiceLevelParam: "Optimized"}, + } + + resp, err := creator.Create(context.Background(), req) + assert.Nil(t, resp) + assert.Error(t, err) + assert.Contains(t, err.Error(), "A volume with the same name exists but has a different size") + assert.Contains(t, err.Error(), "AlreadyExists") +} + +func TestU4P104VolumeCreator_CloneSourceNotFound(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := mocks.NewMockPmaxClient(ctrl) + symmetrixID := "000197900049" + + // Simulate API returning source volume not found + mockClient.EXPECT().CreateVolume(gomock.Any(), symmetrixID, gomock.Any(), gomock.Any()). + Return(nil, fmt.Errorf("create volumes failed: 0x020e0114: Source Volumewith identifier [csi--clone-vol] does not exist")). + Times(1) + + deps := &snapshotDeps{service: &service{}, licenseErr: nil} + creator := &u4p104VolumeCreator{ + s: deps, + pmaxClient: mockClient, + pmaxClient104: mockClient, + symmetrixID: symmetrixID, + params: map[string]string{StoragePoolParam: "SRP_1", ServiceLevelParam: "Optimized"}, + } + + req := &csi.CreateVolumeRequest{ + Name: "clone-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 1073741824}, + Parameters: map[string]string{StoragePoolParam: "SRP_1", ServiceLevelParam: "Optimized"}, + VolumeContentSource: &csi.VolumeContentSource{ + Type: &csi.VolumeContentSource_Volume{ + Volume: &csi.VolumeContentSource_VolumeSource{ + VolumeId: "csi--src-vol-" + symmetrixID + "-00SRC", + }, + }, + }, + } + + resp, err := creator.Create(context.Background(), req) + assert.Nil(t, resp) + assert.Error(t, err) + assert.Contains(t, err.Error(), "Volume content source couldn't be found in the array") +} + +func TestU4P104VolumeCreator_CloneSmallerCapacity(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := mocks.NewMockPmaxClient(ctrl) + symmetrixID := "000197900049" + + // Simulate API returning target smaller than source error + mockClient.EXPECT().CreateVolume(gomock.Any(), symmetrixID, gomock.Any(), gomock.Any()). + Return(nil, fmt.Errorf("create volumes failed: 0x020e0105: Target volume size [547]Cyls cannot be smaller than source volume size [1093]Cyls")). + Times(1) + + deps := &snapshotDeps{service: &service{}, licenseErr: nil} + creator := &u4p104VolumeCreator{ + s: deps, + pmaxClient: mockClient, + pmaxClient104: mockClient, + symmetrixID: symmetrixID, + params: map[string]string{StoragePoolParam: "SRP_1", ServiceLevelParam: "Optimized"}, + } + + req := &csi.CreateVolumeRequest{ + Name: "clone-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 1073741824}, + Parameters: map[string]string{StoragePoolParam: "SRP_1", ServiceLevelParam: "Optimized"}, + VolumeContentSource: &csi.VolumeContentSource{ + Type: &csi.VolumeContentSource_Volume{ + Volume: &csi.VolumeContentSource_VolumeSource{ + VolumeId: "csi--src-vol-" + symmetrixID + "-00SRC", + }, + }, + }, + } + + resp, err := creator.Create(context.Background(), req) + assert.Nil(t, resp) + assert.Error(t, err) + assert.Contains(t, err.Error(), "Requested capacity is smaller than the source") +} + +func TestU4P104VolumeCreator_SnapshotRestoreSmallerCapacity(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := mocks.NewMockPmaxClient(ctrl) + symmetrixID := "000197900049" + srcDevID := "00SRC" + snapName := "mysnap" + + // GetSnapshotInfo succeeds + mockClient.EXPECT().GetSnapshotInfo(gomock.Any(), symmetrixID, srcDevID, snapName). + Return(mockSnapshotInfo(snapName), nil).Times(1) + + // Simulate API returning target smaller than source error + mockClient.EXPECT().CreateVolume(gomock.Any(), symmetrixID, gomock.Any(), gomock.Any()). + Return(nil, fmt.Errorf("create volumes failed: 0x020e0105: Target volume size [547]Cyls cannot be smaller than source volume size [1093]Cyls")). + Times(1) + + deps := &snapshotDeps{service: &service{}, licenseErr: nil} + creator := &u4p104VolumeCreator{ + s: deps, + pmaxClient: mockClient, + pmaxClient104: mockClient, + symmetrixID: symmetrixID, + params: map[string]string{StoragePoolParam: "SRP_1", ServiceLevelParam: "Optimized"}, + } + + snapCSIID := csiSnapshotID(snapName, symmetrixID, srcDevID) + req := &csi.CreateVolumeRequest{ + Name: "snap-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 1073741824}, + Parameters: map[string]string{StoragePoolParam: "SRP_1", ServiceLevelParam: "Optimized"}, + VolumeContentSource: &csi.VolumeContentSource{ + Type: &csi.VolumeContentSource_Snapshot{ + Snapshot: &csi.VolumeContentSource_SnapshotSource{SnapshotId: snapCSIID}, + }, + }, + } + + resp, err := creator.Create(context.Background(), req) + assert.Nil(t, resp) + assert.Error(t, err) + assert.Contains(t, err.Error(), "Requested capacity is smaller than the source") +} diff --git a/service/volume_publisher.go b/service/volume_publisher.go new file mode 100644 index 00000000..200190c1 --- /dev/null +++ b/service/volume_publisher.go @@ -0,0 +1,394 @@ +package service + +import ( + "context" + "strings" + + "github.com/dell/csi-powermax/v2/pkg/file" + + pmax "github.com/dell/gopowermax/v2" + types "github.com/dell/gopowermax/v2/types/v100" + "github.com/container-storage-interface/spec/lib/go/csi" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// --------------------------------------------------------------------------- +// Struct definitions +// --------------------------------------------------------------------------- + +type u4p104VolumePublisher struct { + s *service + legacyClient pmax.Pmax // standard client for lookups (GetVolumeByID, MV connections, etc.) + client104 pmax.Pmax // 10.4-capable client for PublishMaskingViews + symID string + devID string + volID string + volumeName string + remoteSymID string + remoteVolumeID string + reqID string +} + +type legacyVolumePublisher struct { + s *service + pmaxClient pmax.Pmax + symID string + devID string + volID string + volumeName string + remoteSymID string + remoteVolumeID string + reqID string +} + +// --------------------------------------------------------------------------- +// 10.4 volume publishing +// --------------------------------------------------------------------------- + +// buildPublishMaskingViewsParam constructs the request payload for the 10.4 +// PublishMaskingViews API call, binding a volume to a masking view, storage +// group, host, and port group. +func buildPublishMaskingViewsParam(tgtMaskingViewID, tgtStorageGroupID, devID, hostID, portGroupID string) *types.PublishMaskingViewsParam { + return &types.PublishMaskingViewsParam{ + MaskingViews: []types.MaskingViewPublishParam{ + { + ID: tgtMaskingViewID, + StorageGroup: &types.StorageGroupPublishParam{ + ID: tgtStorageGroupID, + Actions: &types.StorageGroupPublishActions{ + AddVolumesToStorageGroupAction: &types.AddVolumesToStorageGroupAction{ + Volumes: []types.VolumePublishParam{ + { + ExistingVolumes: []types.ExistingVolumeParam{ + {ID: devID}, + }, + }, + }, + }, + }, + }, + Host: &types.HostPublishParam{ + ID: hostID, + }, + PortGroup: &types.PortGroupPublishParam{ + ID: portGroupID, + }, + }, + }, + } +} + +// 10.4 publish — uses the single PublishMaskingViews API to atomically ensure +// the volume is in the correct storage group, the masking view exists with the +// right host and port group, and is connected. This replaces the legacy multi-step +// sequence of GetStorageGroup/CreateStorageGroup + AddVolumesToStorageGroup + +// GetHost + SelectOrCreatePortGroup + CreateMaskingView. +func (p *u4p104VolumePublisher) Publish(ctx context.Context, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) { + log := log.WithContext(ctx) + + nodeID := req.GetNodeId() + if nodeID == "" { + log.Error("node ID is required") + return nil, status.Error(codes.InvalidArgument, "node ID is required") + } + + vc := req.GetVolumeCapability() + if vc == nil { + log.Error("volume capability is required") + return nil, status.Error(codes.InvalidArgument, "volume capability is required") + } + am := vc.GetAccessMode() + if am == nil { + log.Error("access mode is required") + return nil, status.Error(codes.InvalidArgument, "access mode is required") + } + if am.Mode == csi.VolumeCapability_AccessMode_UNKNOWN { + log.Error(errUnknownAccessMode) + return nil, status.Error(codes.InvalidArgument, errUnknownAccessMode) + } + + // Fetch the volume details from array (need EffectiveWWN for publish context) + symID, devID, vol, err := p.s.GetVolumeByID(ctx, p.volID, p.legacyClient) + if err != nil { + log.Error("GetVolumeByID Error: " + err.Error()) + return nil, err + } + + fields := map[string]interface{}{ + "SymmetrixID": symID, + "VolumeId": p.volID, + "NodeId": nodeID, + "AccessMode": am.Mode, + "CSIRequestID": p.reqID, + } + log.WithFields(fields).Info("Executing 10.4 ControllerPublishVolume with following fields") + + // Determine host type (NVMe, iSCSI, FC) — check node cache first + isNVMETCP := false + isISCSI := false + nodeInCache := false + cacheID := symID + ":" + nodeID + if tempHostID, ok := nodeCache.Load(cacheID); ok { + log.Debugf("10.4 Loaded nodeID: %s, hostID: %s from node cache", nodeID, tempHostID.(string)) + nodeInCache = true + if !strings.Contains(tempHostID.(string), "-FC") { + isISCSI = true + } + if strings.Contains(tempHostID.(string), "-NVMETCP") { + isISCSI = false + isNVMETCP = true + } + } else { + isNVMETCP, err = p.s.IsNodeNVMe(ctx, symID, nodeID, p.legacyClient) + if err != nil { + return nil, status.Error(codes.NotFound, err.Error()) + } + if !isNVMETCP { + isISCSI, err = p.s.IsNodeISCSI(ctx, symID, nodeID, p.legacyClient) + if err != nil { + return nil, status.Error(codes.NotFound, err.Error()) + } + } + } + + hostID, tgtStorageGroupID, tgtMaskingViewID := p.s.GetNVMETCPHostSGAndMVIDFromNodeID(nodeID) + if !isNVMETCP { + hostID, tgtStorageGroupID, tgtMaskingViewID = p.s.GetHostSGAndMVIDFromNodeID(nodeID, isISCSI) + } + + if !nodeInCache { + val, loaded := nodeCache.LoadOrStore(cacheID, hostID) + if !loaded { + log.Debugf("10.4 Added nodeID: %s, hostID: %s to node cache", nodeID, hostID) + } else { + log.Debugf("10.4 Another goroutine added hostID: %s for node: %s to node cache", val.(string), nodeID) + if hostID != val.(string) { + log.Warnf("10.4 Mismatch between calculated hostID: %s and cached hostID: %s from node cache", hostID, val.(string)) + } + } + } + + // Get host details for port group selection + host, err := p.legacyClient.GetHostByID(ctx, symID, hostID) + if err != nil { + log.Errorf("10.4 ControllerPublishVolume: Failed to fetch host details for %s on %s: %s", hostID, symID, err.Error()) + return nil, status.Errorf(codes.NotFound, "Failed to fetch host details for %s on %s: %s", hostID, symID, err.Error()) + } + + // Select or create the port group + portGroupID, err := p.s.SelectOrCreatePortGroup(ctx, symID, host, p.legacyClient) + if err != nil { + log.Errorf("10.4 ControllerPublishVolume: Failed to select/create port group for host %s on %s: %s", hostID, symID, err.Error()) + return nil, status.Errorf(codes.Internal, "Failed to select/create port group for host %s on %s: %s", hostID, symID, err.Error()) + } + + publishContext := make(map[string]string) + if len(vol.EffectiveWWN) > 0 { + publishContext[PublishContextDeviceWWN] = vol.EffectiveWWN + } else { + return nil, status.Errorf(codes.Internal, + "PublishVolume: Volume %s has no effective WWN, Unisphere may not be synchronized with array or synchronization may be in progress", p.volID) + } + + publishParam := buildPublishMaskingViewsParam(tgtMaskingViewID, tgtStorageGroupID, devID, hostID, portGroupID) + + log.Infof("10.4 PublishMaskingViews request for volume %s on array %s, MV: %s, SG: %s, Host: %s, PG: %s", + devID, symID, tgtMaskingViewID, tgtStorageGroupID, hostID, portGroupID) + + publishResp, err := p.client104.PublishMaskingViews(ctx, symID, publishParam) + if err != nil { + log.Errorf("10.4 PublishMaskingViews failed for volume %s on array %s: %v", devID, symID, err) + return nil, status.Errorf(codes.Internal, "10.4 PublishMaskingViews failed: %s", err.Error()) + } + + if publishResp != nil && publishResp.Summary.Failed > 0 { + log.Errorf("10.4 PublishMaskingViews reported failures for volume %s on array %s: %+v", devID, symID, publishResp.Results) + return nil, status.Errorf(codes.Internal, "10.4 PublishMaskingViews failed for masking view %s", tgtMaskingViewID) + } + + // Fetch MV connections for the device after successful publish + lockNum := RequestLock(getMVLockKey(symID, tgtMaskingViewID), p.reqID) + connections, connErr := p.legacyClient.GetMaskingViewConnections(ctx, symID, tgtMaskingViewID, devID) + ReleaseLock(getMVLockKey(symID, tgtMaskingViewID), p.reqID, lockNum) + if connErr != nil { + log.Warnf("10.4 ControllerPublishVolume: initial connection fetch failed for %s on %s: %v, will retry in updatePublishContext", devID, symID, connErr) + connections = nil + } else { + log.Infof("10.4 ControllerPublishVolume: fetched %d connections for volume %s on MV %s", len(connections), devID, tgtMaskingViewID) + } + + // Build publish context from MV connections (LUN address and port identifiers) + return p.s.updatePublishContext(ctx, publishContext, symID, tgtMaskingViewID, devID, p.reqID, connections, p.legacyClient, true) +} + +// --------------------------------------------------------------------------- +// Legacy volume publishing (pre-10.4) +// --------------------------------------------------------------------------- + +func (p *legacyVolumePublisher) Publish(ctx context.Context, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) { + log := log.WithContext(ctx) + + reqID := p.reqID + symID := p.symID + devID := p.devID + volID := p.volID + volumeName := p.volumeName + remoteSymID := p.remoteSymID + remoteVolumeID := p.remoteVolumeID + pmaxClient := p.pmaxClient + + volumeContext := req.GetVolumeContext() + if volumeContext != nil { + log.Infof("VolumeContext:") + for key, value := range volumeContext { + log.Infof(" [%s]=%s", key, value) + } + } + + nodeID := req.GetNodeId() + if nodeID == "" { + log.Error("node ID is required") + return nil, status.Error(codes.InvalidArgument, + "node ID is required") + } + + vc := req.GetVolumeCapability() + if vc == nil { + log.Error("volume capability is required") + return nil, status.Error(codes.InvalidArgument, + "volume capability is required") + } + am := vc.GetAccessMode() + if am == nil { + log.Error("access mode is required") + return nil, status.Error(codes.InvalidArgument, + "access mode is required") + } + + if am.Mode == csi.VolumeCapability_AccessMode_UNKNOWN { + log.Error(errUnknownAccessMode) + return nil, status.Error(codes.InvalidArgument, errUnknownAccessMode) + } + isNFS := accTypeIsNFS([]*csi.VolumeCapability{vc}) + if isNFS { + // incoming request for file system volume + return file.CreateNFSExport(ctx, reqID, symID, devID, am, volumeContext, pmaxClient) + } + + if vc.GetMount().GetFsType() == "" { + // can happen when doing static provisioning, check for filesystem existence + log.Debug("fsType empty...checking for file system existence") + _, err := pmaxClient.GetFileSystemByID(ctx, symID, devID) + if err == nil { + // we found fs, proceed to CreateNFSExport + return nil, status.Errorf(codes.Unavailable, "static provisioning on a file system is not supported.") + } + } + + // Fetch the volume details from array + symID, devID, vol, err := p.s.GetVolumeByID(ctx, volID, pmaxClient) + if err != nil { + log.Error("GetVolumeByID Error: " + err.Error()) + return nil, err + } + + // log all parameters used in ControllerPublishVolume call + fields := map[string]interface{}{ + "SymmetrixID": symID, + "VolumeId": volID, + "NodeId": nodeID, + "AccessMode": am.Mode, + "CSIRequestID": reqID, + "IsVsphereVolume": p.s.opts.IsVsphereEnabled, + } + log.WithFields(fields).Info("Executing ControllerPublishVolume with following fields") + // flag createNFSExport() + isNVMETCP := false + isISCSI := false + // Check if node ID is present in cache + nodeInCache := false + cacheID := symID + ":" + nodeID + tempHostID, ok := nodeCache.Load(cacheID) + if ok { + log.Debugf("REQ ID: %s Loaded nodeID: %s, hostID: %s from node cache", + reqID, nodeID, tempHostID.(string)) + nodeInCache = true + if !strings.Contains(tempHostID.(string), "-FC") { + isISCSI = true + } + if strings.Contains(tempHostID.(string), "-NVMETCP") { + isISCSI = false + isNVMETCP = true + } + } else { + log.Debugf("REQ ID: %s nodeID: %s not present in node cache", reqID, nodeID) + isNVMETCP, err = p.s.IsNodeNVMe(ctx, symID, nodeID, pmaxClient) + if err != nil { + return nil, status.Error(codes.NotFound, err.Error()) + } + if !isNVMETCP { + isISCSI, err = p.s.IsNodeISCSI(ctx, symID, nodeID, pmaxClient) + if err != nil { + return nil, status.Error(codes.NotFound, err.Error()) + } + } + } + + hostID, tgtStorageGroupID, tgtMaskingViewID := p.s.GetNVMETCPHostSGAndMVIDFromNodeID(nodeID) + + if !isNVMETCP { + // Update the values, if NVME is false + hostID, tgtStorageGroupID, tgtMaskingViewID = p.s.GetHostSGAndMVIDFromNodeID(nodeID, isISCSI) + } + + if !nodeInCache { + // Update the map + val, ok := nodeCache.LoadOrStore(cacheID, hostID) + if !ok { + log.Debugf("REQ ID: %s Added nodeID: %s, hostID: %s to node cache", reqID, nodeID, hostID) + } else { + log.Debugf("REQ ID: %s Some other goroutine added hostID: %s for node: %s to node cache", + reqID, val.(string), nodeID) + if hostID != val.(string) { + log.Warnf("REQ ID: %s Mismatch between calculated value: %s and latest value: %s from node cache", + reqID, val.(string), hostID) + } + } + } + + publishContext := make(map[string]string) + if len(vol.EffectiveWWN) > 0 { + publishContext[PublishContextDeviceWWN] = vol.EffectiveWWN + } else { + return nil, status.Errorf(codes.Internal, "PublishVolume: Volume %s has no effective WWN, Unisphere may not be synchronized with array or synchronization may be in progress", volID) + } + + ctrlPubRes, ctrlPubErr := p.s.publishVolume(ctx, publishContext, tgtStorageGroupID, hostID, symID, symID, tgtMaskingViewID, devID, reqID, volumeName, am, pmaxClient, true) + if ctrlPubErr != nil { + return nil, ctrlPubErr + } + + if remoteSymID != "" && remoteVolumeID != "" { + remoteVol, err := pmaxClient.GetVolumeByID(ctx, remoteSymID, remoteVolumeID) + if strings.Compare(remoteVol.EffectiveWWN, vol.EffectiveWWN) != 0 { + // Refresh the symmetrix + err := pmaxClient.RefreshSymmetrix(ctx, symID) + if err != nil { + if !strings.Contains(err.Error(), "Too Many Requests") { + return nil, status.Errorf(codes.Internal, "PublishVolume: Could not refresh symmetrix: (%s)", err.Error()) + } + return nil, status.Errorf(codes.Internal, "symmetrix sync in progress, waiting for cache to update") + } + // wait till the remote volume has an effective wwn + return nil, status.Errorf(codes.Internal, "PublishVolume: Could not publish remote volume: (%s)", "remote volume does not have effective wwn, waiting for it to SYNC") + } + log.Debugf("remote-vol: %#v, error: %#v", remoteVol, err) + if err != nil { + return nil, status.Errorf(codes.Internal, "PublishVolume: Could not retrieve remote volume: (%s)", err.Error()) + } + publishContext[RemotePublishContextDeviceWWN] = remoteVol.EffectiveWWN + return p.s.publishVolume(ctx, publishContext, tgtStorageGroupID, hostID, symID, remoteSymID, tgtMaskingViewID, remoteVolumeID, reqID, volumeName, am, pmaxClient, false) + } + return ctrlPubRes, ctrlPubErr +} diff --git a/service/volume_publisher_test.go b/service/volume_publisher_test.go new file mode 100644 index 00000000..490124c5 --- /dev/null +++ b/service/volume_publisher_test.go @@ -0,0 +1,683 @@ +package service + +import ( + "context" + "errors" + "testing" + + "github.com/dell/csi-powermax/v2/pkg/symmetrix/mocks" + types "github.com/dell/gopowermax/v2/types/v100" + "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" +) + +func TestU4P104VolumePublisher_MissingNodeID(t *testing.T) { + p := &u4p104VolumePublisher{s: &service{}} + resp, err := p.Publish(context.Background(), &csi.ControllerPublishVolumeRequest{}) + assert.Nil(t, resp) + assert.Error(t, err) + assert.Contains(t, err.Error(), "node ID is required") +} + +func TestU4P104VolumePublisher_MissingVolumeCapability(t *testing.T) { + p := &u4p104VolumePublisher{s: &service{}} + resp, err := p.Publish(context.Background(), &csi.ControllerPublishVolumeRequest{ + NodeId: "worker-1", + }) + assert.Nil(t, resp) + assert.Error(t, err) + assert.Contains(t, err.Error(), "volume capability is required") +} + +func TestU4P104VolumePublisher_MissingAccessMode(t *testing.T) { + p := &u4p104VolumePublisher{s: &service{}} + resp, err := p.Publish(context.Background(), &csi.ControllerPublishVolumeRequest{ + NodeId: "worker-1", + VolumeCapability: &csi.VolumeCapability{ + AccessType: &csi.VolumeCapability_Mount{ + Mount: &csi.VolumeCapability_MountVolume{FsType: "ext4"}, + }, + }, + }) + assert.Nil(t, resp) + assert.Error(t, err) + assert.Contains(t, err.Error(), "access mode is required") +} + +func TestU4P104VolumePublisher_UnknownAccessMode(t *testing.T) { + p := &u4p104VolumePublisher{s: &service{}} + resp, err := p.Publish(context.Background(), &csi.ControllerPublishVolumeRequest{ + NodeId: "worker-1", + VolumeCapability: &csi.VolumeCapability{ + AccessType: &csi.VolumeCapability_Mount{ + Mount: &csi.VolumeCapability_MountVolume{FsType: "ext4"}, + }, + AccessMode: &csi.VolumeCapability_AccessMode{ + Mode: csi.VolumeCapability_AccessMode_UNKNOWN, + }, + }, + }) + assert.Nil(t, resp) + assert.Error(t, err) + assert.Contains(t, err.Error(), errUnknownAccessMode) +} + +func TestU4P104VolumePublisher_GetVolumeByIDError(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockClient := mocks.NewMockPmaxClient(ctrl) + // fsType is ext4 so no GetFileSystemByID call + // GetVolumeByID fails + mockClient.EXPECT().GetVolumeByID(gomock.Any(), "000120000001", "011AB"). + Return(nil, errors.New("Could not find device 011AB")).Times(1) + p := &u4p104VolumePublisher{ + s: &service{}, + legacyClient: mockClient, + symID: "000120000001", + devID: "011AB", + volID: "csi-ABC-pmax-testvol-000120000001-011AB", + volumeName: "csi-ABC-pmax-testvol", + } + resp, err := p.Publish(context.Background(), &csi.ControllerPublishVolumeRequest{ + NodeId: "worker-1", + VolumeCapability: &csi.VolumeCapability{ + AccessType: &csi.VolumeCapability_Mount{ + Mount: &csi.VolumeCapability_MountVolume{FsType: "ext4"}, + }, + AccessMode: &csi.VolumeCapability_AccessMode{ + Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, + }, + }, + }) + assert.Nil(t, resp) + assert.Error(t, err) + assert.Contains(t, err.Error(), "Could not find") +} + +func TestU4P104VolumePublisher_IsNodeNVMeError(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockClient := mocks.NewMockPmaxClient(ctrl) + + volID := "csi-ABC-pmax-testvol-000120000001-011AB" + // Clear stale node cache entries from prior tests + nodeCache.Delete("000120000001:worker-1") + + mockClient.EXPECT().GetVolumeByID(gomock.Any(), "000120000001", "011AB"). + Return(&types.Volume{ + VolumeID: "011AB", + VolumeIdentifier: "csi-ABC-pmax-testvol", + EffectiveWWN: "60000970000120000001533030314142", + }, nil).Times(1) + + // IsNodeNVMe calls - NVMe MV and host lookups both fail + mockClient.EXPECT().GetMaskingViewByID(gomock.Any(), "000120000001", gomock.Any()). + Return(nil, errors.New("not found")).AnyTimes() + mockClient.EXPECT().GetHostByID(gomock.Any(), "000120000001", gomock.Any()). + Return(nil, errors.New("not found")).AnyTimes() + + // Set NVMe transport so IsNodeNVMe actually returns an error when host is not found + svc := &service{ + opts: Opts{ + TransportProtocol: NvmeTCPTransportProtocol, + }, + } + p := &u4p104VolumePublisher{ + s: svc, + legacyClient: mockClient, + symID: "000120000001", + devID: "011AB", + volID: volID, + volumeName: "csi-ABC-pmax-testvol", + } + resp, err := p.Publish(context.Background(), &csi.ControllerPublishVolumeRequest{ + NodeId: "worker-1", + VolumeCapability: &csi.VolumeCapability{ + AccessType: &csi.VolumeCapability_Mount{ + Mount: &csi.VolumeCapability_MountVolume{FsType: "ext4"}, + }, + AccessMode: &csi.VolumeCapability_AccessMode{ + Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, + }, + }, + }) + assert.Nil(t, resp) + assert.Error(t, err) + assert.Contains(t, err.Error(), "Failed to fetch host id from array") +} + +func TestU4P104VolumePublisher_NoEffectiveWWN(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockClient := mocks.NewMockPmaxClient(ctrl) + + volID := "csi-ABC-pmax-testvol-000120000001-011AB" + // Clear stale node cache entries from prior tests + nodeCache.Delete("000120000001:worker-1") + + mockClient.EXPECT().GetVolumeByID(gomock.Any(), "000120000001", "011AB"). + Return(&types.Volume{ + VolumeID: "011AB", + VolumeIdentifier: "csi-ABC-pmax-testvol", + EffectiveWWN: "", // empty WWN + }, nil).Times(1) + + // IsNodeNVMe → false (default transport protocol is not NVMe) + mockClient.EXPECT().GetMaskingViewByID(gomock.Any(), "000120000001", gomock.Any()). + Return(nil, errors.New("not found")).AnyTimes() + // IsNodeISCSI → iSCSI host found + mockClient.EXPECT().GetHostByID(gomock.Any(), "000120000001", gomock.Any()). + Return(&types.Host{HostID: "csi-node--worker-1", HostType: "iSCSI"}, nil).AnyTimes() + + svc := &service{ + opts: Opts{ + PortGroups: []string{"csi-pg-1"}, + }, + } + p := &u4p104VolumePublisher{ + s: svc, + legacyClient: mockClient, + symID: "000120000001", + devID: "011AB", + volID: volID, + volumeName: "csi-ABC-pmax-testvol", + } + resp, err := p.Publish(context.Background(), &csi.ControllerPublishVolumeRequest{ + NodeId: "worker-1", + VolumeCapability: &csi.VolumeCapability{ + AccessType: &csi.VolumeCapability_Mount{ + Mount: &csi.VolumeCapability_MountVolume{FsType: "ext4"}, + }, + AccessMode: &csi.VolumeCapability_AccessMode{ + Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, + }, + }, + }) + assert.Nil(t, resp) + assert.Error(t, err) + assert.Contains(t, err.Error(), "has no effective WWN") +} + +func TestU4P104VolumePublisher_GetHostByIDError(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockClient := mocks.NewMockPmaxClient(ctrl) + + volID := "csi-ABC-pmax-testvol-000120000001-011AB" + symID := "000120000001" + // Clear stale node cache entries from prior tests + nodeCache.Delete(symID + ":worker-1") + + mockClient.EXPECT().GetVolumeByID(gomock.Any(), symID, "011AB"). + Return(&types.Volume{ + VolumeID: "011AB", + VolumeIdentifier: "csi-ABC-pmax-testvol", + EffectiveWWN: "60000970000120000001533030314142", + }, nil).Times(1) + + // IsNodeNVMe → false (default transport) + mockClient.EXPECT().GetMaskingViewByID(gomock.Any(), symID, gomock.Any()). + Return(nil, errors.New("not found")).AnyTimes() + // IsNodeISCSI → iSCSI host found (default transport checks FC host, then iSCSI host) + // All GetHostByID calls during IsNodeISCSI succeed with iSCSI host + // but the final GetHostByID for the derived hostID (for PG selection) fails + hostCall := mockClient.EXPECT().GetHostByID(gomock.Any(), symID, gomock.Any()). + Return(&types.Host{HostID: "csi-node--worker-1", HostType: "iSCSI"}, nil).AnyTimes() + // Override: the specific call for derived hostID during host lookup fails + // We use a counter to simulate: first N calls succeed (IsNodeISCSI), last call fails + callCount := 0 + hostCall.DoAndReturn(func(_ context.Context, _ string, hostID string) (*types.Host, error) { + callCount++ + // The last call is the explicit GetHostByID for PG selection + // IsNodeISCSI with default FC transport checks: fcHost, iscsiHost = 2 calls + // Then the Publish method makes 1 more call for PG selection + if callCount > 2 { + return nil, errors.New("host not found on array") + } + return &types.Host{HostID: hostID, HostType: "iSCSI"}, nil + }) + + svc := &service{ + opts: Opts{ + PortGroups: []string{"csi-pg-1"}, + }, + } + p := &u4p104VolumePublisher{ + s: svc, + legacyClient: mockClient, + symID: symID, + devID: "011AB", + volID: volID, + volumeName: "csi-ABC-pmax-testvol", + } + resp, err := p.Publish(context.Background(), &csi.ControllerPublishVolumeRequest{ + NodeId: "worker-1", + VolumeCapability: &csi.VolumeCapability{ + AccessType: &csi.VolumeCapability_Mount{ + Mount: &csi.VolumeCapability_MountVolume{FsType: "ext4"}, + }, + AccessMode: &csi.VolumeCapability_AccessMode{ + Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, + }, + }, + }) + assert.Nil(t, resp) + assert.Error(t, err) + assert.Contains(t, err.Error(), "Failed to fetch host details") +} + +func TestU4P104VolumePublisher_SelectOrCreatePortGroupError(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockClient := mocks.NewMockPmaxClient(ctrl) + + volID := "csi-ABC-pmax-testvol-000120000001-011AB" + symID := "000120000001" + // Clear stale node cache entries from prior tests + nodeCache.Delete("000120000001:worker-1") + + mockClient.EXPECT().GetVolumeByID(gomock.Any(), symID, "011AB"). + Return(&types.Volume{ + VolumeID: "011AB", + VolumeIdentifier: "csi-ABC-pmax-testvol", + EffectiveWWN: "60000970000120000001533030314142", + }, nil).Times(1) + + // IsNodeNVMe → false + mockClient.EXPECT().GetMaskingViewByID(gomock.Any(), symID, gomock.Any()). + Return(nil, errors.New("not found")).AnyTimes() + // IsNodeISCSI → iSCSI host found + mockClient.EXPECT().GetHostByID(gomock.Any(), symID, gomock.Any()). + Return(&types.Host{HostID: "csi-node--worker-1", HostType: "iSCSI"}, nil).AnyTimes() + + // No port groups configured → SelectPortGroup will fail + svc := &service{ + opts: Opts{ + PortGroups: []string{}, + }, + } + p := &u4p104VolumePublisher{ + s: svc, + legacyClient: mockClient, + symID: symID, + devID: "011AB", + volID: volID, + volumeName: "csi-ABC-pmax-testvol", + } + resp, err := p.Publish(context.Background(), &csi.ControllerPublishVolumeRequest{ + NodeId: "worker-1", + VolumeCapability: &csi.VolumeCapability{ + AccessType: &csi.VolumeCapability_Mount{ + Mount: &csi.VolumeCapability_MountVolume{FsType: "ext4"}, + }, + AccessMode: &csi.VolumeCapability_AccessMode{ + Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, + }, + }, + }) + assert.Nil(t, resp) + assert.Error(t, err) + assert.Contains(t, err.Error(), "Failed to select/create port group") +} + +func TestU4P104VolumePublisher_PublishMaskingViewsAPIError(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := mocks.NewMockPmaxClient(ctrl) + mockClient104 := mocks.NewMockPmaxClient(ctrl) + + volID := "csi-ABC-pmax-testvol-000120000001-011AB" + // Clear stale node cache entries from prior tests + nodeCache.Delete("000120000001:worker-1") + + mockClient.EXPECT().GetVolumeByID(gomock.Any(), "000120000001", "011AB"). + Return(&types.Volume{ + VolumeID: "011AB", + VolumeIdentifier: "csi-ABC-pmax-testvol", + EffectiveWWN: "60000970000120000001533030314142", + }, nil).Times(1) + + // IsNodeNVMe → false, no error + mockClient.EXPECT().GetMaskingViewByID(gomock.Any(), "000120000001", gomock.Any()). + Return(nil, errors.New("not found")).AnyTimes() + // IsNodeISCSI → iSCSI host found + mockClient.EXPECT().GetHostByID(gomock.Any(), "000120000001", gomock.Any()). + Return(&types.Host{HostID: "csi-node--worker-1", HostType: "iSCSI"}, nil).AnyTimes() + + // SelectOrCreatePortGroup for iSCSI uses SelectPortGroup + svc := &service{ + opts: Opts{ + PortGroups: []string{"csi-pg-1"}, + }, + } + + // PublishMaskingViews API call fails + mockClient104.EXPECT().PublishMaskingViews(gomock.Any(), "000120000001", gomock.Any()). + Return(nil, errors.New("array connection timeout")).Times(1) + + p := &u4p104VolumePublisher{ + s: svc, + legacyClient: mockClient, + client104: mockClient104, + symID: "000120000001", + devID: "011AB", + volID: volID, + volumeName: "csi-ABC-pmax-testvol", + } + resp, err := p.Publish(context.Background(), &csi.ControllerPublishVolumeRequest{ + NodeId: "worker-1", + VolumeCapability: &csi.VolumeCapability{ + AccessType: &csi.VolumeCapability_Mount{ + Mount: &csi.VolumeCapability_MountVolume{FsType: "ext4"}, + }, + AccessMode: &csi.VolumeCapability_AccessMode{ + Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, + }, + }, + }) + assert.Nil(t, resp) + assert.Error(t, err) + assert.Contains(t, err.Error(), "PublishMaskingViews failed") +} + +func TestU4P104VolumePublisher_PublishMaskingViewsPartialFailure(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := mocks.NewMockPmaxClient(ctrl) + mockClient104 := mocks.NewMockPmaxClient(ctrl) + + volID := "csi-ABC-pmax-testvol-000120000001-011AB" + // Clear stale node cache entries from prior tests + nodeCache.Delete("000120000001:worker-1") + + mockClient.EXPECT().GetVolumeByID(gomock.Any(), "000120000001", "011AB"). + Return(&types.Volume{ + VolumeID: "011AB", + VolumeIdentifier: "csi-ABC-pmax-testvol", + EffectiveWWN: "60000970000120000001533030314142", + }, nil).Times(1) + + mockClient.EXPECT().GetMaskingViewByID(gomock.Any(), "000120000001", gomock.Any()). + Return(nil, errors.New("not found")).AnyTimes() + mockClient.EXPECT().GetHostByID(gomock.Any(), "000120000001", gomock.Any()). + Return(&types.Host{HostID: "csi-node--worker-1", HostType: "iSCSI"}, nil).AnyTimes() + + svc := &service{ + opts: Opts{ + PortGroups: []string{"csi-pg-1"}, + }, + } + + // PublishMaskingViews returns partial failure + mockClient104.EXPECT().PublishMaskingViews(gomock.Any(), "000120000001", gomock.Any()). + Return(&types.PublishMaskingViewResponse{ + Summary: types.PublishSummary{Total: 1, Failed: 1}, + Results: types.PublishResultsBlock{ + Result: []types.PublishResult{ + {Status: "failed", ResourceID: "csi-mv--worker-1"}, + }, + }, + }, nil).Times(1) + + p := &u4p104VolumePublisher{ + s: svc, + legacyClient: mockClient, + client104: mockClient104, + symID: "000120000001", + devID: "011AB", + volID: volID, + volumeName: "csi-ABC-pmax-testvol", + } + resp, err := p.Publish(context.Background(), &csi.ControllerPublishVolumeRequest{ + NodeId: "worker-1", + VolumeCapability: &csi.VolumeCapability{ + AccessType: &csi.VolumeCapability_Mount{ + Mount: &csi.VolumeCapability_MountVolume{FsType: "ext4"}, + }, + AccessMode: &csi.VolumeCapability_AccessMode{ + Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, + }, + }, + }) + assert.Nil(t, resp) + assert.Error(t, err) + assert.Contains(t, err.Error(), "PublishMaskingViews failed for masking view") +} + +func TestU4P104VolumePublisher_PublishSuccess(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + // Start the lock manager required by updatePublishContext + LockRequestHandler() + + mockClient := mocks.NewMockPmaxClient(ctrl) + mockClient104 := mocks.NewMockPmaxClient(ctrl) + + volID := "csi-ABC-pmax-testvol-000120000001-011AB" + devID := "011AB" + symID := "000120000001" + effectiveWWN := "60000970000120000001533030314142" + tgtMaskingViewID := "csi-mv--worker-1" + portIdentifier := "iqn.1994-05.com.redhat:target1" + // Clear stale node cache entries from prior tests + nodeCache.Delete("000120000001:worker-1") + + mockClient.EXPECT().GetVolumeByID(gomock.Any(), symID, devID). + Return(&types.Volume{ + VolumeID: devID, + VolumeIdentifier: "csi-ABC-pmax-testvol", + EffectiveWWN: effectiveWWN, + }, nil).Times(1) + + // IsNodeNVMe → false (default transport protocol) + mockClient.EXPECT().GetMaskingViewByID(gomock.Any(), symID, gomock.Any()). + Return(nil, errors.New("not found")).AnyTimes() + // IsNodeISCSI → iSCSI host + mockClient.EXPECT().GetHostByID(gomock.Any(), symID, gomock.Any()). + Return(&types.Host{HostID: "csi-node--worker-1", HostType: "iSCSI"}, nil).AnyTimes() + + svc := &service{ + opts: Opts{ + PortGroups: []string{"csi-pg-1"}, + }, + } + getPmaxCache(symID) + + // PublishMaskingViews succeeds + mockClient104.EXPECT().PublishMaskingViews(gomock.Any(), symID, gomock.Any()). + DoAndReturn(func(_ context.Context, _ string, param *types.PublishMaskingViewsParam) (*types.PublishMaskingViewResponse, error) { + // Verify the request structure + assert.Len(t, param.MaskingViews, 1) + mv := param.MaskingViews[0] + assert.Equal(t, tgtMaskingViewID, mv.ID) + assert.NotNil(t, mv.StorageGroup) + assert.NotNil(t, mv.Host) + assert.NotNil(t, mv.PortGroup) + assert.Equal(t, "csi-pg-1", mv.PortGroup.ID) + // Verify volume is in the request + assert.NotNil(t, mv.StorageGroup.Actions) + assert.NotNil(t, mv.StorageGroup.Actions.AddVolumesToStorageGroupAction) + assert.Len(t, mv.StorageGroup.Actions.AddVolumesToStorageGroupAction.Volumes, 1) + assert.Len(t, mv.StorageGroup.Actions.AddVolumesToStorageGroupAction.Volumes[0].ExistingVolumes, 1) + assert.Equal(t, devID, mv.StorageGroup.Actions.AddVolumesToStorageGroupAction.Volumes[0].ExistingVolumes[0].ID) + return &types.PublishMaskingViewResponse{ + Summary: types.PublishSummary{Total: 1, Succeeded: 1}, + }, nil + }).Times(1) + + // GetMaskingViewConnections for updatePublishContext + mockClient.EXPECT().GetMaskingViewConnections(gomock.Any(), symID, tgtMaskingViewID, devID). + Return([]*types.MaskingViewConnection{ + {VolumeID: devID, HostLUNAddress: "0001", DirectorPort: "SE-1E:4"}, + {VolumeID: devID, HostLUNAddress: "0001", DirectorPort: "SE-2E:4"}, + }, nil).Times(1) + + // GetPort for port identifiers + mockClient.EXPECT().GetPort(gomock.Any(), symID, "SE-1E", "4"). + Return(&types.Port{ + SymmetrixPort: types.SymmetrixPortType{ + Identifier: portIdentifier, + }, + }, nil).Times(1) + mockClient.EXPECT().GetPort(gomock.Any(), symID, "SE-2E", "4"). + Return(&types.Port{ + SymmetrixPort: types.SymmetrixPortType{ + Identifier: portIdentifier, + }, + }, nil).Times(1) + + p := &u4p104VolumePublisher{ + s: svc, + legacyClient: mockClient, + client104: mockClient104, + symID: symID, + devID: devID, + volID: volID, + volumeName: "csi-ABC-pmax-testvol", + reqID: "req-1", + } + resp, err := p.Publish(context.Background(), &csi.ControllerPublishVolumeRequest{ + NodeId: "worker-1", + VolumeCapability: &csi.VolumeCapability{ + AccessType: &csi.VolumeCapability_Mount{ + Mount: &csi.VolumeCapability_MountVolume{FsType: "ext4"}, + }, + AccessMode: &csi.VolumeCapability_AccessMode{ + Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, + }, + }, + }) + assert.NoError(t, err) + assert.NotNil(t, resp) + assert.Equal(t, effectiveWWN, resp.PublishContext[PublishContextDeviceWWN]) + assert.Equal(t, "0001", resp.PublishContext[PublishContextLUNAddress]) + assert.NotEmpty(t, resp.PublishContext[PortIdentifiers+"_1"]) +} + +func TestU4P104VolumePublisher_PublishSuccessWithConnectionFetchFailure(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + // Start the lock manager required by updatePublishContext + LockRequestHandler() + + mockClient := mocks.NewMockPmaxClient(ctrl) + mockClient104 := mocks.NewMockPmaxClient(ctrl) + + volID := "csi-ABC-pmax-testvol-000120000001-011AB" + devID := "011AB" + symID := "000120000001" + effectiveWWN := "60000970000120000001533030314142" + tgtMaskingViewID := "csi-mv--worker-1" + portIdentifier := "iqn.1994-05.com.redhat:target1" + // Clear stale node cache entries from prior tests + nodeCache.Delete("000120000001:worker-1") + + mockClient.EXPECT().GetVolumeByID(gomock.Any(), symID, devID). + Return(&types.Volume{ + VolumeID: devID, + VolumeIdentifier: "csi-ABC-pmax-testvol", + EffectiveWWN: effectiveWWN, + }, nil).Times(1) + + // IsNodeNVMe → false + mockClient.EXPECT().GetMaskingViewByID(gomock.Any(), symID, gomock.Any()). + Return(nil, errors.New("not found")).AnyTimes() + // IsNodeISCSI → iSCSI host + mockClient.EXPECT().GetHostByID(gomock.Any(), symID, gomock.Any()). + Return(&types.Host{HostID: "csi-node--worker-1", HostType: "iSCSI"}, nil).AnyTimes() + + svc := &service{ + opts: Opts{ + PortGroups: []string{"csi-pg-1"}, + }, + } + getPmaxCache(symID) + + // PublishMaskingViews succeeds + mockClient104.EXPECT().PublishMaskingViews(gomock.Any(), symID, gomock.Any()). + Return(&types.PublishMaskingViewResponse{ + Summary: types.PublishSummary{Total: 1, Succeeded: 1}, + }, nil).Times(1) + + // First GetMaskingViewConnections (in Publish) fails + // Second GetMaskingViewConnections (retry in updatePublishContext) succeeds + gomock.InOrder( + mockClient.EXPECT().GetMaskingViewConnections(gomock.Any(), symID, tgtMaskingViewID, devID). + Return(nil, errors.New("connection timeout")).Times(1), + mockClient.EXPECT().GetMaskingViewConnections(gomock.Any(), symID, tgtMaskingViewID, devID). + Return([]*types.MaskingViewConnection{ + {VolumeID: devID, HostLUNAddress: "0002", DirectorPort: "SE-1E:4"}, + {VolumeID: devID, HostLUNAddress: "0002", DirectorPort: "SE-2E:4"}, + }, nil).Times(1), + ) + + // GetPort for port identifiers + mockClient.EXPECT().GetPort(gomock.Any(), symID, "SE-1E", "4"). + Return(&types.Port{ + SymmetrixPort: types.SymmetrixPortType{ + Identifier: portIdentifier, + }, + }, nil).Times(1) + mockClient.EXPECT().GetPort(gomock.Any(), symID, "SE-2E", "4"). + Return(&types.Port{ + SymmetrixPort: types.SymmetrixPortType{ + Identifier: portIdentifier, + }, + }, nil).Times(1) + + p := &u4p104VolumePublisher{ + s: svc, + legacyClient: mockClient, + client104: mockClient104, + symID: symID, + devID: devID, + volID: volID, + volumeName: "csi-ABC-pmax-testvol", + reqID: "req-2", + } + resp, err := p.Publish(context.Background(), &csi.ControllerPublishVolumeRequest{ + NodeId: "worker-1", + VolumeCapability: &csi.VolumeCapability{ + AccessType: &csi.VolumeCapability_Mount{ + Mount: &csi.VolumeCapability_MountVolume{FsType: "ext4"}, + }, + AccessMode: &csi.VolumeCapability_AccessMode{ + Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, + }, + }, + }) + assert.NoError(t, err) + assert.NotNil(t, resp) + assert.Equal(t, effectiveWWN, resp.PublishContext[PublishContextDeviceWWN]) + assert.Equal(t, "0002", resp.PublishContext[PublishContextLUNAddress]) + assert.NotEmpty(t, resp.PublishContext[PortIdentifiers+"_1"]) +} + +func TestBuildPublishMaskingViewsParam(t *testing.T) { + param := buildPublishMaskingViewsParam("csi-mv--worker-1", "csi-sg-1", "011AB", "csi-node--worker-1", "csi-pg-1") + + assert.NotNil(t, param) + assert.Len(t, param.MaskingViews, 1) + + mv := param.MaskingViews[0] + assert.Equal(t, "csi-mv--worker-1", mv.ID) + + assert.NotNil(t, mv.StorageGroup) + assert.Equal(t, "csi-sg-1", mv.StorageGroup.ID) + assert.NotNil(t, mv.StorageGroup.Actions) + assert.NotNil(t, mv.StorageGroup.Actions.AddVolumesToStorageGroupAction) + assert.Len(t, mv.StorageGroup.Actions.AddVolumesToStorageGroupAction.Volumes, 1) + assert.Len(t, mv.StorageGroup.Actions.AddVolumesToStorageGroupAction.Volumes[0].ExistingVolumes, 1) + assert.Equal(t, "011AB", mv.StorageGroup.Actions.AddVolumesToStorageGroupAction.Volumes[0].ExistingVolumes[0].ID) + + assert.NotNil(t, mv.Host) + assert.Equal(t, "csi-node--worker-1", mv.Host.ID) + + assert.NotNil(t, mv.PortGroup) + assert.Equal(t, "csi-pg-1", mv.PortGroup.ID) +} diff --git a/service/vsphere.go b/service/vsphere.go index 69b1ef09..8d669ce0 100644 --- a/service/vsphere.go +++ b/service/vsphere.go @@ -18,6 +18,7 @@ package service import ( + "context" "errors" "fmt" "net" @@ -33,7 +34,6 @@ import ( "github.com/vmware/govmomi" "github.com/vmware/govmomi/object" "github.com/vmware/govmomi/vim25/types" - "golang.org/x/net/context" ) // useHTTP - This variable should remain false by default. It was added specifically for unit testing purposes, as unit tests require HTTP instead of HTTPS. @@ -50,7 +50,7 @@ type VMHost struct { // NewVMHost connects to a ESXi or vCenter instance and returns a *VMHost // This method is referenced from https://github.com/codedellemc/govmax/blob/master/api/v1/vmomi.go func NewVMHost(insecure bool, hostURLparam, user, pass string) (*VMHost, error) { - ctx, _ := context.WithCancel(context.Background()) + ctx := context.Background() protocol := "https://" if useHTTP { protocol = "http://" diff --git a/service/vsphere_test.go b/service/vsphere_test.go index 9c8014d6..fa135b05 100644 --- a/service/vsphere_test.go +++ b/service/vsphere_test.go @@ -19,6 +19,7 @@ package service import ( + "context" "errors" "testing" @@ -28,8 +29,6 @@ import ( "github.com/vmware/govmomi/simulator" "github.com/vmware/govmomi/vim25" "github.com/vmware/govmomi/vim25/types" - - "golang.org/x/net/context" ) func TestNewVMHost(t *testing.T) {