From d31c0352b4b1904b3698300e0049c076c65debf5 Mon Sep 17 00:00:00 2001 From: theboringstuff <39027092+theboringstuff@users.noreply.github.com> Date: Fri, 15 May 2026 11:41:57 +0500 Subject: [PATCH 1/4] initial --- documentation/Installation.md | 4 + kubemarine.spec | 1 + kubemarine/patches/software_upgrade.yaml | 1 + .../csi-snapshot-controller-8.5.0/Chart.yaml | 5 + .../templates/_helpers.tpl | 50 + .../templates/crds.yaml | 951 ++++++++++++++++++ .../templates/deployment_controller.yaml | 88 ++ .../templates/serviceaccount_controller.yaml | 93 ++ .../csi-snapshot-controller-8.5.0/values.yaml | 60 ++ kubemarine/plugins/csi_snapshot_controller.py | 51 + .../compatibility/internal/plugins.yaml | 13 + .../compatibility/kubernetes_versions.yaml | 6 + .../resources/configurations/defaults.yaml | 25 + .../resources/configurations/globals.yaml | 4 + .../etalons/patches/software_upgrade.yaml | 1 + .../schemas/definitions/plugins.json | 6 + .../plugins/csi-snapshot-controller.json | 37 + test/unit/tools/thirdparties/test_sync.py | 2 +- 18 files changed, 1397 insertions(+), 1 deletion(-) create mode 100644 kubemarine/plugins/charts/csi-snapshot-controller-8.5.0/Chart.yaml create mode 100644 kubemarine/plugins/charts/csi-snapshot-controller-8.5.0/templates/_helpers.tpl create mode 100644 kubemarine/plugins/charts/csi-snapshot-controller-8.5.0/templates/crds.yaml create mode 100644 kubemarine/plugins/charts/csi-snapshot-controller-8.5.0/templates/deployment_controller.yaml create mode 100644 kubemarine/plugins/charts/csi-snapshot-controller-8.5.0/templates/serviceaccount_controller.yaml create mode 100644 kubemarine/plugins/charts/csi-snapshot-controller-8.5.0/values.yaml create mode 100644 kubemarine/plugins/csi_snapshot_controller.py create mode 100644 kubemarine/resources/schemas/definitions/plugins/csi-snapshot-controller.json diff --git a/documentation/Installation.md b/documentation/Installation.md index c212d9168..f6e8b983a 100644 --- a/documentation/Installation.md +++ b/documentation/Installation.md @@ -4594,6 +4594,10 @@ plugins: * If you use registry other than `docker.io`, you need to edit `csi.plugin.image.repository` in above configuration accordingly. * For old OpenStack version, OpenStack credentials should be stored on nodes under `/etc/kubernetes/cloud.conf` +##### csi-snapshot-controller + +TODO + #### Plugins Features This section provides information about the plugin features in detail. diff --git a/kubemarine.spec b/kubemarine.spec index e85c6b277..62a3e026f 100644 --- a/kubemarine.spec +++ b/kubemarine.spec @@ -15,6 +15,7 @@ a = Analysis(['./kubemarine/__main__.py'], 'kubemarine.plugins.nginx_ingress', 'kubemarine.plugins.envoy_gateway', 'kubemarine.plugins.openstack_cinder_csi', + 'kubemarine.plugins.csi_snapshot_controller', 'kubemarine.plugins.kubernetes_dashboard', ], pathex=[], diff --git a/kubemarine/patches/software_upgrade.yaml b/kubemarine/patches/software_upgrade.yaml index d190351a8..ef7a0c6ec 100644 --- a/kubemarine/patches/software_upgrade.yaml +++ b/kubemarine/patches/software_upgrade.yaml @@ -39,3 +39,4 @@ plugins: local-path-provisioner: [] envoy-gateway: [] openstack-cinder-csi: [] + csi-snapshot-controller: [] diff --git a/kubemarine/plugins/charts/csi-snapshot-controller-8.5.0/Chart.yaml b/kubemarine/plugins/charts/csi-snapshot-controller-8.5.0/Chart.yaml new file mode 100644 index 000000000..ba2784550 --- /dev/null +++ b/kubemarine/plugins/charts/csi-snapshot-controller-8.5.0/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +appVersion: v8.5.0 +description: 'Deploys a Snapshot Controller in a cluster.' +name: csi-snapshot-controller +version: 8.5.0 diff --git a/kubemarine/plugins/charts/csi-snapshot-controller-8.5.0/templates/_helpers.tpl b/kubemarine/plugins/charts/csi-snapshot-controller-8.5.0/templates/_helpers.tpl new file mode 100644 index 000000000..0270b125b --- /dev/null +++ b/kubemarine/plugins/charts/csi-snapshot-controller-8.5.0/templates/_helpers.tpl @@ -0,0 +1,50 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "snapshot-controller.name" -}} +{{- .Chart.Name | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "snapshot-controller.fullname" -}} +{{- if .Values.controller.fullnameOverride -}} +{{- .Values.controller.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- if contains .Chart.Name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name .Chart.Name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "snapshot-controller.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "snapshot-controller.labels" -}} +helm.sh/chart: {{ include "snapshot-controller.chart" . }} +{{ include "snapshot-controller.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "snapshot-controller.selectorLabels" -}} +app.kubernetes.io/name: {{ include "snapshot-controller.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} \ No newline at end of file diff --git a/kubemarine/plugins/charts/csi-snapshot-controller-8.5.0/templates/crds.yaml b/kubemarine/plugins/charts/csi-snapshot-controller-8.5.0/templates/crds.yaml new file mode 100644 index 000000000..0aebded96 --- /dev/null +++ b/kubemarine/plugins/charts/csi-snapshot-controller-8.5.0/templates/crds.yaml @@ -0,0 +1,951 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/814" + controller-gen.kubebuilder.io/version: v0.15.0 + name: volumesnapshotclasses.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotClass + listKind: VolumeSnapshotClassList + plural: volumesnapshotclasses + shortNames: + - vsclass + - vsclasses + singular: volumesnapshotclass + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .driver + name: Driver + type: string + - description: Determines whether a VolumeSnapshotContent created through the + VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. + jsonPath: .deletionPolicy + name: DeletionPolicy + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: |- + VolumeSnapshotClass specifies parameters that a underlying storage system uses when + creating a volume snapshot. A specific VolumeSnapshotClass is used by specifying its + name in a VolumeSnapshot object. + VolumeSnapshotClasses are non-namespaced + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + deletionPolicy: + description: |- + deletionPolicy determines whether a VolumeSnapshotContent created through + the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. + Supported values are "Retain" and "Delete". + "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. + "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. + Required. + enum: + - Delete + - Retain + type: string + driver: + description: |- + driver is the name of the storage driver that handles this VolumeSnapshotClass. + Required. + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + parameters: + additionalProperties: + type: string + description: |- + parameters is a key-value map with storage driver specific parameters for creating snapshots. + These values are opaque to Kubernetes. + type: object + required: + - deletionPolicy + - driver + type: object + served: true + storage: true + subresources: {} + - additionalPrinterColumns: + - jsonPath: .driver + name: Driver + type: string + - description: Determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. + jsonPath: .deletionPolicy + name: DeletionPolicy + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + # This indicates the v1beta1 version of the custom resource is deprecated. + # API requests to this version receive a warning in the server response. + deprecated: true + # This overrides the default warning returned to clients making v1beta1 API requests. + deprecationWarning: "snapshot.storage.k8s.io/v1beta1 VolumeSnapshotClass is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshotClass" + schema: + openAPIV3Schema: + description: VolumeSnapshotClass specifies parameters that a underlying storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses are non-namespaced + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + deletionPolicy: + description: deletionPolicy determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the storage driver that handles this VolumeSnapshotClass. Required. + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + parameters: + additionalProperties: + type: string + description: parameters is a key-value map with storage driver specific parameters for creating snapshots. These values are opaque to Kubernetes. + type: object + required: + - deletionPolicy + - driver + type: object + served: false + storage: false + subresources: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/955" + name: volumesnapshotcontents.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotContent + listKind: VolumeSnapshotContentList + plural: volumesnapshotcontents + shortNames: + - vsc + - vscs + singular: volumesnapshotcontent + scope: Cluster + versions: + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: Represents the complete size of the snapshot in bytes + jsonPath: .status.restoreSize + name: RestoreSize + type: integer + - description: Determines whether this VolumeSnapshotContent and its physical + snapshot on the underlying storage system should be deleted when its bound + VolumeSnapshot is deleted. + jsonPath: .spec.deletionPolicy + name: DeletionPolicy + type: string + - description: Name of the CSI driver used to create the physical snapshot on + the underlying storage system. + jsonPath: .spec.driver + name: Driver + type: string + - description: Name of the VolumeSnapshotClass to which this snapshot belongs. + jsonPath: .spec.volumeSnapshotClassName + name: VolumeSnapshotClass + type: string + - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent + object is bound. + jsonPath: .spec.volumeSnapshotRef.name + name: VolumeSnapshot + type: string + - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent + object is bound. + jsonPath: .spec.volumeSnapshotRef.namespace + name: VolumeSnapshotNamespace + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: |- + VolumeSnapshotContent represents the actual "on-disk" snapshot object in the + underlying storage system + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + spec defines properties of a VolumeSnapshotContent created by the underlying storage system. + Required. + properties: + deletionPolicy: + description: |- + deletionPolicy determines whether this VolumeSnapshotContent and its physical snapshot on + the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. + Supported values are "Retain" and "Delete". + "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. + "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. + For dynamically provisioned snapshots, this field will automatically be filled in by the + CSI snapshotter sidecar with the "DeletionPolicy" field defined in the corresponding + VolumeSnapshotClass. + For pre-existing snapshots, users MUST specify this field when creating the + VolumeSnapshotContent object. + Required. + enum: + - Delete + - Retain + type: string + driver: + description: |- + driver is the name of the CSI driver used to create the physical snapshot on + the underlying storage system. + This MUST be the same as the name returned by the CSI GetPluginName() call for + that driver. + Required. + type: string + source: + description: |- + source specifies whether the snapshot is (or should be) dynamically provisioned + or already exists, and just requires a Kubernetes object representation. + This field is immutable after creation. + Required. + properties: + snapshotHandle: + description: |- + snapshotHandle specifies the CSI "snapshot_id" of a pre-existing snapshot on + the underlying storage system for which a Kubernetes object representation + was (or should be) created. + This field is immutable. + type: string + x-kubernetes-validations: + - message: snapshotHandle is immutable + rule: self == oldSelf + volumeHandle: + description: |- + volumeHandle specifies the CSI "volume_id" of the volume from which a snapshot + should be dynamically taken from. + This field is immutable. + type: string + x-kubernetes-validations: + - message: volumeHandle is immutable + rule: self == oldSelf + type: object + x-kubernetes-validations: + - message: volumeHandle is required once set + rule: '!has(oldSelf.volumeHandle) || has(self.volumeHandle)' + - message: snapshotHandle is required once set + rule: '!has(oldSelf.snapshotHandle) || has(self.snapshotHandle)' + - message: exactly one of volumeHandle and snapshotHandle must be + set + rule: (has(self.volumeHandle) && !has(self.snapshotHandle)) || (!has(self.volumeHandle) + && has(self.snapshotHandle)) + sourceVolumeMode: + description: |- + SourceVolumeMode is the mode of the volume whose snapshot is taken. + Can be either “Filesystem” or “Block”. + If not specified, it indicates the source volume's mode is unknown. + This field is immutable. + This field is an alpha field. + type: string + x-kubernetes-validations: + - message: sourceVolumeMode is immutable + rule: self == oldSelf + volumeSnapshotClassName: + description: |- + name of the VolumeSnapshotClass from which this snapshot was (or will be) + created. + Note that after provisioning, the VolumeSnapshotClass may be deleted or + recreated with different set of values, and as such, should not be referenced + post-snapshot creation. + type: string + volumeSnapshotRef: + description: |- + volumeSnapshotRef specifies the VolumeSnapshot object to which this + VolumeSnapshotContent object is bound. + VolumeSnapshot.Spec.VolumeSnapshotContentName field must reference to + this VolumeSnapshotContent's name for the bidirectional binding to be valid. + For a pre-existing VolumeSnapshotContent object, name and namespace of the + VolumeSnapshot object MUST be provided for binding to happen. + This field is immutable after creation. + Required. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. + TODO: this design is not final and this field is subject to change in the future. + type: string + kind: + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + namespace: + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + type: string + resourceVersion: + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + type: string + uid: + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids + type: string + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: both spec.volumeSnapshotRef.name and spec.volumeSnapshotRef.namespace + must be set + rule: has(self.name) && has(self.__namespace__) + required: + - deletionPolicy + - driver + - source + - volumeSnapshotRef + type: object + x-kubernetes-validations: + - message: sourceVolumeMode is required once set + rule: '!has(oldSelf.sourceVolumeMode) || has(self.sourceVolumeMode)' + status: + description: status represents the current information of a snapshot. + properties: + creationTime: + description: |- + creationTime is the timestamp when the point-in-time snapshot is taken + by the underlying storage system. + In dynamic snapshot creation case, this field will be filled in by the + CSI snapshotter sidecar with the "creation_time" value returned from CSI + "CreateSnapshot" gRPC call. + For a pre-existing snapshot, this field will be filled with the "creation_time" + value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. + If not specified, it indicates the creation time is unknown. + The format of this field is a Unix nanoseconds time encoded as an int64. + On Unix, the command `date +%s%N` returns the current time in nanoseconds + since 1970-01-01 00:00:00 UTC. + format: int64 + type: integer + error: + description: |- + error is the last observed error during snapshot creation, if any. + Upon success after retry, this error field will be cleared. + properties: + message: + description: |- + message is a string detailing the encountered error during snapshot + creation if specified. + NOTE: message may be logged, and it should not contain sensitive + information. + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: |- + readyToUse indicates if a snapshot is ready to be used to restore a volume. + In dynamic snapshot creation case, this field will be filled in by the + CSI snapshotter sidecar with the "ready_to_use" value returned from CSI + "CreateSnapshot" gRPC call. + For a pre-existing snapshot, this field will be filled with the "ready_to_use" + value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, + otherwise, this field will be set to "True". + If not specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + description: |- + restoreSize represents the complete size of the snapshot in bytes. + In dynamic snapshot creation case, this field will be filled in by the + CSI snapshotter sidecar with the "size_bytes" value returned from CSI + "CreateSnapshot" gRPC call. + For a pre-existing snapshot, this field will be filled with the "size_bytes" + value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. + When restoring a volume from this snapshot, the size of the volume MUST NOT + be smaller than the restoreSize if it is specified, otherwise the restoration will fail. + If not specified, it indicates that the size is unknown. + format: int64 + minimum: 0 + type: integer + snapshotHandle: + description: |- + snapshotHandle is the CSI "snapshot_id" of a snapshot on the underlying storage system. + If not specified, it indicates that dynamic snapshot creation has either failed + or it is still in progress. + type: string + volumeGroupSnapshotHandle: + description: |- + VolumeGroupSnapshotHandle is the CSI "group_snapshot_id" of a group snapshot + on the underlying storage system. + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: Represents the complete size of the snapshot in bytes + jsonPath: .status.restoreSize + name: RestoreSize + type: integer + - description: Determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. + jsonPath: .spec.deletionPolicy + name: DeletionPolicy + type: string + - description: Name of the CSI driver used to create the physical snapshot on the underlying storage system. + jsonPath: .spec.driver + name: Driver + type: string + - description: Name of the VolumeSnapshotClass to which this snapshot belongs. + jsonPath: .spec.volumeSnapshotClassName + name: VolumeSnapshotClass + type: string + - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. + jsonPath: .spec.volumeSnapshotRef.name + name: VolumeSnapshot + type: string + - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. + jsonPath: .spec.volumeSnapshotRef.namespace + name: VolumeSnapshotNamespace + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + # This indicates the v1beta1 version of the custom resource is deprecated. + # API requests to this version receive a warning in the server response. + deprecated: true + # This overrides the default warning returned to clients making v1beta1 API requests. + deprecationWarning: "snapshot.storage.k8s.io/v1beta1 VolumeSnapshotContent is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshotContent" + schema: + openAPIV3Schema: + description: VolumeSnapshotContent represents the actual "on-disk" snapshot object in the underlying storage system + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: spec defines properties of a VolumeSnapshotContent created by the underlying storage system. Required. + properties: + deletionPolicy: + description: deletionPolicy determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. For dynamically provisioned snapshots, this field will automatically be filled in by the CSI snapshotter sidecar with the "DeletionPolicy" field defined in the corresponding VolumeSnapshotClass. For pre-existing snapshots, users MUST specify this field when creating the VolumeSnapshotContent object. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the CSI driver used to create the physical snapshot on the underlying storage system. This MUST be the same as the name returned by the CSI GetPluginName() call for that driver. Required. + type: string + source: + description: source specifies whether the snapshot is (or should be) dynamically provisioned or already exists, and just requires a Kubernetes object representation. This field is immutable after creation. Required. + properties: + snapshotHandle: + description: snapshotHandle specifies the CSI "snapshot_id" of a pre-existing snapshot on the underlying storage system for which a Kubernetes object representation was (or should be) created. This field is immutable. + type: string + volumeHandle: + description: volumeHandle specifies the CSI "volume_id" of the volume from which a snapshot should be dynamically taken from. This field is immutable. + type: string + type: object + volumeSnapshotClassName: + description: name of the VolumeSnapshotClass from which this snapshot was (or will be) created. Note that after provisioning, the VolumeSnapshotClass may be deleted or recreated with different set of values, and as such, should not be referenced post-snapshot creation. + type: string + volumeSnapshotRef: + description: volumeSnapshotRef specifies the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName field must reference to this VolumeSnapshotContent's name for the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent object, name and namespace of the VolumeSnapshot object MUST be provided for binding to happen. This field is immutable after creation. Required. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + required: + - deletionPolicy + - driver + - source + - volumeSnapshotRef + type: object + status: + description: status represents the current information of a snapshot. + properties: + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it indicates the creation time is unknown. The format of this field is a Unix nanoseconds time encoded as an int64. On Unix, the command `date +%s%N` returns the current time in nanoseconds since 1970-01-01 00:00:00 UTC. + format: int64 + type: integer + error: + description: error is the last observed error during snapshot creation, if any. Upon success after retry, this error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot in bytes. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. + format: int64 + minimum: 0 + type: integer + snapshotHandle: + description: snapshotHandle is the CSI "snapshot_id" of a snapshot on the underlying storage system. If not specified, it indicates that dynamic snapshot creation has either failed or it is still in progress. + type: string + type: object + required: + - spec + type: object + served: false + storage: false + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/814" + name: volumesnapshots.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshot + listKind: VolumeSnapshotList + plural: volumesnapshots + shortNames: + - vs + singular: volumesnapshot + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: If a new snapshot needs to be created, this contains the name of + the source PVC from which this snapshot was (or will be) created. + jsonPath: .spec.source.persistentVolumeClaimName + name: SourcePVC + type: string + - description: If a snapshot already exists, this contains the name of the existing + VolumeSnapshotContent object representing the existing snapshot. + jsonPath: .spec.source.volumeSnapshotContentName + name: SourceSnapshotContent + type: string + - description: Represents the minimum size of volume required to rehydrate from + this snapshot. + jsonPath: .status.restoreSize + name: RestoreSize + type: string + - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot. + jsonPath: .spec.volumeSnapshotClassName + name: SnapshotClass + type: string + - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot + object intends to bind to. Please note that verification of binding actually + requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure + both are pointing at each other. Binding MUST be verified prior to usage of + this object. + jsonPath: .status.boundVolumeSnapshotContentName + name: SnapshotContent + type: string + - description: Timestamp when the point-in-time snapshot was taken by the underlying + storage system. + jsonPath: .status.creationTime + name: CreationTime + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: |- + VolumeSnapshot is a user's request for either creating a point-in-time + snapshot of a persistent volume, or binding to a pre-existing snapshot. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + spec defines the desired characteristics of a snapshot requested by a user. + More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots + Required. + properties: + source: + description: |- + source specifies where a snapshot will be created from. + This field is immutable after creation. + Required. + properties: + persistentVolumeClaimName: + description: |- + persistentVolumeClaimName specifies the name of the PersistentVolumeClaim + object representing the volume from which a snapshot should be created. + This PVC is assumed to be in the same namespace as the VolumeSnapshot + object. + This field should be set if the snapshot does not exists, and needs to be + created. + This field is immutable. + type: string + x-kubernetes-validations: + - message: persistentVolumeClaimName is immutable + rule: self == oldSelf + volumeSnapshotContentName: + description: |- + volumeSnapshotContentName specifies the name of a pre-existing VolumeSnapshotContent + object representing an existing volume snapshot. + This field should be set if the snapshot already exists and only needs a representation in Kubernetes. + This field is immutable. + type: string + x-kubernetes-validations: + - message: volumeSnapshotContentName is immutable + rule: self == oldSelf + type: object + x-kubernetes-validations: + - message: persistentVolumeClaimName is required once set + rule: '!has(oldSelf.persistentVolumeClaimName) || has(self.persistentVolumeClaimName)' + - message: volumeSnapshotContentName is required once set + rule: '!has(oldSelf.volumeSnapshotContentName) || has(self.volumeSnapshotContentName)' + - message: exactly one of volumeSnapshotContentName and persistentVolumeClaimName + must be set + rule: (has(self.volumeSnapshotContentName) && !has(self.persistentVolumeClaimName)) + || (!has(self.volumeSnapshotContentName) && has(self.persistentVolumeClaimName)) + volumeSnapshotClassName: + description: |- + VolumeSnapshotClassName is the name of the VolumeSnapshotClass + requested by the VolumeSnapshot. + VolumeSnapshotClassName may be left nil to indicate that the default + SnapshotClass should be used. + A given cluster may have multiple default Volume SnapshotClasses: one + default per CSI Driver. If a VolumeSnapshot does not specify a SnapshotClass, + VolumeSnapshotSource will be checked to figure out what the associated + CSI Driver is, and the default VolumeSnapshotClass associated with that + CSI Driver will be used. If more than one VolumeSnapshotClass exist for + a given CSI Driver and more than one have been marked as default, + CreateSnapshot will fail and generate an event. + Empty string is not allowed for this field. + type: string + x-kubernetes-validations: + - message: volumeSnapshotClassName must not be the empty string when + set + rule: size(self) > 0 + required: + - source + type: object + status: + description: |- + status represents the current information of a snapshot. + Consumers must verify binding between VolumeSnapshot and + VolumeSnapshotContent objects is successful (by validating that both + VolumeSnapshot and VolumeSnapshotContent point at each other) before + using this object. + properties: + boundVolumeSnapshotContentName: + description: |- + boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent + object to which this VolumeSnapshot object intends to bind to. + If not specified, it indicates that the VolumeSnapshot object has not been + successfully bound to a VolumeSnapshotContent object yet. + NOTE: To avoid possible security issues, consumers must verify binding between + VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that + both VolumeSnapshot and VolumeSnapshotContent point at each other) before using + this object. + type: string + creationTime: + description: |- + creationTime is the timestamp when the point-in-time snapshot is taken + by the underlying storage system. + In dynamic snapshot creation case, this field will be filled in by the + snapshot controller with the "creation_time" value returned from CSI + "CreateSnapshot" gRPC call. + For a pre-existing snapshot, this field will be filled with the "creation_time" + value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. + If not specified, it may indicate that the creation time of the snapshot is unknown. + format: date-time + type: string + error: + description: |- + error is the last observed error during snapshot creation, if any. + This field could be helpful to upper level controllers(i.e., application controller) + to decide whether they should continue on waiting for the snapshot to be created + based on the type of error reported. + The snapshot controller will keep retrying when an error occurs during the + snapshot creation. Upon success, this error field will be cleared. + properties: + message: + description: |- + message is a string detailing the encountered error during snapshot + creation if specified. + NOTE: message may be logged, and it should not contain sensitive + information. + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: |- + readyToUse indicates if the snapshot is ready to be used to restore a volume. + In dynamic snapshot creation case, this field will be filled in by the + snapshot controller with the "ready_to_use" value returned from CSI + "CreateSnapshot" gRPC call. + For a pre-existing snapshot, this field will be filled with the "ready_to_use" + value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, + otherwise, this field will be set to "True". + If not specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + type: string + description: |- + restoreSize represents the minimum size of volume required to create a volume + from this snapshot. + In dynamic snapshot creation case, this field will be filled in by the + snapshot controller with the "size_bytes" value returned from CSI + "CreateSnapshot" gRPC call. + For a pre-existing snapshot, this field will be filled with the "size_bytes" + value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. + When restoring a volume from this snapshot, the size of the volume MUST NOT + be smaller than the restoreSize if it is specified, otherwise the restoration will fail. + If not specified, it indicates that the size is unknown. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + volumeGroupSnapshotName: + description: |- + VolumeGroupSnapshotName is the name of the VolumeGroupSnapshot of which this + VolumeSnapshot is a part of. + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: If a new snapshot needs to be created, this contains the name of the source PVC from which this snapshot was (or will be) created. + jsonPath: .spec.source.persistentVolumeClaimName + name: SourcePVC + type: string + - description: If a snapshot already exists, this contains the name of the existing VolumeSnapshotContent object representing the existing snapshot. + jsonPath: .spec.source.volumeSnapshotContentName + name: SourceSnapshotContent + type: string + - description: Represents the minimum size of volume required to rehydrate from this snapshot. + jsonPath: .status.restoreSize + name: RestoreSize + type: string + - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot. + jsonPath: .spec.volumeSnapshotClassName + name: SnapshotClass + type: string + - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot object intends to bind to. Please note that verification of binding actually requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure both are pointing at each other. Binding MUST be verified prior to usage of this object. + jsonPath: .status.boundVolumeSnapshotContentName + name: SnapshotContent + type: string + - description: Timestamp when the point-in-time snapshot was taken by the underlying storage system. + jsonPath: .status.creationTime + name: CreationTime + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + # This indicates the v1beta1 version of the custom resource is deprecated. + # API requests to this version receive a warning in the server response. + deprecated: true + # This overrides the default warning returned to clients making v1beta1 API requests. + deprecationWarning: "snapshot.storage.k8s.io/v1beta1 VolumeSnapshot is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshot" + schema: + openAPIV3Schema: + description: VolumeSnapshot is a user's request for either creating a point-in-time snapshot of a persistent volume, or binding to a pre-existing snapshot. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: 'spec defines the desired characteristics of a snapshot requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots Required.' + properties: + source: + description: source specifies where a snapshot will be created from. This field is immutable after creation. Required. + properties: + persistentVolumeClaimName: + description: persistentVolumeClaimName specifies the name of the PersistentVolumeClaim object representing the volume from which a snapshot should be created. This PVC is assumed to be in the same namespace as the VolumeSnapshot object. This field should be set if the snapshot does not exists, and needs to be created. This field is immutable. + type: string + volumeSnapshotContentName: + description: volumeSnapshotContentName specifies the name of a pre-existing VolumeSnapshotContent object representing an existing volume snapshot. This field should be set if the snapshot already exists and only needs a representation in Kubernetes. This field is immutable. + type: string + type: object + volumeSnapshotClassName: + description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass requested by the VolumeSnapshot. VolumeSnapshotClassName may be left nil to indicate that the default SnapshotClass should be used. A given cluster may have multiple default Volume SnapshotClasses: one default per CSI Driver. If a VolumeSnapshot does not specify a SnapshotClass, VolumeSnapshotSource will be checked to figure out what the associated CSI Driver is, and the default VolumeSnapshotClass associated with that CSI Driver will be used. If more than one VolumeSnapshotClass exist for a given CSI Driver and more than one have been marked as default, CreateSnapshot will fail and generate an event. Empty string is not allowed for this field.' + type: string + required: + - source + type: object + status: + description: status represents the current information of a snapshot. Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object. + properties: + boundVolumeSnapshotContentName: + description: 'boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent object to which this VolumeSnapshot object intends to bind to. If not specified, it indicates that the VolumeSnapshot object has not been successfully bound to a VolumeSnapshotContent object yet. NOTE: To avoid possible security issues, consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object.' + type: string + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it may indicate that the creation time of the snapshot is unknown. + format: date-time + type: string + error: + description: error is the last observed error during snapshot creation, if any. This field could be helpful to upper level controllers(i.e., application controller) to decide whether they should continue on waiting for the snapshot to be created based on the type of error reported. The snapshot controller will keep retrying when an error occurs during the snapshot creation. Upon success, this error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if the snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + type: string + description: restoreSize represents the minimum size of volume required to create a volume from this snapshot. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + required: + - spec + type: object + served: false + storage: false + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] \ No newline at end of file diff --git a/kubemarine/plugins/charts/csi-snapshot-controller-8.5.0/templates/deployment_controller.yaml b/kubemarine/plugins/charts/csi-snapshot-controller-8.5.0/templates/deployment_controller.yaml new file mode 100644 index 000000000..e214a64ea --- /dev/null +++ b/kubemarine/plugins/charts/csi-snapshot-controller-8.5.0/templates/deployment_controller.yaml @@ -0,0 +1,88 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "snapshot-controller.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "snapshot-controller.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.controller.replicaCount }} + revisionHistoryLimit: {{ .Values.controller.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "snapshot-controller.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.controller.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "snapshot-controller.selectorLabels" . | nindent 8 }} + {{- with .Values.controller.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.controller.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "snapshot-controller.fullname" . }} + securityContext: + {{- toYaml .Values.controller.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.controller.securityContext | nindent 12 }} + image: "{{ .Values.controller.image.repository }}:{{ .Values.controller.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.controller.image.pullPolicy }} + args: + {{- range $flag, $val := .Values.controller.args }} + - --{{ $flag | kebabcase }}={{ $val }} + {{- end }} + ports: + - name: http + containerPort: 8080 + protocol: TCP + readinessProbe: + httpGet: + port: http + path: /healthz/leader-election + scheme: HTTP + livenessProbe: + httpGet: + port: http + path: /healthz/leader-election + scheme: HTTP + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + resources: + {{- toYaml .Values.controller.resources | nindent 12 }} + {{- with .Values.controller.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.controller.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.controller.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.controller.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.controller.priorityClassName }} + priorityClassName: {{ .Values.controller.priorityClassName }} + {{- end }} + hostNetwork: {{ .Values.controller.hostNetwork }} + {{- with .Values.controller.dnsConfig }} + dnsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + dnsPolicy: {{ .Values.controller.dnsPolicy }} diff --git a/kubemarine/plugins/charts/csi-snapshot-controller-8.5.0/templates/serviceaccount_controller.yaml b/kubemarine/plugins/charts/csi-snapshot-controller-8.5.0/templates/serviceaccount_controller.yaml new file mode 100644 index 000000000..58088b76b --- /dev/null +++ b/kubemarine/plugins/charts/csi-snapshot-controller-8.5.0/templates/serviceaccount_controller.yaml @@ -0,0 +1,93 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "snapshot-controller.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "snapshot-controller.labels" . | nindent 4 }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "snapshot-controller.fullname" . }} +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["create", "get", "list", "watch", "update", "patch", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update", "patch"] + - apiGroups: ["groupsnapshot.storage.k8s.io"] + resources: ["volumegroupsnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["groupsnapshot.storage.k8s.io"] + resources: ["volumegroupsnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete", "patch"] + - apiGroups: ["groupsnapshot.storage.k8s.io"] + resources: ["volumegroupsnapshotcontents/status"] + verbs: ["patch"] + - apiGroups: ["groupsnapshot.storage.k8s.io"] + resources: ["volumegroupsnapshots"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["groupsnapshot.storage.k8s.io"] + resources: ["volumegroupsnapshots/status"] + verbs: ["update", "patch"] + {{- if get .Values.controller.args "enableDistributedSnapshotting" }} + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + {{- end }} +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "snapshot-controller.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ include "snapshot-controller.fullname" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "snapshot-controller.fullname" . }} + apiGroup: rbac.authorization.k8s.io +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "snapshot-controller.fullname" . }} + namespace: {{ .Release.Namespace }} +rules: + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "snapshot-controller.fullname" . }} + namespace: {{ .Release.Namespace }} +subjects: + - kind: ServiceAccount + name: {{ include "snapshot-controller.fullname" . }} +roleRef: + kind: Role + name: {{ include "snapshot-controller.fullname" . }} + apiGroup: rbac.authorization.k8s.io diff --git a/kubemarine/plugins/charts/csi-snapshot-controller-8.5.0/values.yaml b/kubemarine/plugins/charts/csi-snapshot-controller-8.5.0/values.yaml new file mode 100644 index 000000000..138164ff2 --- /dev/null +++ b/kubemarine/plugins/charts/csi-snapshot-controller-8.5.0/values.yaml @@ -0,0 +1,60 @@ +controller: + replicaCount: 1 + + revisionHistoryLimit: 10 + + fullnameOverride: "" + + args: + leaderElection: true + leaderElectionNamespace: "$(NAMESPACE)" + httpEndpoint: ":8080" + + image: + repository: registry.k8s.io/sig-storage/snapshot-controller + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + + imagePullSecrets: [] + + podAnnotations: {} + + podLabels: {} + + podSecurityContext: {} + # fsGroup: 2000 + + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + + + resources: {} + + nodeSelector: {} + + tolerations: [] + + affinity: {} + + topologySpreadConstraints: [] + + priorityClassName: "" + # Specifies wether a Priority Class should be attached to deployment pods + + # Change `hostNetwork` to `true` when you want the pod to share its host's network namespace. + hostNetwork: false + + # DNS settings for the controller pod. https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-dns-config + dnsConfig: {} + # DNS Policy for controller pod. For Pods running with hostNetwork, set to `ClusterFirstWithHostNet` + # For further reference: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy. + dnsPolicy: ClusterFirst \ No newline at end of file diff --git a/kubemarine/plugins/csi_snapshot_controller.py b/kubemarine/plugins/csi_snapshot_controller.py new file mode 100644 index 000000000..f8e0f00c0 --- /dev/null +++ b/kubemarine/plugins/csi_snapshot_controller.py @@ -0,0 +1,51 @@ +import io +import yaml +from kubemarine import plugins +from kubemarine.core import utils +from kubemarine.core.cluster import KubernetesCluster +from kubemarine.core.yaml_merger import default_merger + +def apply_snapshot_controller_chart(cluster: KubernetesCluster) -> None: + snapshot_controller_plugin = cluster.inventory["plugins"]["csi-snapshot-controller"] + chart_version = snapshot_controller_plugin["version"] + + helm_plugin_config = { + "chart_path": utils.get_internal_resource_path(f"plugins/charts/csi-snapshot-controller-{chart_version}"), + "namespace": snapshot_controller_plugin["namespace"], + "release": snapshot_controller_plugin["releaseName"], + "take_ownership": True, + } + + if "registry" in snapshot_controller_plugin["installation"]: + registry = snapshot_controller_plugin["installation"]["registry"] + helm_plugin_config["values"] = { + "controller": { + "image": { + "repository": f"{registry}/sig-storage/snapshot-controller" + }, + } + } + + helm_plugin_config["values"] = default_merger.merge(helm_plugin_config["values"], snapshot_controller_plugin["values"]) + utils.dump_file( + cluster.context, + yaml.dump(helm_plugin_config["values"]), "csi-snapshot-controller-values.yaml", + dump_location=True + ) + plugins.apply_helm(cluster=cluster, config=helm_plugin_config) + + +def apply_additional_resources(cluster: KubernetesCluster) -> None: + snapshot_controller_plugin = cluster.inventory["plugins"]["csi-snapshot-controller"] + if not snapshot_controller_plugin["additionalResources"]: + cluster.log.debug(f"Additional resources are not specified, skipping") + return + + destination = '/etc/kubernetes/csi-snapshot-controller-additional-resources.yaml' + config = { + "source": io.StringIO(snapshot_controller_plugin["additionalResources"]), + "destination": destination, + "do_render": False + } + plugins.apply_source(cluster, config) + diff --git a/kubemarine/resources/configurations/compatibility/internal/plugins.yaml b/kubemarine/resources/configurations/compatibility/internal/plugins.yaml index 10828c50e..e7db9c8a9 100644 --- a/kubemarine/resources/configurations/compatibility/internal/plugins.yaml +++ b/kubemarine/resources/configurations/compatibility/internal/plugins.yaml @@ -166,3 +166,16 @@ openstack-cinder-csi: livenessprobe-version: v2.17.0 nodeDriverRegistrar-version: v2.15.0 plugin-version: v1.35.0 +csi-snapshot-controller: + v1.33.0: + version: 8.5.0 + v1.33.6: + version: 8.5.0 + v1.34.1: + version: 8.5.0 + v1.34.2: + version: 8.5.0 + v1.35.0: + version: 8.5.0 + v1.36.0: + version: 8.5.0 diff --git a/kubemarine/resources/configurations/compatibility/kubernetes_versions.yaml b/kubemarine/resources/configurations/compatibility/kubernetes_versions.yaml index 08765376e..f187583f9 100644 --- a/kubemarine/resources/configurations/compatibility/kubernetes_versions.yaml +++ b/kubemarine/resources/configurations/compatibility/kubernetes_versions.yaml @@ -18,6 +18,7 @@ compatibility_map: local-path-provisioner: v0.0.31 crictl: v1.33.0 openstack-cinder-csi: 2.35.0 + csi-snapshot-controller: 8.5.0 v1.33.6: calico: v3.31.2 nginx-ingress-controller: v1.16.0 @@ -26,6 +27,7 @@ compatibility_map: local-path-provisioner: v0.0.32 crictl: v1.33.0 openstack-cinder-csi: 2.35.0 + csi-snapshot-controller: 8.5.0 v1.34.1: calico: v3.30.3 nginx-ingress-controller: v1.16.0 @@ -34,6 +36,7 @@ compatibility_map: local-path-provisioner: v0.0.32 crictl: v1.34.0 openstack-cinder-csi: 2.35.0 + csi-snapshot-controller: 8.5.0 v1.34.2: calico: v3.32.0 nginx-ingress-controller: v1.16.0 @@ -42,6 +45,7 @@ compatibility_map: local-path-provisioner: v0.0.32 crictl: v1.34.0 openstack-cinder-csi: 2.35.0 + csi-snapshot-controller: 8.5.0 v1.35.0: calico: v3.32.0 nginx-ingress-controller: v1.16.0 @@ -50,6 +54,7 @@ compatibility_map: local-path-provisioner: v0.0.32 crictl: v1.35.0 openstack-cinder-csi: 2.35.0 + csi-snapshot-controller: 8.5.0 v1.36.0: calico: v3.32.0 nginx-ingress-controller: v1.16.0 @@ -58,6 +63,7 @@ compatibility_map: local-path-provisioner: v0.0.32 crictl: v1.36.0 openstack-cinder-csi: 2.35.0 + csi-snapshot-controller: 8.5.0 # After any change, please run scripts/thirdparties/sync.py # The following optional keys are supported in addition to the 5 required software keys: diff --git a/kubemarine/resources/configurations/defaults.yaml b/kubemarine/resources/configurations/defaults.yaml index d6a4fa7ee..ea4f8a6b3 100644 --- a/kubemarine/resources/configurations/defaults.yaml +++ b/kubemarine/resources/configurations/defaults.yaml @@ -925,6 +925,31 @@ plugins: - '{{ plugins["openstack-cinder-csi"].releaseName }}-nodeplugin' namespace: '{{ plugins["openstack-cinder-csi"].namespace }}' + csi-snapshot-controller: + version: '{{ globals.compatibility_map.software["csi-snapshot-controller"][services.kubeadm.kubernetesVersion].version }}' + install: false + namespace: "csi-snapshot-controller" + releaseName: "csi-snapshot-controller" + values: {} + installation: + priority: 3 + procedures: + - python: + module: plugins/csi_snapshot_controller.py + method: apply_snapshot_controller_chart + - expect: + deployments: + - name: '{{ plugins["csi-snapshot-controller"].releaseName }}' + namespace: '{{ plugins["csi-snapshot-controller"].namespace }}' + pods: + list: + - '{{ plugins["csi-snapshot-controller"].releaseName }}' + namespace: '{{ plugins["csi-snapshot-controller"].namespace }}' + - python: + module: plugins/csi_snapshot_controller.py + method: apply_additional_resources + additionalResources: "" + rbac: account_defaults: namespace: kube-system diff --git a/kubemarine/resources/configurations/globals.yaml b/kubemarine/resources/configurations/globals.yaml index f369f347a..3145943cb 100644 --- a/kubemarine/resources/configurations/globals.yaml +++ b/kubemarine/resources/configurations/globals.yaml @@ -125,6 +125,10 @@ plugins: manifests: [] # manifests are empty because it is helm chart requirements: default: https://github.com/kubernetes/cloud-provider-openstack/blob/openstack-cinder-csi-{version}/docs/cinder-csi-plugin/using-cinder-csi-plugin.md#csi-compatibility + csi-snapshot-controller: + manifests: [] # manifests are empty because it is helm chart + requirements: + default: https://github.com/kubernetes-csi/external-snapshotter#compatibility software: containerd: diff --git a/kubemarine/resources/etalons/patches/software_upgrade.yaml b/kubemarine/resources/etalons/patches/software_upgrade.yaml index cbd2824d4..f1fd27f60 100644 --- a/kubemarine/resources/etalons/patches/software_upgrade.yaml +++ b/kubemarine/resources/etalons/patches/software_upgrade.yaml @@ -35,3 +35,4 @@ plugins: local-path-provisioner: [] envoy-gateway: [] openstack-cinder-csi: [] + csi-snapshot-controller: [] diff --git a/kubemarine/resources/schemas/definitions/plugins.json b/kubemarine/resources/schemas/definitions/plugins.json index b19102a71..90d26f71f 100644 --- a/kubemarine/resources/schemas/definitions/plugins.json +++ b/kubemarine/resources/schemas/definitions/plugins.json @@ -17,6 +17,12 @@ }, "envoy-gateway": { "$ref": "plugins/envoy-gateway.json" + }, + "openstack-cinder-csi": { + "$ref": "plugins/openstack-cinder-csi.json" + }, + "csi-snapshot-controller": { + "$ref": "plugins/csi-snapshot-controller.json" } }, "additionalProperties": { diff --git a/kubemarine/resources/schemas/definitions/plugins/csi-snapshot-controller.json b/kubemarine/resources/schemas/definitions/plugins/csi-snapshot-controller.json new file mode 100644 index 000000000..4bffc18e7 --- /dev/null +++ b/kubemarine/resources/schemas/definitions/plugins/csi-snapshot-controller.json @@ -0,0 +1,37 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "type": "object", + "description": "Change the default settings for the 'csi-snapshot-controller' plugin", + "allOf": [{"$ref": "generic_plugin.json#/definitions/Properties"}], + "properties": { + "version": { + "type": "string", + "description": "Version of csi-snapshot-controller helm chart" + }, + "namespace": { + "type": "string", + "description": "Namespace where csi-snapshot-controller should be installed", + "default": "kube-system" + }, + "releaseName": { + "type": "string", + "description": "Release name for csi-snapshot-controller helm chart", + "default": "openstack-cinder-csi" + }, + "values": { + "type": "object", + "description": "Override values for csi-snapshot-controller helm chart" + }, + "additionalResources": { + "type": "string", + "description": "Multi-line string containing additional resources manifests to apply after plugin installation (e.g. CRs)", + "default": "" + } + }, + "propertyNames": { + "anyOf": [ + {"$ref": "generic_plugin.json#/definitions/PropertyNames"}, + {"enum": ["version", "namespace", "releaseName", "values", "additionalResources"]} + ] + } +} \ No newline at end of file diff --git a/test/unit/tools/thirdparties/test_sync.py b/test/unit/tools/thirdparties/test_sync.py index 5c2ad4f7b..f22b237b8 100644 --- a/test/unit/tools/thirdparties/test_sync.py +++ b/test/unit/tools/thirdparties/test_sync.py @@ -103,7 +103,7 @@ def test_compatibility_mapping_add_new_versions(self): requirements = tracker.get_changed_software_requirements() self.assertEqual(new_versions, list(requirements), "Unexpected Kubernetes versions were changed") expected_software = {'calico', 'nginx-ingress-controller', 'envoy-gateway', 'kubernetes-dashboard', - 'local-path-provisioner', 'containerd', 'crictl', 'openstack-cinder-csi'} + 'local-path-provisioner', 'containerd', 'crictl', 'openstack-cinder-csi', 'csi-snapshot-controller'} for ver in new_versions: self.assertEqual(expected_software, {name for name, _ in requirements[ver]}, From 8449947a150768b16d0805cf842d228e9c9e21b8 Mon Sep 17 00:00:00 2001 From: theboringstuff <39027092+theboringstuff@users.noreply.github.com> Date: Fri, 15 May 2026 12:24:32 +0500 Subject: [PATCH 2/4] change recommendation --- documentation/Installation.md | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/documentation/Installation.md b/documentation/Installation.md index f6e8b983a..6cb35c599 100644 --- a/documentation/Installation.md +++ b/documentation/Installation.md @@ -72,6 +72,7 @@ This section provides information about the inventory, features, and steps for i - [kubernetes-dashboard](#kubernetes-dashboard) - [local-path-provisioner](#local-path-provisioner) - [openstack-cinder-csi](#openstack-cinder-csi) + - [csi-snapshot-controller](#csi-snapshot-controller) - [Plugins Features](#plugins-features) - [plugin_defaults](#plugin_defaults) - [Plugins Reinstallation](#plugins-reinstallation) @@ -4554,17 +4555,25 @@ plugins: * Above configuration assumes that nodes have OpenStack credentials stored on all nodes under `/etc/config/cloud.conf` file. You need to put this file on nodes during nodes provisioning. * By default, this plugin uses `openstack-cinder-csi` namespace and helm release name. -If you are using an old OpenStack, above configuration will not work, because it installs new version of Cinder CSI plugin, which is not compatible with old OpenStack APIs. You may see issues like following: +If you are using an old OpenStack, above configuration may not work, because it installs new version of Cinder CSI plugin, which expects new OpenStack APIs. You may see issues like following: ``` E0428 10:09:13.203111 12 utils.go:99] [ID:36] GRPC error: rpc error: code = Internal desc = Failed to get volumes: Expected HTTP response code [200 204 300] when accessing [GET ...], but got 400 instead: {"badRequest": {"message": "Invalid filters name are found in query options.", "code": 400}} ``` -For old OpenStack, you should use older version of openstack-cinder-csi chart and older plugin image, like following: +To fix above issue with old OpenStack, recommended approach is to simply change your `cloud.conf` file adding below configuration: +```toml +[BlockStorage] +ignore-volume-microversion=true +``` + +This configuration tells openstack-cinder-csi plugin to work in a mode compatible with old OpenStack. With this change, new versions of openstack-cinder-csi will work with old OpenStack. + +Previously, for old OpenStack we recommeded to use old version of openstack-cinder-csi plugin. This approach is no longer recommeded, since `cloud.conf` option is better one. If you still want to use older version of openstack-cinder-csi plugin, you need to specify older version of openstack-cinder-csi chart and older plugin image, like following: ```yaml plugins: openstack-cinder-csi: install: true - version: 2.2.0 # changed for old OpenStack + version: 2.2.0 # changed values: storageClass: enabled: false @@ -4582,7 +4591,7 @@ plugins: allowVolumeExpansion: true parameters: type: $YOUR_OPENSTACK_VOLUME_TYPE_HERE - # changed for old OpenStack + # changed csi: plugin: image: @@ -4591,8 +4600,9 @@ plugins: ``` **Note**: +* Using old openstack-cinder-csi version is NOT recommended, please use approach with `cloud.conf` option. * If you use registry other than `docker.io`, you need to edit `csi.plugin.image.repository` in above configuration accordingly. -* For old OpenStack version, OpenStack credentials should be stored on nodes under `/etc/kubernetes/cloud.conf` +* For old openstack-cinder-csi version, OpenStack credentials should be stored on nodes under `/etc/kubernetes/cloud.conf` ##### csi-snapshot-controller From 422c512a2c2593e4f2c623556e10339f1ee1b601 Mon Sep 17 00:00:00 2001 From: theboringstuff <39027092+theboringstuff@users.noreply.github.com> Date: Fri, 15 May 2026 12:31:00 +0500 Subject: [PATCH 3/4] note that option is for old openstack only --- documentation/Installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/Installation.md b/documentation/Installation.md index 6cb35c599..046a40732 100644 --- a/documentation/Installation.md +++ b/documentation/Installation.md @@ -4566,7 +4566,7 @@ To fix above issue with old OpenStack, recommended approach is to simply change ignore-volume-microversion=true ``` -This configuration tells openstack-cinder-csi plugin to work in a mode compatible with old OpenStack. With this change, new versions of openstack-cinder-csi will work with old OpenStack. +This configuration tells openstack-cinder-csi plugin to work in a mode compatible with old OpenStack. With this change, new versions of openstack-cinder-csi will work with old OpenStack. DO NOT set this option for new OpenStack versions, only for old ones. Previously, for old OpenStack we recommeded to use old version of openstack-cinder-csi plugin. This approach is no longer recommeded, since `cloud.conf` option is better one. If you still want to use older version of openstack-cinder-csi plugin, you need to specify older version of openstack-cinder-csi chart and older plugin image, like following: ```yaml From 0566e2682d9a16c2c7595abcdbbd72698880b6b1 Mon Sep 17 00:00:00 2001 From: theboringstuff <39027092+theboringstuff@users.noreply.github.com> Date: Fri, 15 May 2026 12:45:15 +0500 Subject: [PATCH 4/4] csi-snapshot-controller docs --- documentation/Installation.md | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/documentation/Installation.md b/documentation/Installation.md index 046a40732..304cb7364 100644 --- a/documentation/Installation.md +++ b/documentation/Installation.md @@ -4606,7 +4606,36 @@ plugins: ##### csi-snapshot-controller -TODO +`csi-snapshot-controller` deploys a controller which is required for `VolumeSnapshot` functionality to work: +* https://github.com/kubernetes-csi/external-snapshotter#csi-snapshotter + +By default, the csi-snapshot-controller plugin is not installed, however, you can install it by enabling the plugin and providing required configuration, for example: +```yaml +plugins: + csi-snapshot-controller: + install: true + installation: + registry: $YOUR_K8S_REGISTRY_PROXY_HERE + additionalResources: | + --- + apiVersion: snapshot.storage.k8s.io/v1 + kind: VolumeSnapshotClass + metadata: + annotations: + snapshot.storage.kubernetes.io/is-default-class: "true" + labels: + velero.io/csi-volumesnapshot-class: "true" + name: cinder-csi-retain + driver: cinder.csi.openstack.org + deletionPolicy: Retain + parameters: + force-create: "true" +``` + +**Note**: +* Above configuration also creates default `VolumeSnapshotClass` for openstack cinder driver. +* `csi-snapshot-controller` only works with recent versions of `openstack-cinder-csi` plugin, because old version contains a bug: + * https://github.com/kubernetes/cloud-provider-openstack/issues/1812 #### Plugins Features