Implement KEP3751 ("ControllerModifyVolume")

Makefile

@@ -118,7 +118,7 @@ e2e/single-az: bin/helm bin/ginkgo
 	TEST_PATH=./tests/e2e/... \
 	GINKGO_FOCUS="\[ebs-csi-e2e\] \[single-az\]" \
 	GINKGO_PARALLEL=5 \
-	HELM_EXTRA_FLAGS="--set=controller.volumeModificationFeature.enabled=true" \
+	HELM_EXTRA_FLAGS="--set=controller.volumeModificationFeature.enabled=true,sidecars.provisioner.additionalArgs[0]='--feature-gates=VolumeAttributesClass=true',sidecars.resizer.additionalArgs[0]='--feature-gates=VolumeAttributesClass=true'" \
 	./hack/e2e/run.sh
 
 .PHONY: e2e/multi-az

charts/aws-ebs-csi-driver/templates/clusterrole-provisioner.yaml

@@ -33,6 +33,9 @@ rules:
   - apiGroups: [ "storage.k8s.io" ]
     resources: [ "volumeattachments" ]
     verbs: [ "get", "list", "watch" ]
+  - apiGroups: [ "storage.k8s.io" ]
+    resources: [ "volumeattributesclasses" ]
+    verbs: [ "get" ]
   {{- with .Values.sidecars.provisioner.additionalClusterRoleRules }}
     {{- . | toYaml | nindent 2 }}
   {{- end }}

charts/aws-ebs-csi-driver/templates/clusterrole-resizer.yaml

@@ -29,6 +29,9 @@ rules:
   - apiGroups: [ "" ]
     resources: [ "pods" ]
     verbs: [ "get", "list", "watch" ]
+  - apiGroups: [ "storage.k8s.io" ]
+    resources: [ "volumeattributesclasses" ]
+    verbs: [ "get", "list", "watch" ]
   {{- with .Values.sidecars.resizer.additionalClusterRoleRules }}
     {{- . | toYaml | nindent 2 }}
   {{- end }}

deploy/kubernetes/base/clusterrole-provisioner.yaml

@@ -34,3 +34,6 @@ rules:
   - apiGroups: [ "storage.k8s.io" ]
     resources: [ "volumeattachments" ]
     verbs: [ "get", "list", "watch" ]
+  - apiGroups: [ "storage.k8s.io" ]
+    resources: [ "volumeattributesclasses" ]
+    verbs: [ "get" ]

deploy/kubernetes/base/clusterrole-resizer.yaml

@@ -30,3 +30,6 @@ rules:
   - apiGroups: [ "" ]
     resources: [ "pods" ]
     verbs: [ "get", "list", "watch" ]
+  - apiGroups: [ "storage.k8s.io" ]
+    resources: [ "volumeattributesclasses" ]
+    verbs: [ "get", "list", "watch" ]

docs/modify-volume.md

@@ -1,21 +1,37 @@
 # Volume Modification
 
-The EBS CSI Driver (starting from v1.19.0) supports volume modification through PVC annotations. This allows users to modify volume properties (e.g., volume type, IOPS, and throughput). 
+The EBS CSI Driver (starting from v1.19.0) supports volume modification through two methods:
+- Via the standardized CSI RPC `ControllerModifyVolume` (on Kubernetes, this is done via [`VolumeAttributesClass`](https://github.com/awslabs/volume-modifier-for-k8s))
+- Volume annotations via [`volume-modifier-for-k8s`](https://github.com/awslabs/volume-modifier-for-k8s)
 
 ## Installation
-This feature is opt-in. 
 
-To install this feature through the Helm chart, users must set `controller.volumeModificationFeature.enabled` in `values.yaml` to `true`.
+### `ControllerModifyVolume` via `VolumeAttributesClass` (Recommended)
+
+`VolumeAttributesClass` support is controlled by the Kubernetes `VolumeAttributesClass` [feature gate](https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/).
+
+To use this feature, it must be enabled in the following places:
+- `VolumeAttributesClass` feature gate on `kube-apiserver` (consult your Kubernetes distro's documentation)
+- `storage.k8s.io/v1alpha1` enabled in `kube-apiserver` via [`runtime-config`](https://kubernetes.io/docs/tasks/administer-cluster/enable-disable-api/) (consult your Kubernetes distro's documentation)
+- `VolumeAttributesClass` feature gate on `kube-controller-manager` (consult your Kubernetes distro's documentation)
+- `VolumeAttributesClass` feature gate on `external-provisioner` (add `--feature-gates=VolumeAttributesClass=true` to `sidecars.provisioner.additionalArgs` when using the EBS CSI Helm chart)
+- `VolumeAttributesClass` feature gate on `kube-controller-manager` (add `--feature-gates=VolumeAttributesClass=true` to `sidecars.resizer.additionalArgs` when using the EBS CSI Helm chart)
+
+For more information, see the [Kubernetes documentation for the feature](https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/).
+
+### `volume-modifier-for-k8s`
+
+To enable this feature through the Helm chart, users must set `controller.volumeModificationFeature.enabled` in `values.yaml` to `true`.
 
 This will install an additional sidecar (`volumemodifier`) that watches the Kubernetes API server for changes to PVC annotations and triggers an RPC call against the CSI driver.
 
-## Usage
+## Parameters
 
-Users can specify the following PVC annotations:
+Users can specify the following modification parameters:
 
-- `ebs.csi.aws.com/volumeType`: to update the volume type
-- `ebs.csi.aws.com/iops`: to update the IOPS
-- `ebs.csi.aws.com/throughput`: to update the throughput
+- `type`: to update the volume type
+- `iops`: to update the IOPS
+- `throughput`: to update the throughput
 
 ## Considerations
 
@@ -24,6 +40,12 @@ Users can specify the following PVC annotations:
 
 ## Example
 
+### `ControllerModifyVolume` via `VolumeAttributesClass`
+
+See the [EBS CSI example with manifests](../examples/kubernetes/modify-volume).
+
+### `volume-modifier-for-k8s`
+
 #### 1) Create a PVC.
 
 

examples/kubernetes/modify-volume/README.md

@@ -0,0 +1,59 @@
+# Volume Modification via `VolumeAttributesClass`
+
+## Prerequisites
+
+This example will only work on a cluster with the `VolumeAttributesClass` feature enabled. For more information see the [installation instructions in the EBS CSI `ModifyVolume` documentation](../docs/modify-volume.md).
+
+## Usage
+
+1. Deploy the example `Pod`, `PersistentVolumeClaim`, and `StorageClass` to your cluster
+    ```sh
+    $ kubectl apply -f manifests/pod-with-volume.yaml
+
+    storageclass.storage.k8s.io/ebs-sc created
+    persistentvolumeclaim/ebs-claim created
+    pod/app created
+    ```
+
+2. Wait for the `PersistentVolumeClaim` to bind and the pod to reach the `Running` state
+    ```sh
+    $ kubectl get pvc ebs-claim
+    NAME        STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   VOLUMEATTRIBUTESCLASS   AGEebs-claim   Bound    pvc-076b2d14-b643-47d4-a2ce-fbf9cd36572b   100Gi      RWO            ebs-sc         <unset>                 2m51s
+
+    $ kubectl get pod app
+    NAME   READY   STATUS    RESTARTS   AGE
+    app    1/1     Running   0          3m24
+    ```
+
+3. Watch the logs of the pod
+    ```sh
+    $ kubectl logs -f app
+    Mon Feb 26 22:28:19 UTC 2024
+    Mon Feb 26 22:28:24 UTC 2024
+    Mon Feb 26 22:28:29 UTC 2024
+    Mon Feb 26 22:28:34 UTC 2024
+    Mon Feb 26 22:28:39 UTC 2024
+    ...
+    ```
+
+4. Simultaneously, deploy the `VolumeAttributesClass` and edit the `PersistentVolumeClaim` to point to this class
+    ```sh
+    $ kubectl patch pvc ebs-claim --patch '{"spec": {"volumeAttributesClassName": "io2-class"}}'
+    persistentvolumeclaim/ebs-claim patched
+    ```
+
+5. Wait for the `VolumeAttributesClass` to apply to the volume
+    ```sh
+    $ kubectl get pvc ebs-claim
+    NAME        STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   VOLUMEATTRIBUTESCLASS   AGE
+    ebs-claim   Bound    pvc-076b2d14-b643-47d4-a2ce-fbf9cd36572b   100Gi      RWO            ebs-sc         io2-class               5m54s
+    ```
+
+6. (Optional) Delete example resources
+    ```sh
+    $ kubectl delete -f manifests 
+    storageclass.storage.k8s.io "ebs-sc" deleted
+    persistentvolumeclaim "ebs-claim" deleted
+    pod "app" deleted
+    volumeattributesclass.storage.k8s.io "io2-class" deleted
+    ```

examples/kubernetes/modify-volume/manifests/pod-with-volume.yaml

@@ -0,0 +1,39 @@
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: ebs-sc
+provisioner: ebs.csi.aws.com
+volumeBindingMode: WaitForFirstConsumer
+parameters:
+  type: gp3
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: ebs-claim
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: ebs-sc
+  resources:
+    requests:
+      storage: 100Gi
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: app
+spec:
+  containers:
+  - name: app
+    image: centos
+    command: ["/bin/sh"]
+    args: ["-c", "while true; do echo $(date -u) | tee /data/out.txt; sleep 5; done"]
+    volumeMounts:
+    - name: persistent-storage
+      mountPath: /data
+  volumes:
+  - name: persistent-storage
+    persistentVolumeClaim:
+      claimName: ebs-claim

examples/kubernetes/modify-volume/manifests/volumeattributesclass.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: storage.k8s.io/v1alpha1
+kind: VolumeAttributesClass
+metadata:
+  name: io2-class
+driverName: ebs.csi.aws.com
+parameters:
+  type: io2
+  iops: "10000"

hack/e2e/kops/patch-cluster.yaml

@@ -1,4 +1,12 @@
 spec:
+  kubeAPIServer:
+    featureGates:
+      VolumeAttributesClass: "true"
+    runtimeConfig:
+      storage.k8s.io/v1alpha1: "true"
+  kubeControllerManager:
+    featureGates:
+      VolumeAttributesClass: "true"
   additionalPolicies:
     node: |
       [

pkg/driver/controller.go

@@ -51,6 +51,7 @@ var (
 		csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT,
 		csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS,
 		csi.ControllerServiceCapability_RPC_EXPAND_VOLUME,
+		csi.ControllerServiceCapability_RPC_MODIFY_VOLUME,
 	}
 )
 
@@ -234,6 +235,23 @@ func (d *controllerService) CreateVolume(ctx context.Context, req *csi.CreateVol
 		}
 	}
 
+	modifyOptions, err := parseModifyVolumeParameters(req.GetMutableParameters())
+	if err != nil {
+		return nil, status.Errorf(codes.InvalidArgument, "Invalid mutable parameter: %v", err)
+	}
+
+	// "Values specified in mutable_parameters MUST take precedence over the values from parameters."
+	// https://github.com/container-storage-interface/spec/blob/master/spec.md#createvolume
+	if modifyOptions.VolumeType != "" {
+		volumeType = modifyOptions.VolumeType
+	}
+	if modifyOptions.IOPS != 0 {
+		iops = modifyOptions.IOPS
+	}
+	if modifyOptions.Throughput != 0 {
+		throughput = modifyOptions.Throughput
+	}
+
 	responseCtx := map[string]string{}
 
 	if len(blockSize) > 0 {
@@ -602,7 +620,23 @@ func (d *controllerService) ControllerExpandVolume(ctx context.Context, req *csi
 
 func (d *controllerService) ControllerModifyVolume(ctx context.Context, req *csi.ControllerModifyVolumeRequest) (*csi.ControllerModifyVolumeResponse, error) {
 	klog.V(4).InfoS("ControllerModifyVolume: called", "args", *req)
-	return nil, status.Error(codes.Unimplemented, "")
+
+	volumeID := req.GetVolumeId()
+	if len(volumeID) == 0 {
+		return nil, status.Error(codes.InvalidArgument, "Volume ID not provided")
+	}
+
+	options, err := parseModifyVolumeParameters(req.GetMutableParameters())
+	if err != nil {
+		return nil, err
+	}
+
+	err = d.modifyVolumeWithCoalescing(ctx, volumeID, options)
+	if err != nil {
+		return nil, err
+	}
+
+	return &csi.ControllerModifyVolumeResponse{}, nil
 }
 
 func (d *controllerService) ControllerGetVolume(ctx context.Context, req *csi.ControllerGetVolumeRequest) (*csi.ControllerGetVolumeResponse, error) {