Skip to content

Commit

Permalink
Merge pull request #1846 from AndrewSirenko/E2EModifyVolumePostRefactor
Browse files Browse the repository at this point in the history
Add E2E tests for modifying volumes via annotations
  • Loading branch information
k8s-ci-robot authored Dec 5, 2023
2 parents 551b59a + de85e64 commit cb20546
Show file tree
Hide file tree
Showing 6 changed files with 334 additions and 59 deletions.
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@ test-sanity:
test-e2e-single-az:
AWS_REGION=us-west-2 \
AWS_AVAILABILITY_ZONES=us-west-2a \
HELM_EXTRA_FLAGS='--set=controller.k8sTagClusterId=$$CLUSTER_NAME' \
HELM_EXTRA_FLAGS='--set=controller.k8sTagClusterId=$$CLUSTER_NAME,controller.volumeModificationFeature.enabled=true' \
EBS_INSTALL_SNAPSHOT="true" \
TEST_PATH=./tests/e2e/... \
GINKGO_FOCUS="\[ebs-csi-e2e\] \[single-az\]" \
Expand Down
131 changes: 131 additions & 0 deletions tests/e2e/modify_volume.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
awscloud "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/cloud"
ebscsidriver "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/driver"
"github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/driver"
"github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/testsuites"
. "github.com/onsi/ginkgo/v2"
v1 "k8s.io/api/core/v1"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/test/e2e/framework"
admissionapi "k8s.io/pod-security-admission/api"
)

var (
defaultModifyVolumeTestGp3CreateVolumeParameters = map[string]string{
ebscsidriver.VolumeTypeKey: awscloud.VolumeTypeGP3,
ebscsidriver.FSTypeKey: ebscsidriver.FSTypeExt4,
}
)

var (
modifyVolumeTests = map[string]testsuites.ModifyVolumeTest{
"with a new iops annotation": {
CreateVolumeParameters: defaultModifyVolumeTestGp3CreateVolumeParameters,
ModifyVolumeAnnotations: map[string]string{
testsuites.AnnotationIops: "4000",
},
ShouldResizeVolume: false,
ShouldTestInvalidModificationRecovery: false,
},
"with a new io2 volumeType annotation": {
CreateVolumeParameters: defaultModifyVolumeTestGp3CreateVolumeParameters,
ModifyVolumeAnnotations: map[string]string{
testsuites.AnnotationVolumeType: awscloud.VolumeTypeIO2,
testsuites.AnnotationIops: testsuites.DefaultIopsIoVolumes, // As of aws-ebs-csi-driver v1.25.0, parameter iops must be re-specified when modifying volumeType io2 volumes.
},
ShouldResizeVolume: false,
ShouldTestInvalidModificationRecovery: false,
},
"with a new throughput annotation": {
CreateVolumeParameters: defaultModifyVolumeTestGp3CreateVolumeParameters,
ModifyVolumeAnnotations: map[string]string{
testsuites.AnnotationThroughput: "150",
},
ShouldResizeVolume: false,
ShouldTestInvalidModificationRecovery: false,
},
"with new throughput and iops annotations": {
CreateVolumeParameters: defaultModifyVolumeTestGp3CreateVolumeParameters,
ModifyVolumeAnnotations: map[string]string{
testsuites.AnnotationIops: "4000",
testsuites.AnnotationThroughput: "150",
},
ShouldResizeVolume: false,
ShouldTestInvalidModificationRecovery: false,
},
"with a larger size and new throughput and iops annotations": {
CreateVolumeParameters: defaultModifyVolumeTestGp3CreateVolumeParameters,
ModifyVolumeAnnotations: map[string]string{
testsuites.AnnotationIops: "4000",
testsuites.AnnotationThroughput: "150",
},
ShouldResizeVolume: true,
ShouldTestInvalidModificationRecovery: false,
},
"with a larger size and new throughput and iops annotations after providing an invalid annotation": {
CreateVolumeParameters: defaultModifyVolumeTestGp3CreateVolumeParameters,
ModifyVolumeAnnotations: map[string]string{
testsuites.AnnotationIops: "4000",
testsuites.AnnotationThroughput: "150",
},
ShouldResizeVolume: true,
ShouldTestInvalidModificationRecovery: true,
},
"from io2 to gp3 with larger size and new iops and throughput annotations": {
CreateVolumeParameters: map[string]string{
ebscsidriver.VolumeTypeKey: awscloud.VolumeTypeIO2,
ebscsidriver.FSTypeKey: ebscsidriver.FSTypeExt4,
ebscsidriver.IopsKey: testsuites.DefaultIopsIoVolumes,
},
ModifyVolumeAnnotations: map[string]string{
testsuites.AnnotationVolumeType: awscloud.VolumeTypeGP3,
testsuites.AnnotationIops: "4000",
testsuites.AnnotationThroughput: "150",
},
ShouldResizeVolume: true,
ShouldTestInvalidModificationRecovery: false,
},
}
)

var _ = Describe("[ebs-csi-e2e] [single-az] [modify-volume] Modifying a PVC", func() {
f := framework.NewDefaultFramework("ebs")
f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged

var (
cs clientset.Interface
ns *v1.Namespace
ebsDriver driver.PVTestDriver
)

BeforeEach(func() {
cs = f.ClientSet
ns = f.Namespace
ebsDriver = driver.InitEbsCSIDriver()
})

for testName, modifyVolumeTest := range modifyVolumeTests {
modifyVolumeTest := modifyVolumeTest
Context(testName, func() {
It("will modify associated PV and EBS Volume", func() {
modifyVolumeTest.Run(cs, ns, ebsDriver)
})
})
}
})
Original file line number Diff line number Diff line change
Expand Up @@ -15,18 +15,9 @@ limitations under the License.
package testsuites

import (
"context"
"fmt"
"time"

"github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/util"
"github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/driver"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/kubernetes/test/e2e/framework"

. "github.com/onsi/ginkgo/v2"
v1 "k8s.io/api/core/v1"
clientset "k8s.io/client-go/kubernetes"
)

Expand All @@ -45,9 +36,10 @@ func (t *DynamicallyProvisionedResizeVolumeTest) Run(client clientset.Interface,
tpvc, _ := volume.SetupDynamicPersistentVolumeClaim(client, namespace, t.CSIDriver)
defer tpvc.Cleanup()

ResizeTestPvc(client, namespace, tpvc, 1)
By("resizing the volume")
ResizeTestPvc(client, namespace, tpvc, DefaultSizeIncreaseGi)

By("Validate volume can be attached")
By("validate volume can be attached")
tpod := NewTestPod(client, namespace, t.Pod.Cmd)

tpod.SetupVolume(tpvc.persistentVolumeClaim, volume.VolumeMount.NameGenerate+"1", volume.VolumeMount.MountPathGenerate+"1", volume.VolumeMount.ReadOnly)
Expand All @@ -58,44 +50,4 @@ func (t *DynamicallyProvisionedResizeVolumeTest) Run(client clientset.Interface,
tpod.WaitForSuccess()

defer tpod.Cleanup()

}

// WaitForPvToResize waiting for pvc size to be resized to desired size
func WaitForPvToResize(c clientset.Interface, ns *v1.Namespace, pvName string, desiredSize resource.Quantity, timeout time.Duration, interval time.Duration) error {
By(fmt.Sprintf("Waiting up to %v for pv in namespace %q to be complete", timeout, ns.Name))
for start := time.Now(); time.Since(start) < timeout; time.Sleep(interval) {
newPv, _ := c.CoreV1().PersistentVolumes().Get(context.TODO(), pvName, metav1.GetOptions{})
newPvSize := newPv.Spec.Capacity["storage"]
if desiredSize.Equal(newPvSize) {
By(fmt.Sprintf("Pv size is updated to %v", newPvSize.String()))
return nil
}
}
return fmt.Errorf("Gave up after waiting %v for pv %q to complete resizing", timeout, pvName)
}

// ResizeTestPvc increases size of given persistent volume claim by `sizeIncreaseGi` Gigabytes
func ResizeTestPvc(client clientset.Interface, namespace *v1.Namespace, testPvc *TestPersistentVolumeClaim, sizeIncreaseGi int64) (updatedPvc *v1.PersistentVolumeClaim, updatedSize resource.Quantity) {
By(fmt.Sprintf("getting pvc name: %v", testPvc.persistentVolumeClaim.Name))
pvc, _ := client.CoreV1().PersistentVolumeClaims(namespace.Name).Get(context.TODO(), testPvc.persistentVolumeClaim.Name, metav1.GetOptions{})

originalSize := pvc.Spec.Resources.Requests["storage"]
delta := resource.Quantity{}
delta.Set(util.GiBToBytes(sizeIncreaseGi))
originalSize.Add(delta)
pvc.Spec.Resources.Requests["storage"] = originalSize

By("resizing the pvc")
updatedPvc, err := client.CoreV1().PersistentVolumeClaims(namespace.Name).Update(context.TODO(), pvc, metav1.UpdateOptions{})
if err != nil {
framework.ExpectNoError(err, fmt.Sprintf("fail to resize pvc(%s): %v", pvc.Name, err))
}
updatedSize = updatedPvc.Spec.Resources.Requests["storage"]

By("checking the resizing PV result")
err = WaitForPvToResize(client, namespace, updatedPvc.Spec.VolumeName, updatedSize, 1*time.Minute, 5*time.Second)
framework.ExpectNoError(err)

return
}
104 changes: 103 additions & 1 deletion tests/e2e/testsuites/e2e_utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,19 +14,121 @@ limitations under the License.

package testsuites

import "fmt"
import (
"context"
"fmt"
"github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/util"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/test/e2e/framework"
"time"
)

const (
DefaultVolumeName = "test-volume-1"
DefaultMountPath = "/mnt/default-mount"

DefaultIopsIoVolumes = "100"

DefaultSizeIncreaseGi = 1

DefaultModificationTimeout = 3 * time.Minute
DefaultResizeTimout = 1 * time.Minute
DefaultK8sApiPollingInterval = 5 * time.Second

AnnotationIops = "ebs.csi.aws.com/iops"
AnnotationThroughput = "ebs.csi.aws.com/throughput"
AnnotationVolumeType = "ebs.csi.aws.com/volumeType"
)

// PodCmdWriteToVolume returns pod command that would write to mounted volume
func PodCmdWriteToVolume(volumeMountPath string) string {
return fmt.Sprintf("echo 'hello world' >> %s/data && grep 'hello world' %s/data && sync", volumeMountPath, volumeMountPath)
}

// PodCmdContinuousWrite returns pod command that would continuously write to mounted volume
func PodCmdContinuousWrite(volumeMountPath string) string {
return fmt.Sprintf("while true; do echo \"$(date -u)\" >> /%s/out.txt; sleep 5; done", volumeMountPath)
}

// IncreasePvcObjectStorage increases `storage` of a K8s PVC object by specified Gigabytes
func IncreasePvcObjectStorage(pvc *v1.PersistentVolumeClaim, sizeIncreaseGi int64) resource.Quantity {
pvcSize := pvc.Spec.Resources.Requests["storage"]
delta := resource.Quantity{}
delta.Set(util.GiBToBytes(sizeIncreaseGi))
pvcSize.Add(delta)
pvc.Spec.Resources.Requests["storage"] = pvcSize
return pvcSize
}

// WaitForPvToResize waiting for pvc size to be resized to desired size
func WaitForPvToResize(c clientset.Interface, ns *v1.Namespace, pvName string, desiredSize resource.Quantity, timeout time.Duration, interval time.Duration) error {
framework.Logf("waiting up to %v for pv resize in namespace %q to be complete", timeout, ns.Name)
for start := time.Now(); time.Since(start) < timeout; time.Sleep(interval) {
newPv, _ := c.CoreV1().PersistentVolumes().Get(context.TODO(), pvName, metav1.GetOptions{})
newPvSize := newPv.Spec.Capacity["storage"]
if desiredSize.Equal(newPvSize) {
framework.Logf("pv size is updated to %v", newPvSize.String())
return nil
}
}
return fmt.Errorf("gave up after waiting %v for pv %q to complete resizing", timeout, pvName)
}

// ResizeTestPvc increases size of given `TestPersistentVolumeClaim` by specified Gigabytes
func ResizeTestPvc(client clientset.Interface, namespace *v1.Namespace, testPvc *TestPersistentVolumeClaim, sizeIncreaseGi int64) (updatedSize resource.Quantity) {
framework.Logf("getting pvc name: %v", testPvc.persistentVolumeClaim.Name)
pvc, _ := client.CoreV1().PersistentVolumeClaims(namespace.Name).Get(context.TODO(), testPvc.persistentVolumeClaim.Name, metav1.GetOptions{})

IncreasePvcObjectStorage(pvc, sizeIncreaseGi)

framework.Logf("updating the pvc object")
updatedPvc, err := client.CoreV1().PersistentVolumeClaims(namespace.Name).Update(context.TODO(), pvc, metav1.UpdateOptions{})
if err != nil {
framework.ExpectNoError(err, fmt.Sprintf("fail to resize pvc(%s): %v", pvc.Name, err))
}
updatedSize = updatedPvc.Spec.Resources.Requests["storage"]

framework.Logf("checking the resizing PV result")
err = WaitForPvToResize(client, namespace, updatedPvc.Spec.VolumeName, updatedSize, DefaultResizeTimout, DefaultK8sApiPollingInterval)
framework.ExpectNoError(err)
return updatedSize
}

// AnnotatePvc annotates supplied k8s pvc object with supplied annotations
func AnnotatePvc(pvc *v1.PersistentVolumeClaim, annotations map[string]string) {
for annotation, value := range annotations {
pvc.Annotations[annotation] = value
}
}

// CheckPvAnnotations checks whether supplied k8s pv object contains supplied annotations
func CheckPvAnnotations(pv *v1.PersistentVolume, annotations map[string]string) bool {
for annotation, value := range annotations {
if pv.Annotations[annotation] != value {
return false
}
}
return true
}

// WaitForPvToModify waiting for PV to be modified
func WaitForPvToModify(c clientset.Interface, ns *v1.Namespace, pvName string, expectedAnnotations map[string]string, timeout time.Duration, interval time.Duration) error {
framework.Logf("waiting up to %v for pv in namespace %q to be modified", timeout, ns.Name)

for start := time.Now(); time.Since(start) < timeout; time.Sleep(interval) {
modifyingPv, _ := c.CoreV1().PersistentVolumes().Get(context.TODO(), pvName, metav1.GetOptions{})

if CheckPvAnnotations(modifyingPv, expectedAnnotations) {
framework.Logf("pv annotations are updated to %v", modifyingPv.Annotations)
return nil
}
}
return fmt.Errorf("gave up after waiting %v for pv %q to complete modifying", timeout, pvName)
}

func CreateVolumeDetails(createVolumeParameters map[string]string, volumeSize string) *VolumeDetails {
allowVolumeExpansion := true

Expand Down
6 changes: 1 addition & 5 deletions tests/e2e/testsuites/format_options_tester.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,6 @@ type FormatOptionTest struct {
CreateVolumeParameters map[string]string
}

const (
volumeSizeIncreaseAmtGi = 1
)

func (t *FormatOptionTest) Run(client clientset.Interface, namespace *v1.Namespace, ebsDriver driver.PVTestDriver) {
By("setting up pvc with custom format option")
volumeDetails := CreateVolumeDetails(t.CreateVolumeParameters, driver.MinimumSizeForVolumeType(t.CreateVolumeParameters[ebscsidriver.VolumeTypeKey]))
Expand All @@ -44,7 +40,7 @@ func (t *FormatOptionTest) Run(client clientset.Interface, namespace *v1.Namespa
formatOptionMountPod.WaitForSuccess()

By("testing that pvc is able to be resized")
ResizeTestPvc(client, namespace, testPvc, volumeSizeIncreaseAmtGi)
ResizeTestPvc(client, namespace, testPvc, DefaultSizeIncreaseGi)

By("validating resized pvc by deploying new pod")
resizeTestPod := createPodWithVolume(client, namespace, PodCmdWriteToVolume(DefaultMountPath), testPvc, volumeDetails)
Expand Down
Loading

0 comments on commit cb20546

Please sign in to comment.