Skip uninstall and resource cleanup when fail-fast is enabled.

Bump v1.13 and 1.14 plugin versions for E2E test.

Signed-off-by: Xun Jiang <[email protected]>

test/Makefile

@@ -97,7 +97,6 @@ ADDITIONAL_BSL_PREFIX ?=
 ADDITIONAL_BSL_CONFIG ?=
 
 FEATURES ?=
-DEBUG_E2E_TEST ?= false
 DEBUG_VELERO_POD_RESTART ?= false
 VELERO_SERVER_DEBUG_MODE ?= false
 
@@ -148,10 +147,10 @@ COMMON_ARGS := --velerocli=$(VELERO_CLI) \
 	--features=$(FEATURES) \
 	--install-velero=$(INSTALL_VELERO) \
 	--registry-credential-file=$(REGISTRY_CREDENTIAL_FILE) \
-	--debug-e2e-test=$(DEBUG_E2E_TEST) \
 	--velero-server-debug-mode=$(VELERO_SERVER_DEBUG_MODE) \
 	--uploader-type=$(UPLOADER_TYPE) \
-	--debug-velero-pod-restart=$(DEBUG_VELERO_POD_RESTART)
+	--debug-velero-pod-restart=$(DEBUG_VELERO_POD_RESTART) \
+	--fail-fast=$(FAIL_FAST)
 
 # Make sure ginkgo is in $GOBIN
 .PHONY:ginkgo

test/e2e/backup/backup.go

@@ -97,7 +97,7 @@ func BackupRestoreTest(backupRestoreTestConfig BackupRestoreTestConfig) {
 	})
 
 	AfterEach(func() {
-		if !veleroCfg.Debug {
+		if !CurrentSpecReport().Failed() || !veleroCfg.FailFast {
 			By("Clean backups after test", func() {
 				DeleteAllBackups(context.Background(), &veleroCfg)
 				if backupRestoreTestConfig.isRetainPVTest {
@@ -111,6 +111,8 @@ func BackupRestoreTest(backupRestoreTestConfig BackupRestoreTestConfig) {
 				err = VeleroUninstall(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)
 				Expect(err).To(Succeed())
 			}
+		} else {
+			fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.")
 		}
 	})
 

test/e2e/backups/deletion.go

@@ -62,10 +62,12 @@ func backup_deletion_test(useVolumeSnapshots bool) {
 	})
 
 	AfterEach(func() {
-		if !veleroCfg.Debug {
+		if !CurrentSpecReport().Failed() || !veleroCfg.FailFast {
 			By("Clean backups after test", func() {
 				DeleteAllBackups(context.Background(), &veleroCfg)
 			})
+		} else {
+			fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.")
 		}
 	})
 
@@ -104,13 +106,16 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupLoc
 		if err := CreateNamespace(oneHourTimeout, client, ns); err != nil {
 			return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", ns)
 		}
-		if !veleroCfg.Debug {
+		if !CurrentSpecReport().Failed() || !veleroCfg.FailFast {
 			defer func() {
 				if err := DeleteNamespace(context.Background(), client, ns, true); err != nil {
 					fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", ns))
 				}
 			}()
+		} else {
+			fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.")
 		}
+
 		if err := KibishiiPrepareBeforeBackup(oneHourTimeout, client, providerName, ns,
 			registryCredentialFile, veleroFeatures, kibishiiDirectory, useVolumeSnapshots, DefaultKibishiiData); err != nil {
 			return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", ns)

test/e2e/backups/sync_backups.go

@@ -63,7 +63,7 @@ func BackupsSyncTest() {
 	})
 
 	AfterEach(func() {
-		if !veleroCfg.Debug {
+		if !CurrentSpecReport().Failed() || !veleroCfg.FailFast {
 			By("Clean backups after test", func() {
 				DeleteAllBackups(context.Background(), &veleroCfg)
 			})
@@ -72,6 +72,8 @@ func BackupsSyncTest() {
 				defer ctxCancel()
 				Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).To(Succeed())
 			}
+		} else {
+			fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.")
 		}
 	})
 
@@ -82,10 +84,13 @@ func BackupsSyncTest() {
 		By(fmt.Sprintf("Prepare workload as target to backup by creating namespace %s namespace", test.testNS))
 		Expect(CreateNamespace(ctx, *veleroCfg.ClientToInstallVelero, test.testNS)).To(Succeed(),
 			fmt.Sprintf("Failed to create %s namespace", test.testNS))
-		if !veleroCfg.Debug {
+
+		if !CurrentSpecReport().Failed() || !veleroCfg.FailFast {
 			defer func() {
 				Expect(DeleteNamespace(ctx, *veleroCfg.ClientToInstallVelero, test.testNS, false)).To(Succeed(), fmt.Sprintf("Failed to delete the namespace %s", test.testNS))
 			}()
+		} else {
+			fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.")
 		}
 
 		var BackupCfg BackupConfig
@@ -125,12 +130,16 @@ func BackupsSyncTest() {
 			Expect(CreateNamespace(ctx, *veleroCfg.ClientToInstallVelero, test.testNS)).To(Succeed(),
 				fmt.Sprintf("Failed to create %s namespace", test.testNS))
 		})
-		if !veleroCfg.Debug {
+
+		if !CurrentSpecReport().Failed() || !veleroCfg.FailFast {
 			defer func() {
 				Expect(DeleteNamespace(ctx, *veleroCfg.ClientToInstallVelero, test.testNS, false)).To(Succeed(),
 					fmt.Sprintf("Failed to delete the namespace %s", test.testNS))
 			}()
+		} else {
+			fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.")
 		}
+
 		var BackupCfg BackupConfig
 		BackupCfg.BackupName = test.backupName
 		BackupCfg.Namespace = test.testNS

test/e2e/backups/ttl.go

@@ -74,7 +74,7 @@ func TTLTest() {
 
 	AfterEach(func() {
 		veleroCfg.GCFrequency = ""
-		if !veleroCfg.Debug {
+		if !CurrentSpecReport().Failed() || !veleroCfg.FailFast {
 			By("Clean backups after test", func() {
 				DeleteAllBackups(context.Background(), &veleroCfg)
 			})
@@ -84,6 +84,8 @@ func TTLTest() {
 				Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).To(Succeed())
 			}
 			Expect(DeleteNamespace(ctx, client, test.testNS, false)).To(Succeed(), fmt.Sprintf("Failed to delete the namespace %s", test.testNS))
+		} else {
+			fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.")
 		}
 	})
 

test/e2e/basic/api-group/enable_api_group_extentions.go

@@ -67,7 +67,12 @@ func APIExtensionsVersionsTest() {
 		})
 	})
 	AfterEach(func() {
-		if !veleroCfg.Debug {
+		By(fmt.Sprintf("Switch to default kubeconfig context %s", veleroCfg.DefaultClusterContext), func() {
+			Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed())
+			veleroCfg.ClientToInstallVelero = veleroCfg.DefaultClient
+		})
+
+		if !CurrentSpecReport().Failed() || !veleroCfg.FailFast {
 			By("Clean backups after test", func() {
 				DeleteAllBackups(context.Background(), &veleroCfg)
 			})
@@ -86,10 +91,8 @@ func APIExtensionsVersionsTest() {
 					Expect(DeleteCRDByName(context.Background(), crdName)).To(Succeed())
 				})
 			}
-			By(fmt.Sprintf("Switch to default kubeconfig context %s", veleroCfg.DefaultClusterContext), func() {
-				Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed())
-				veleroCfg.ClientToInstallVelero = veleroCfg.DefaultClient
-			})
+		} else {
+			fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.")
 		}
 	})
 	Context("When EnableAPIGroupVersions flag is set", func() {

test/e2e/basic/api-group/enable_api_group_versions.go

@@ -79,7 +79,7 @@ func APIGroupVersionsTest() {
 	})
 
 	AfterEach(func() {
-		if !veleroCfg.Debug {
+		if !CurrentSpecReport().Failed() || !veleroCfg.FailFast {
 			for i := 0; i < testCaseNum; i++ {
 				curResource := fmt.Sprintf("rockband%ds", i)
 				curGroup := fmt.Sprintf("%s.%d", group, i)
@@ -101,6 +101,8 @@ func APIGroupVersionsTest() {
 					Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).NotTo(HaveOccurred())
 				})
 			}
+		} else {
+			fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.")
 		}
 	})
 

test/e2e/basic/namespace-mapping.go

@@ -116,7 +116,7 @@ func (n *NamespaceMapping) Verify() error {
 }
 
 func (n *NamespaceMapping) Clean() error {
-	if !n.VeleroCfg.Debug {
+	if !CurrentSpecReport().Failed() || !n.VeleroCfg.FailFast {
 		if err := DeleteStorageClass(context.Background(), n.Client, KibishiiStorageClassName); err != nil {
 			return err
 		}
@@ -127,6 +127,9 @@ func (n *NamespaceMapping) Clean() error {
 		}
 
 		return n.GetTestCase().Clean()
+	} else {
+		fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.")
 	}
+
 	return nil
 }

test/e2e/basic/pvc-selected-node-changing.go

@@ -149,11 +149,14 @@ func (p *PVCSelectedNodeChanging) Verify() error {
 }
 
 func (p *PVCSelectedNodeChanging) Clean() error {
-	if !p.VeleroCfg.Debug {
+	if !CurrentSpecReport().Failed() || !p.VeleroCfg.FailFast {
 		p.TestCase.Clean()
 		By(fmt.Sprintf("Clean namespace with prefix %s after test", p.mappedNS), func() {
 			CleanupNamespaces(p.Ctx, p.Client, p.mappedNS)
 		})
+	} else {
+		fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.")
 	}
+
 	return nil
 }

test/e2e/basic/resources-check/rbac.go

@@ -36,6 +36,7 @@ import (
 	"fmt"
 	"strings"
 
+	. "github.com/onsi/ginkgo/v2"
 	"github.com/pkg/errors"
 
 	. "github.com/vmware-tanzu/velero/test/e2e/test"
@@ -176,8 +177,11 @@ func (r *RBACCase) Destroy() error {
 }
 
 func (r *RBACCase) Clean() error {
-	if !r.VeleroCfg.Debug {
+	if !CurrentSpecReport().Failed() || !r.VeleroCfg.FailFast {
 		return r.Destroy()
+	} else {
+		fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.")
 	}
+
 	return nil
 }

test/e2e/basic/storage-class-changing.go

@@ -142,14 +142,17 @@ func (s *StorageClasssChanging) Verify() error {
 }
 
 func (s *StorageClasssChanging) Clean() error {
-	if !s.VeleroCfg.Debug {
+	if !CurrentSpecReport().Failed() || !s.VeleroCfg.FailFast {
 		By(fmt.Sprintf("Start to destroy namespace %s......", s.CaseBaseName), func() {
 			Expect(CleanupNamespacesWithPoll(s.Ctx, s.Client, s.CaseBaseName)).To(Succeed(),
 				fmt.Sprintf("Failed to delete namespace %s", s.CaseBaseName))
 		})
 		DeleteConfigmap(s.Client.ClientGo, s.VeleroCfg.VeleroNamespace, s.configmaptName)
 		DeleteStorageClass(s.Ctx, s.Client, s.desStorageClass)
 		s.TestCase.Clean()
+	} else {
+		fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.")
 	}
+
 	return nil
 }

test/e2e/bsl-mgmt/deletion.go

@@ -74,7 +74,7 @@ func BslDeletionTest(useVolumeSnapshots bool) {
 	})
 
 	AfterEach(func() {
-		if !veleroCfg.Debug {
+		if !CurrentSpecReport().Failed() || !veleroCfg.FailFast {
 			By("Clean backups after test", func() {
 				veleroCfg.ClientToInstallVelero = veleroCfg.DefaultClient
 				DeleteAllBackups(context.Background(), &veleroCfg)
@@ -84,6 +84,8 @@ func BslDeletionTest(useVolumeSnapshots bool) {
 					true)).To(Succeed(), fmt.Sprintf("failed to delete the namespace %q",
 					bslDeletionTestNs))
 			})
+		} else {
+			fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.")
 		}
 	})
 

test/e2e/e2e_suite_test.go

@@ -85,7 +85,6 @@ func init() {
 	flag.StringVar(&VeleroCfg.AdditionalBSLConfig, "additional-bsl-config", "", "configuration to use for the additional backup storage location. Format is key1=value1,key2=value2")
 	flag.StringVar(&VeleroCfg.AdditionalBSLCredentials, "additional-bsl-credentials-file", "", "file containing credentials for additional backup storage location provider. Required if testing multiple credentials support.")
 	flag.StringVar(&VeleroCfg.Features, "features", "", "comma-separated list of features to enable for this Velero process.")
-	flag.BoolVar(&VeleroCfg.Debug, "debug-e2e-test", false, "A Switch for enable or disable test data cleaning action.")
 	flag.StringVar(&VeleroCfg.GCFrequency, "garbage-collection-frequency", "", "frequency of garbage collection.")
 	flag.StringVar(&VeleroCfg.DefaultClusterContext, "default-cluster-context", "", "default cluster's kube config context, it's for migration test.")
 	flag.StringVar(&VeleroCfg.StandbyClusterContext, "standby-cluster-context", "", "standby cluster's kube config context, it's for migration test.")
@@ -103,6 +102,7 @@ func init() {
 	flag.StringVar(&VeleroCfg.EKSPolicyARN, "eks-policy-arn", "", "EKS plicy ARN for creating AWS IAM service account.")
 	flag.StringVar(&VeleroCfg.DefaultCLSServiceAccountName, "default-cls-service-account-name", "", "default cluster service account name.")
 	flag.StringVar(&VeleroCfg.StandbyCLSServiceAccountName, "standby-cls-service-account-name", "", "standby cluster service account name.")
+	flag.BoolVar(&VeleroCfg.FailFast, "fail-fast", true, "a switch for failing fast on meeting error.")
 }
 
 // Add label [SkipVanillaZfs]:
@@ -264,6 +264,8 @@ func GetKubeconfigContext() error {
 	return nil
 }
 
+var testSuitePassed bool
+
 func TestE2e(t *testing.T) {
 	// Skip running E2E tests when running only "short" tests because:
 	// 1. E2E tests are long running tests involving installation of Velero and performing backup and restore operations.
@@ -291,7 +293,7 @@ func TestE2e(t *testing.T) {
 	}
 
 	RegisterFailHandler(Fail)
-	RunSpecs(t, "E2e Suite")
+	testSuitePassed = RunSpecs(t, "E2e Suite")
 }
 
 var _ = BeforeSuite(func() {
@@ -302,7 +304,9 @@ var _ = BeforeSuite(func() {
 })
 
 var _ = AfterSuite(func() {
-	if InstallVelero && !VeleroCfg.Debug {
+	// If the Velero is installed during test, and the FailFast is not enabled,
+	// uninstall Velero. If not, either Velero is not installed, or kept it for debug on failure.
+	if InstallVelero && (testSuitePassed || !VeleroCfg.FailFast) {
 		By("release test resources after testing")
 		ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5)
 		defer ctxCancel()

test/e2e/migration/migration.go

@@ -81,7 +81,12 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
 		}
 	})
 	AfterEach(func() {
-		if !veleroCfg.Debug {
+		By(fmt.Sprintf("Switch to default kubeconfig context %s", veleroCfg.DefaultClusterContext), func() {
+			Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed())
+			veleroCfg.ClientToInstallVelero = veleroCfg.DefaultClient
+			veleroCfg.ClusterToInstallVelero = veleroCfg.DefaultClusterName
+		})
+		if !CurrentSpecReport().Failed() || !veleroCfg.FailFast {
 			By(fmt.Sprintf("Uninstall Velero on cluster %s", veleroCfg.DefaultClusterContext), func() {
 				ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5)
 				defer ctxCancel()
@@ -105,11 +110,8 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
 					DeleteNamespace(context.Background(), *veleroCfg.StandbyClient, migrationNamespace, true)
 				})
 			}
-			By(fmt.Sprintf("Switch to default kubeconfig context %s", veleroCfg.DefaultClusterContext), func() {
-				Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed())
-				veleroCfg.ClientToInstallVelero = veleroCfg.DefaultClient
-				veleroCfg.ClusterToInstallVelero = veleroCfg.DefaultClusterName
-			})
+		} else {
+			fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.")
 		}
 	})
 	When("kibishii is the sample workload", func() {

test/e2e/resourcemodifiers/resource_modifiers.go

@@ -131,7 +131,7 @@ func (r *ResourceModifiersCase) Verify() error {
 
 func (r *ResourceModifiersCase) Clean() error {
 	// If created some resources which is not in current test namespace, we NEED to override the base Clean function
-	if !r.VeleroCfg.Debug {
+	if !r.VeleroCfg.FailFast {
 		if err := DeleteConfigmap(r.Client.ClientGo, r.VeleroCfg.VeleroNamespace, r.cmName); err != nil {
 			return err
 		}

test/e2e/resourcepolicies/resource_policies.go

@@ -178,7 +178,7 @@ func (r *ResourcePoliciesCase) Verify() error {
 
 func (r *ResourcePoliciesCase) Clean() error {
 	// If created some resources which is not in current test namespace, we NEED to override the base Clean function
-	if !r.VeleroCfg.Debug {
+	if !CurrentSpecReport().Failed() || !r.VeleroCfg.FailFast {
 		if err := r.deleteTestStorageClassList([]string{StorageClassName, StorageClassName2}); err != nil {
 			return err
 		}
@@ -188,6 +188,8 @@ func (r *ResourcePoliciesCase) Clean() error {
 		}
 
 		return r.GetTestCase().Clean() // only clean up resources in test namespace
+	} else {
+		fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.")
 	}
 	return nil
 }

test/e2e/schedule/ordered_resources.go

@@ -153,10 +153,13 @@ func (o *OrderedResources) Verify() error {
 }
 
 func (o *OrderedResources) Clean() error {
-	if !o.VeleroCfg.Debug {
+	if !CurrentSpecReport().Failed() || !o.VeleroCfg.FailFast {
 		Expect(VeleroScheduleDelete(o.Ctx, o.VeleroCfg.VeleroCLI, o.VeleroCfg.VeleroNamespace, o.ScheduleName)).To(Succeed())
 		Expect(o.TestCase.Clean()).To(Succeed())
+	} else {
+		fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.")
 	}
+
 	return nil
 }