From 7a3886b39b76010aec855c9419a6cdd87d0ca69a Mon Sep 17 00:00:00 2001 From: Xun Jiang Date: Fri, 13 Sep 2024 14:32:08 +0800 Subject: [PATCH] Skip uninstall and resource cleanup when fail-fast is enabled. Bump v1.13 and 1.14 plugin versions for E2E test. Signed-off-by: Xun Jiang --- test/Makefile | 5 +- test/e2e/README.md | 383 ++++++++++-------- test/e2e/backup/backup.go | 4 +- test/e2e/backups/deletion.go | 9 +- test/e2e/backups/sync_backups.go | 15 +- test/e2e/backups/ttl.go | 4 +- .../api-group/enable_api_group_extentions.go | 13 +- .../api-group/enable_api_group_versions.go | 4 +- test/e2e/basic/namespace-mapping.go | 5 +- test/e2e/basic/pvc-selected-node-changing.go | 5 +- test/e2e/basic/resources-check/rbac.go | 6 +- test/e2e/basic/storage-class-changing.go | 5 +- test/e2e/bsl-mgmt/deletion.go | 4 +- test/e2e/e2e_suite_test.go | 10 +- test/e2e/migration/migration.go | 14 +- .../resourcemodifiers/resource_modifiers.go | 2 +- .../e2e/resourcepolicies/resource_policies.go | 4 +- test/e2e/schedule/ordered_resources.go | 5 +- test/e2e/schedule/schedule-backup-creation.go | 87 ++-- test/e2e/schedule/schedule.go | 5 +- test/e2e/test/test.go | 4 +- test/e2e/upgrade/upgrade.go | 4 +- test/perf/e2e_suite_test.go | 12 +- test/perf/test/test.go | 2 +- test/types.go | 2 +- test/util/kibishii/kibishii_utils.go | 3 +- test/util/velero/velero_utils.go | 18 +- 27 files changed, 380 insertions(+), 254 deletions(-) diff --git a/test/Makefile b/test/Makefile index 2cec6307d7..4ea01186cf 100644 --- a/test/Makefile +++ b/test/Makefile @@ -97,7 +97,6 @@ ADDITIONAL_BSL_PREFIX ?= ADDITIONAL_BSL_CONFIG ?= FEATURES ?= -DEBUG_E2E_TEST ?= false DEBUG_VELERO_POD_RESTART ?= false VELERO_SERVER_DEBUG_MODE ?= false @@ -148,10 +147,10 @@ COMMON_ARGS := --velerocli=$(VELERO_CLI) \ --features=$(FEATURES) \ --install-velero=$(INSTALL_VELERO) \ --registry-credential-file=$(REGISTRY_CREDENTIAL_FILE) \ - --debug-e2e-test=$(DEBUG_E2E_TEST) \ --velero-server-debug-mode=$(VELERO_SERVER_DEBUG_MODE) \ --uploader-type=$(UPLOADER_TYPE) \ - --debug-velero-pod-restart=$(DEBUG_VELERO_POD_RESTART) + --debug-velero-pod-restart=$(DEBUG_VELERO_POD_RESTART) \ + --fail-fast=$(FAIL_FAST) # Make sure ginkgo is in $GOBIN .PHONY:ginkgo diff --git a/test/e2e/README.md b/test/e2e/README.md index 1afb113fb2..65231e01af 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -6,7 +6,7 @@ The E2E tests are validating end-to-end behavior of Velero including install, ba If you previously ran unit tests using the `go test ./...` command or any of its variations, then you will now run the same command with the `-short` flag to `go test` to accomplish the same behavior. Alternatively, you can use the `make test` command to run unit tests. -## Prerequisites +## 1. Prerequisites Running the E2E tests expects: 1. One or two running Kubernetes clusters, migration scenario needs 2 clusters: @@ -16,10 +16,10 @@ Running the E2E tests expects: 1. `kubectl` installed locally. -#### Note for migration scenario +### Note for migration scenario _Default Cluster_ refers to source cluster to backup from. Whereas _Standby Cluster_ refers to destination cluster to restore to. -## Limitations +## 2. Limitations These are the current set of limitations with the E2E tests. @@ -31,61 +31,62 @@ These are the current set of limitations with the E2E tests. 1. If the scheduled test case is the normal one, it will check the existence of Velero instance, if no one there then start a new standard instaillation, otherwise proceeding test steps. -## Configuration for E2E tests +## 3. Configuration for E2E tests Below is a list of the configuration used by E2E tests. These configuration parameters are expected as values to the following command line flags: -1. `-credentials-file`: File containing credentials for backup and volume provider. Required. -1. `-bucket`: Name of the object storage bucket where backups from e2e tests should be stored. Required. -1. `-cloud-provider`: The cloud the tests will be run in. Appropriate plugins will be installed except for kind which requires -the object-store-provider to be specified. -1. `-object-store-provider`: Object store provider to use. Required when kind is the cloud provider. -1. `-velerocli`: Path to the velero application to use. Optional, by default uses `velero` in the `$PATH` -1. `-velero-image`: Image for the velero server to be tested. Optional, by default uses `velero/velero:main` -1. `-restore-helper-image`: Image for the velero restore helper to be tested. Optional, by default it is the built-in image address of velero image. -1. `-plugins `: Provider plugins to be tested. -1. `-bsl-config`: Configuration to use for the backup storage location. Format is key1=value1,key2=value2. Optional. -1. `-prefix`: Prefix in the `bucket`, under which all Velero data should be stored within the bucket. Optional. -1. `-vsl-config`: Configuration to use for the volume snapshot location. Format is key1=value1,key2=value2. Optional. -1. `-velero-namespace`: Namespace to install velero in. Optional, defaults to "velero". -1. `-install-velero`: Specifies whether to install/uninstall velero for the tests. Optional, defaults to "true". -1. `-use-node-agent`: Whether deploy node agent daemonset velero during the test. Optional. -1. `-use-volume-snapshots`: Whether or not to create snapshot location automatically. Set to false if you do not plan to create volume snapshots via a storage provider. -1. `-additional-bsl-plugins`: Additional plugins to be tested. -1. `-additional-bsl-object-store-provider`: Provider of object store plugin for additional backup storage location. Required if testing multiple credentials support. -1. `-additional-bsl-bucket`: Name of the object storage bucket for additional backup storage location. Required if testing multiple credentials support. -1. `-additional-bsl-prefix`: Prefix in the `additional-bsl-bucket`, under which all Velero data should be stored. Optional. -1. `-additional-bsl-config`: Configuration to use for the additional backup storage location. Format is key1=value1,key2=value2. Optional. -1. `-additional-bsl-credentials-file`: File containing credentials for the additional backup storage location. Required if testing multiple credentials support. -1. `-velero-version`: Image version for the velero server to be tested with. It's set for upgrade test to verify Velero image version is installed as expected. Required if upgrade test is included in the test. -1. `-upgrade-from-velero-version`: Comma-separated list of Velero version to be tested with for the pre-upgrade velero server. -1. `-upgrade-from-velero-cli`: Comma-separated list of velero application for the pre-upgrade velero server. -1. `-migrate-from-velero-version`: Comma-separated list of Velero version to be tested with on source cluster. -1. `-migrate-from-velero-cli`: Comma-separated list of velero application on source cluster. -1. `-features`: Comma-separated list of features to enable for this Velero process. -1. `-registry-credential-file`: File containing credential for the image registry, follows the same format rules as the ~/.docker/config.json file. This credential will be loaded in Velero server pod to help on Docker Hub rate limit issue. -1. `-kibishii-directory`: The file directory or URL path to install Kibishii. It's configurable in case the default path is not accessible for your own test environment. -1. `-debug-e2e-test`: A Switch for enable or disable test data cleaning action. -1. `-garbage-collection-frequency`: frequency of garbage collection. It is a parameter for Velero installation. Optional. -1. `-velero-server-debug-mode`: A switch for enable or disable having debug log of Velero server. -1. `-default-cluster-context`: Default (source) cluster's kube config context, it's for migration test. -1. `-standby-cluster-context`: Standby (destination) cluster's kube config context, it's for migration test. -1. `-uploader-type`: Type of uploader for persistent volume backup. -1. `-snapshot-move-data`: A Switch for taking backup with Velero's data mover, if data-mover-plugin is not provided, using built-in plugin. -1. `-data-mover-plugin`: Customized plugin for data mover. -1. `-standby-cluster-cloud-provider`: Cloud provider for standby cluster. -1. `-standby-cluster-plugins`: Plugins provider for standby cluster. -1. `-standby-cluster-object-store-provider`: Object store provider for standby cluster. -1. `-debug-velero-pod-restart`: A switch for debugging velero pod restart. +1. `--credentials-file`: File containing credentials for backup and volume provider. Required. +1. `--bucket`: Name of the object storage bucket where backups from e2e tests should be stored. Required. +1. `--cloud-provider`: The cloud the tests will be run in. Appropriate plugins will be installed except for kind which requires the object-store-provider to be specified. +1. `--object-store-provider`: Object store provider to use. Required when kind is the cloud provider. +1. `--velerocli`: Path to the velero application to use. Optional, by default uses `velero` in the `$PATH` +1. `--velero-image`: Image for the velero server to be tested. Optional, by default uses `velero/velero:main` +1. `--restore-helper-image`: Image for the velero restore helper to be tested. Optional, by default it is the built-in image address of velero image. +1. `--plugins `: Provider plugins to be tested. +1. `--bsl-config`: Configuration to use for the backup storage location. Format is key1=value1,key2=value2. Optional. +1. `--prefix`: Prefix in the `bucket`, under which all Velero data should be stored within the bucket. Optional. +1. `--vsl-config`: Configuration to use for the volume snapshot location. Format is key1=value1,key2=value2. Optional. +1. `--velero-namespace`: Namespace to install velero in. Optional, defaults to "velero". +1. `--install-velero`: Specifies whether to install/uninstall velero for the tests. Optional, defaults to "true". +1. `--use-node-agent`: Whether deploy node agent DaemonSet velero during the test. Optional. +1. `--use-volume-snapshots`: Whether or not to create snapshot location automatically. Set to false if you do not plan to create volume snapshots via a storage provider. +1. `--additional-bsl-plugins`: Additional plugins to be tested. +1. `--additional-bsl-object-store-provider`: Provider of object store plugin for additional backup storage location. Required if testing multiple credentials support. +1. `--additional-bsl-bucket`: Name of the object storage bucket for additional backup storage location. Required if testing multiple credentials support. +1. `--additional-bsl-prefix`: Prefix in the `additional-bsl-bucket`, under which all Velero data should be stored. Optional. +1. `--additional-bsl-config`: Configuration to use for the additional backup storage location. Format is key1=value1,key2=value2. Optional. +1. `--additional-bsl-credentials-file`: File containing credentials for the additional backup storage location. Required if testing multiple credentials support. +1. `--velero-version`: Image version for the velero server to be tested with. It's set for upgrade test to verify Velero image version is installed as expected. Required if upgrade test is included in the test. +1. `--upgrade-from-velero-version`: Comma-separated list of Velero version to be tested with for the pre-upgrade velero server. +1. `--upgrade-from-velero-cli`: Comma-separated list of velero application for the pre-upgrade velero server. +1. `--migrate-from-velero-version`: Comma-separated list of Velero version to be tested with on source cluster. +1. `--migrate-from-velero-cli`: Comma-separated list of velero application on source cluster. +1. `--features`: Comma-separated list of features to enable for this Velero process. +1. `--registry-credential-file`: File containing credential for the image registry, follows the same format rules as the ~/.docker/config.json file. This credential will be loaded in Velero server pod to help on Docker Hub rate limit issue. +1. `--kibishii-directory`: The file directory or URL path to install Kibishii. It's configurable in case the default path is not accessible for your own test environment. +1. `--debug-e2e-test`: A Switch for enable or disable test data cleaning action. +1. `--garbage-collection-frequency`: frequency of garbage collection. It is a parameter for Velero installation. Optional. +1. `--velero-server-debug-mode`: A switch for enable or disable having debug log of Velero server. +1. `--default-cluster-context`: Default (source) cluster's kube config context, it's for migration test. +1. `--standby-cluster-context`: Standby (destination) cluster's kube config context, it's for migration test. +1. `--uploader-type`: Type of uploader for persistent volume backup. +1. `--snapshot-move-data`: A Switch for taking backup with Velero's data mover, if data-mover-plugin is not provided, using built-in plugin. +1. `--data-mover-plugin`: Customized plugin for data mover. +1. `--standby-cluster-cloud-provider`: Cloud provider for standby cluster. +1. `--standby-cluster-plugins`: Plugins provider for standby cluster. +1. `--standby-cluster-object-store-provider`: Object store provider for standby cluster. +1. `--debug-velero-pod-restart`: A switch for debugging velero pod restart. +1. `--fail-fast`: A switch for for failing fast on meeting error. These configurations or parameters are used to generate install options for Velero for each test suite. Tests can be run with the Kubernetes cluster hosted in various cloud providers or in a _kind_ cluster with storage in a specified object store type. Currently supported cloud provider types are _aws_, _azure_, _vsphere_ and _kind_. -## Running tests locally -### Running using `make` +## 4. Running tests + +### Parameters for `make` E2E tests can be run from the Velero repository root by running `make test-e2e`. While running E2E tests using `make` the E2E test configuration values are passed using `make` variables. @@ -116,7 +117,6 @@ Below is a mapping between `make` variables to E2E configuration flags. 1. `FEATURES`: `-features`. Optional. 1. `REGISTRY_CREDENTIAL_FILE`: `-registry-credential-file`. Optional. 1. `KIBISHII_DIRECTORY`: `-kibishii-directory`. Optional. -1. `DEBUG_E2E_TEST`: `-debug-e2e-test`. Optional. 1. `VELERO_SERVER_DEBUG_MODE`: `-velero-server-debug-mode`. Optional. 1. `DEFAULT_CLUSTER`: `-default-cluster-context`. Optional. 1. `STANDBY_CLUSTER`: `-standby-cluster-context`. Optional. @@ -128,110 +128,154 @@ Below is a mapping between `make` variables to E2E configuration flags. 1. `STANDBY_CLUSTER_OBJECT_STORE_PROVIDER`: `-standby-cluster-object-store-provider`. Optional. 1. `INSTALL_VELERO `: `-install-velero`. Optional. 1. `DEBUG_VELERO_POD_RESTART`: `-debug-velero-pod-restart`. Optional. +1. `FAIL_FAST`: `--fail-fast`. Optional. -#### For example, E2E tests can be run from Velero repository roots using the commands below: +### Examples Basic examples: -1. Run Velero tests in a kind cluster with AWS (or Minio) as the storage provider: +1. Run Velero tests in a kind cluster with AWS (or MinIO) as the storage provider: - Start kind cluster - ```bash - kind create cluster - ``` - ```bash - BSL_PREFIX= BSL_BUCKET= CREDS_FILE=/path/to/aws-creds CLOUD_PROVIDER=kind OBJECT_STORE_PROVIDER=aws make test-e2e - ``` - Stop kind cluster - ```bash - kind delete cluster - ``` +Start kind cluster +``` bash +kind create cluster +``` + +``` bash +BSL_PREFIX= \ +BSL_BUCKET= \ +CREDS_FILE=/path/to/aws-creds \ +CLOUD_PROVIDER=kind \ +OBJECT_STORE_PROVIDER=aws \ +make test-e2e +``` + +Stop kind cluster +``` bash +kind delete cluster +``` + 1. Run Velero tests in an AWS cluster: - ```bash - BSL_PREFIX= BSL_BUCKET= CREDS_FILE=/path/to/aws-creds CLOUD_PROVIDER=aws make test-e2e - ``` +```bash +BSL_PREFIX= \ +BSL_BUCKET= \ +CREDS_FILE=/path/to/aws-creds \ +CLOUD_PROVIDER=aws \ +make test-e2e +``` + 1. Run Velero tests in a Microsoft Azure cluster: - ```bash - BSL_CONFIG="resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,storageAccount=$AZURE_STORAGE_ACCOUNT_ID,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID" BSL_BUCKET= CREDS_FILE=/path/to/azure-creds CLOUD_PROVIDER=azure make test-e2e - ``` - Please refer to `velero-plugin-for-microsoft-azure` documentation for instruction to [set up permissions for Velero](https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure#set-permissions-for-velero) and to [set up azure storage account and blob container](https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure#setup-azure-storage-account-and-blob-container) +```bash +BSL_CONFIG="resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,storageAccount=$AZURE_STORAGE_ACCOUNT_ID,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID" \ +BSL_BUCKET= \ +CREDS_FILE=/path/to/azure-creds \ +CLOUD_PROVIDER=azure \ +make test-e2e +``` + +Please refer to `velero-plugin-for-microsoft-azure` documentation for instruction: +* [set up permissions for Velero](https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure#set-permissions-for-velero) +* [set up azure storage account and blob container](https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure#setup-azure-storage-account-and-blob-container) + 1. Run Multi-API group and version tests using MinIO as the backup storage location: - ```bash - BSL_CONFIG="region=minio,s3ForcePathStyle=\"true\",s3Url=:9000" BSL_PREFIX= BSL_BUCKET= CREDS_FILE= CLOUD_PROVIDER=kind OBJECT_STORE_PROVIDER=aws VELERO_NAMESPACE="velero" GINKGO_LABELS="APIGroup && APIVersion" make test-e2e + ``` bash + BSL_CONFIG="region=minio,s3ForcePathStyle=true,s3Url=:9000" \ + BSL_PREFIX= \ + BSL_BUCKET= \ + CREDS_FILE= \ + CLOUD_PROVIDER=kind \ + OBJECT_STORE_PROVIDER=aws \ + VELERO_NAMESPACE="velero" \ + GINKGO_LABELS="APIGroup && APIVersion" \ + make test-e2e ``` -1. Run Velero tests in a kind cluster with AWS (or Minio) as the storage provider and use Microsoft Azure as the storage provider for an additional Backup Storage Location: - ```bash - make test-e2e \ - CLOUD_PROVIDER=kind OBJECT_STORE_PROVIDER=aws BSL_BUCKET= BSL_PREFIX= CREDS_FILE=/path/to/aws-creds \ - ADDITIONAL_OBJECT_STORE_PROVIDER=azure ADDITIONAL_BSL_BUCKET= ADDITIONAL_BSL_PREFIX= ADDITIONAL_BSL_CONFIG= ADDITIONAL_CREDS_FILE=/path/to/azure-creds - ``` + +1. Run Velero tests in a kind cluster with AWS (or MinIO) as the storage provider and use Microsoft Azure as the storage provider for an additional Backup Storage Location: + +```bash +CLOUD_PROVIDER=kind \ +OBJECT_STORE_PROVIDER=aws \ +BSL_BUCKET= \ +BSL_PREFIX= \ +CREDS_FILE=/path/to/aws-creds \ +ADDITIONAL_OBJECT_STORE_PROVIDER=azure \ +ADDITIONAL_BSL_BUCKET= \ +ADDITIONAL_BSL_PREFIX= \ +ADDITIONAL_BSL_CONFIG= \ +ADDITIONAL_CREDS_FILE=/path/to/azure-creds \ +make test-e2e +``` Upgrade examples: 1. Run Velero upgrade tests with pre-upgrade version: - This example will run 1 upgrade tests: v1.11.0 ~ target. - - ```bash - make test-e2e \ - CLOUD_PROVIDER=aws BSL_BUCKET= CREDS_FILE=/path/to/aws-creds \ - UPGRADE_FROM_VELERO_VERSION=v1.11.0 - ``` -1. Run Velero upgrade tests with pre-upgrade version list: +This example will run 1 upgrade tests: v1.11.0 ~ target. - This example will run 2 upgrade tests: v1.10.2 ~ target and v1.11.0 ~ target. - - ```bash - make test-e2e \ - CLOUD_PROVIDER=aws BSL_BUCKET= CREDS_FILE=/path/to/aws-creds \ - UPGRADE_FROM_VELERO_VERSION=v1.10.2,v1.11.0 - ``` +``` bash +CLOUD_PROVIDER=aws \ +BSL_BUCKET= \ +CREDS_FILE=/path/to/aws-creds \ +UPGRADE_FROM_VELERO_VERSION=v1.11.0 \ +make test-e2e +``` +1. Run Velero upgrade tests with pre-upgrade version list: + +This example will run 2 upgrade tests: v1.10.2 ~ target and v1.11.0 ~ target. +```bash +CLOUD_PROVIDER=aws \ +BSL_BUCKET= \ +CREDS_FILE=/path/to/aws-creds \ +UPGRADE_FROM_VELERO_VERSION=v1.10.2,v1.11.0 \ +make test-e2e +``` Migration examples: 1. Migration between 2 cluster of the same provider tests: - Before running migration test, we should prepare kubeconfig file which contains config of each cluster under test, and set `DEFAULT_CLUSTER` and `STANDBY_CLUSTER` with the corresponding context value in kubeconfig file. - - `MIGRATE_FROM_VELERO_VERSION` includes a keyword of `self`, which means migration from Velero under test (specified by `VELERO_IMAGE`) to the same Velero. This variable can be set to `v1.10.0`, `v1.10.0,v1.11.1`, `self` or `v1.11.0,self`. +Before running migration test, we should prepare `kube-config` file which contains config of each cluster under test, and set `DEFAULT_CLUSTER` and `STANDBY_CLUSTER` with the corresponding context value in `kube-config` file. - ```bash - make test-e2e \ - CLOUD_PROVIDER=aws BSL_BUCKET= CREDS_FILE=/path/to/aws-creds \ - DEFAULT_CLUSTER= \ - STANDBY_CLUSTER= \ - MIGRATE_FROM_VELERO_VERSION=v1.11.0,self - ``` +`MIGRATE_FROM_VELERO_VERSION` includes a keyword of `self`, which means migration from Velero under test (specified by `VELERO_IMAGE`) to the same Velero. This variable can be set to `v1.10.0`, `v1.10.0,v1.11.1`, `self` or `v1.11.0,self`. +```bash +CLOUD_PROVIDER=aws \ +BSL_BUCKET= \ +CREDS_FILE=/path/to/aws-creds \ +DEFAULT_CLUSTER= \ +STANDBY_CLUSTER= \ +MIGRATE_FROM_VELERO_VERSION=v1.11.0,self \ +make test-e2e +``` 1. Data mover tests: - The example shows all essential `make` variables for a data mover test which is migrate from a AKS cluster to a EKS cluster. - - Note: STANDBY_CLUSTER_CLOUD_PROVIDER and STANDBY_CLUSTER_OBJECT_STORE_PROVIDER is essential here, it is for identify plugins to be installed on target cluster, since DEFAULT cluster's provider is different from STANDBY cluster, plugins are different as well. - ```bash - CLOUD_PROVIDER=azure \ - DEFAULT_CLUSTER= \ - STANDBY_CLUSTER= \ - FEATURES=EnableCSI \ - OBJECT_STORE_PROVIDER=aws \ - CREDS_FILE= \ - BSL_CONFIG=region= \ - BSL_BUCKET= \ - BSL_PREFIX= \ - VSL_CONFIG=region= \ - SNAPSHOT_MOVE_DATA=true \ - STANDBY_CLUSTER_CLOUD_PROVIDER=aws \ - STANDBY_CLUSTER_OBJECT_STORE_PROVIDER=aws \ - GINKGO_LABELS="Migration" \ - make test-e2e - ``` - -## Filtering tests +The example shows all essential `make` variables for a data mover test which is migrate from a AKS cluster to a EKS cluster. + +Note: STANDBY_CLUSTER_CLOUD_PROVIDER and STANDBY_CLUSTER_OBJECT_STORE_PROVIDER is essential here, it is for identify plugins to be installed on target cluster, since DEFAULT cluster's provider is different from STANDBY cluster, plugins are different as well. +```bash +CLOUD_PROVIDER=azure \ +DEFAULT_CLUSTER= \ +STANDBY_CLUSTER= \ +FEATURES=EnableCSI \ +OBJECT_STORE_PROVIDER=aws \ +CREDS_FILE= \ +BSL_CONFIG=region= \ +BSL_BUCKET= \ +BSL_PREFIX= \ +VSL_CONFIG=region= \ +SNAPSHOT_MOVE_DATA=true \ +STANDBY_CLUSTER_CLOUD_PROVIDER=aws \ +STANDBY_CLUSTER_OBJECT_STORE_PROVIDER=aws \ +GINKGO_LABELS="Migration" \ +make test-e2e +``` + +## 5. Filtering tests In release-1.15, Velero bumps the [Ginkgo](https://onsi.github.io/ginkgo/) version to [v2](https://onsi.github.io/ginkgo/MIGRATING_TO_V2). Velero E2E start to use [labels](https://onsi.github.io/ginkgo/#spec-labels) to filter cases instead of [`-focus` and `-skip`](https://onsi.github.io/ginkgo/#focused-specs) parameters. @@ -243,36 +287,41 @@ Both `make run-e2e` and `make run-perf` CLI support using parameter `GINKGO_LABE ### Examples E2E tests can be run with specific cases to be included and/or excluded using the commands below: -1. Run Velero tests with specific cases to be included: - ```bash - GINKGO_LABELS="Basic && Restic" \ - CLOUD_PROVIDER=aws \ - BSL_BUCKET=example-bucket \ - CREDS_FILE=/path/to/aws-creds \ - make test-e2e \ - ``` - In this example, only case have both `Basic` and `Restic` labels are included. -1. Run Velero tests with specific cases to be excluded: - ```bash - GINKGO_LABELS="!(Scale || Schedule || TTL || (Upgrade && Restic) || (Migration && Restic))" \ - CLOUD_PROVIDER=aws \ - BSL_BUCKET=example-bucket \ - CREDS_FILE=/path/to/aws-creds \ - make test-e2e - ``` - In this example, cases are labelled as - * `Scale` - * `Schedule` - * `TTL` - * `Upgrade` and `Restic` - * `Migration` and `Restic` - will be skipped. - -## Full Tests execution +1. Run Velero tests with specific cases to be included: +```bash +GINKGO_LABELS="Basic && Restic" \ +CLOUD_PROVIDER=aws \ +BSL_BUCKET=example-bucket \ +CREDS_FILE=/path/to/aws-creds \ +make test-e2e \ +``` -As we provided several examples for E2E test execution, what if no filter is involved and despite difference of test environment, is that a full test that covered all features on some certain environment, unfortunately, to prevent long time running or for some special test scenarioes, there're some tests need to be run in a single execution or pipeline with specific parameters provided. +In this example, only case have both `Basic` and `Restic` labels are included. +1. Run Velero tests with specific cases to be excluded: +```bash +GINKGO_LABELS="!(Scale || Schedule || TTL || (Upgrade && Restic) || (Migration && Restic))" \ +CLOUD_PROVIDER=aws \ +BSL_BUCKET=example-bucket \ +CREDS_FILE=/path/to/aws-creds \ +make test-e2e +``` + +In this example, cases are labelled as +* `Scale` +* `Schedule` +* `TTL` +* `Upgrade` and `Restic` +* `Migration` and `Restic` +will be skipped. + +## 6. Full Tests execution + +As we provided several examples for E2E test execution, if no filter is involved and despite difference of test environment, +that is a full test that covered all features on some certain environment. +Unfortunately, to prevent long time running or for some special test scenarios, +there're some tests need to be run in a single execution or pipeline with specific parameters provided. ### Suggested pipelines for full test @@ -287,39 +336,51 @@ Following pipelines should cover all E2E tests along with proper filters: **Note:** please organize filters among proper pipelines for other test cases. -## Adding tests +## 7. Adding tests ### API clients When adding a test, aim to instantiate an API client only once at the beginning of the test. There is a constructor `newTestClient` that facilitates the configuration and instantiation of clients. Also, please use the `kubebuilder` runtime controller client for any new test, as we will phase out usage of `client-go` API clients. -## TestCase frame related +## 8. TestCase frame related TestCase frame provide a serials of interface to concatenate one complete e2e test. it's makes the testing be concise and explicit. + ### VeleroBackupRestoreTest interface VeleroBackupRestoreTest interface provided a standard workflow of backup and restore, which makes the whole testing process clearer and code reusability. For programming conventions, Take ResourcePoliciesCase case for example: -##### Init +#### Init - It's need first call the base `TestCase` Init function to generate random number as UUIDgen and set one default timeout duration - Assigning CaseBaseName variable with a case related prefix, all others variables will follow the prefix - Assigning NamespacesTotal variable, and generating namespaces - Assigning values to the inner variable for specific case - For BackupArgs, as Velero installation is using global Velero configuration, it's NEED to specify the value of the variable snapshot-volumes or default-volumes-to-fs-backup explicitly. -##### CreateResources + +#### CreateResources - It's better to set a global timeout in CreateResources function which is the real beginning of one e2e test -##### Destroy + +#### Destroy - It only cleans up resources in currently test namespaces, if you wish to clean up all resources including resources created which are not in currently test namespaces, it's better to override base Destroy function -##### Clean + +#### Clean - Clean function only clean resources in namespaces which has the prefix CaseBaseName. So the the names of test namespaces should start with prefix of CaseBaseName. -- If wantting to clean up all resources including resources created which is not in currently test namespaces, it's better to override base Clean function +- It's better to override base Clean function, if need to clean up all resources including resources created which is not in currently test namespaces. + #### Velero Installation - Velero is installed with global velero config before the E2E test start, so if the case (such as upgrade/migration, etc.) does not want to use the global velero config, it is NEED TO UNINSTALL velero, or the global velero config may affect the current test. + ### TestFunc The TestFunc function concatenate all the flows in a test. -It will reduce the frequency of velero installation by installing and checking Velero with the global Velero. It's Need to explicit reinstall Velero if the case has specicial configuration, such as API Group test case we need to enable feature EnableCSI. +It will reduce the frequency of velero installation by installing and checking Velero with the global Velero. It's Need to explicit reinstall Velero if the case has special configuration, such as API Group test case we need to enable feature EnableCSI. ### Tips -Look for the ⛵ emoji printed at the end of each install and uninstall log. There should not be two install/unintall in a row, and there should be tests between an install and an uninstall. +Look for the ⛵ emoji printed at the end of each install and uninstall log. There should not be two install/uninstall in a row, and there should be tests between an install and an uninstall. -#### Troubleshooting +## 9. Troubleshooting +## `Failed to get bucket region` error If velero log shows `level=error msg="Failed to get bucket region, bucket: xbucket, error: operation error S3: HeadBucket, failed to resolve service endpoint, endpoint rule error, A region must be set when sending requests to S3." backup-storage-location=velero/default cmd=/plugins/velero-plugin-for-aws controller=backup-storage-location logSource="/go/src/velero-plugin-for-aws/velero-plugin-for-aws/object_store.go:136" pluginName=velero-plugin-for-aws`, it means you need to set `BSL_CONFIG` to include `region=`. + +## fail fast +If need to debug the failed test case, please set the `FAIL_FAST=true` for the `make test-e2e` CLI. +If `FAIL_FAST` is enabled, the pipeline with failed test case will not delete the test bed, +and the resources(including Velero instance, CRs, and other k8s resources) created during the test are kept. diff --git a/test/e2e/backup/backup.go b/test/e2e/backup/backup.go index 1c8c434d8d..b65c16d153 100644 --- a/test/e2e/backup/backup.go +++ b/test/e2e/backup/backup.go @@ -97,7 +97,7 @@ func BackupRestoreTest(backupRestoreTestConfig BackupRestoreTestConfig) { }) AfterEach(func() { - if !veleroCfg.Debug { + if !CurrentSpecReport().Failed() || !veleroCfg.FailFast { By("Clean backups after test", func() { DeleteAllBackups(context.Background(), &veleroCfg) if backupRestoreTestConfig.isRetainPVTest { @@ -111,6 +111,8 @@ func BackupRestoreTest(backupRestoreTestConfig BackupRestoreTestConfig) { err = VeleroUninstall(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace) Expect(err).To(Succeed()) } + } else { + fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.") } }) diff --git a/test/e2e/backups/deletion.go b/test/e2e/backups/deletion.go index 23e24a7f62..1f0fc5ed89 100644 --- a/test/e2e/backups/deletion.go +++ b/test/e2e/backups/deletion.go @@ -62,10 +62,12 @@ func backup_deletion_test(useVolumeSnapshots bool) { }) AfterEach(func() { - if !veleroCfg.Debug { + if !CurrentSpecReport().Failed() || !veleroCfg.FailFast { By("Clean backups after test", func() { DeleteAllBackups(context.Background(), &veleroCfg) }) + } else { + fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.") } }) @@ -104,13 +106,16 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupLoc if err := CreateNamespace(oneHourTimeout, client, ns); err != nil { return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", ns) } - if !veleroCfg.Debug { + if !CurrentSpecReport().Failed() || !veleroCfg.FailFast { defer func() { if err := DeleteNamespace(context.Background(), client, ns, true); err != nil { fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", ns)) } }() + } else { + fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.") } + if err := KibishiiPrepareBeforeBackup(oneHourTimeout, client, providerName, ns, registryCredentialFile, veleroFeatures, kibishiiDirectory, useVolumeSnapshots, DefaultKibishiiData); err != nil { return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", ns) diff --git a/test/e2e/backups/sync_backups.go b/test/e2e/backups/sync_backups.go index de5c1a5f03..6a8e6684de 100644 --- a/test/e2e/backups/sync_backups.go +++ b/test/e2e/backups/sync_backups.go @@ -63,7 +63,7 @@ func BackupsSyncTest() { }) AfterEach(func() { - if !veleroCfg.Debug { + if !CurrentSpecReport().Failed() || !veleroCfg.FailFast { By("Clean backups after test", func() { DeleteAllBackups(context.Background(), &veleroCfg) }) @@ -72,6 +72,8 @@ func BackupsSyncTest() { defer ctxCancel() Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).To(Succeed()) } + } else { + fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.") } }) @@ -82,10 +84,13 @@ func BackupsSyncTest() { By(fmt.Sprintf("Prepare workload as target to backup by creating namespace %s namespace", test.testNS)) Expect(CreateNamespace(ctx, *veleroCfg.ClientToInstallVelero, test.testNS)).To(Succeed(), fmt.Sprintf("Failed to create %s namespace", test.testNS)) - if !veleroCfg.Debug { + + if !CurrentSpecReport().Failed() || !veleroCfg.FailFast { defer func() { Expect(DeleteNamespace(ctx, *veleroCfg.ClientToInstallVelero, test.testNS, false)).To(Succeed(), fmt.Sprintf("Failed to delete the namespace %s", test.testNS)) }() + } else { + fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.") } var BackupCfg BackupConfig @@ -125,12 +130,16 @@ func BackupsSyncTest() { Expect(CreateNamespace(ctx, *veleroCfg.ClientToInstallVelero, test.testNS)).To(Succeed(), fmt.Sprintf("Failed to create %s namespace", test.testNS)) }) - if !veleroCfg.Debug { + + if !CurrentSpecReport().Failed() || !veleroCfg.FailFast { defer func() { Expect(DeleteNamespace(ctx, *veleroCfg.ClientToInstallVelero, test.testNS, false)).To(Succeed(), fmt.Sprintf("Failed to delete the namespace %s", test.testNS)) }() + } else { + fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.") } + var BackupCfg BackupConfig BackupCfg.BackupName = test.backupName BackupCfg.Namespace = test.testNS diff --git a/test/e2e/backups/ttl.go b/test/e2e/backups/ttl.go index 578f6161d8..9bb3521d58 100644 --- a/test/e2e/backups/ttl.go +++ b/test/e2e/backups/ttl.go @@ -74,7 +74,7 @@ func TTLTest() { AfterEach(func() { veleroCfg.GCFrequency = "" - if !veleroCfg.Debug { + if !CurrentSpecReport().Failed() || !veleroCfg.FailFast { By("Clean backups after test", func() { DeleteAllBackups(context.Background(), &veleroCfg) }) @@ -84,6 +84,8 @@ func TTLTest() { Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).To(Succeed()) } Expect(DeleteNamespace(ctx, client, test.testNS, false)).To(Succeed(), fmt.Sprintf("Failed to delete the namespace %s", test.testNS)) + } else { + fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.") } }) diff --git a/test/e2e/basic/api-group/enable_api_group_extentions.go b/test/e2e/basic/api-group/enable_api_group_extentions.go index f78db34175..b6c35f5331 100644 --- a/test/e2e/basic/api-group/enable_api_group_extentions.go +++ b/test/e2e/basic/api-group/enable_api_group_extentions.go @@ -67,7 +67,12 @@ func APIExtensionsVersionsTest() { }) }) AfterEach(func() { - if !veleroCfg.Debug { + By(fmt.Sprintf("Switch to default kubeconfig context %s", veleroCfg.DefaultClusterContext), func() { + Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed()) + veleroCfg.ClientToInstallVelero = veleroCfg.DefaultClient + }) + + if !CurrentSpecReport().Failed() || !veleroCfg.FailFast { By("Clean backups after test", func() { DeleteAllBackups(context.Background(), &veleroCfg) }) @@ -86,10 +91,8 @@ func APIExtensionsVersionsTest() { Expect(DeleteCRDByName(context.Background(), crdName)).To(Succeed()) }) } - By(fmt.Sprintf("Switch to default kubeconfig context %s", veleroCfg.DefaultClusterContext), func() { - Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed()) - veleroCfg.ClientToInstallVelero = veleroCfg.DefaultClient - }) + } else { + fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.") } }) Context("When EnableAPIGroupVersions flag is set", func() { diff --git a/test/e2e/basic/api-group/enable_api_group_versions.go b/test/e2e/basic/api-group/enable_api_group_versions.go index 7f67dbf6e1..8675c327cd 100644 --- a/test/e2e/basic/api-group/enable_api_group_versions.go +++ b/test/e2e/basic/api-group/enable_api_group_versions.go @@ -79,7 +79,7 @@ func APIGroupVersionsTest() { }) AfterEach(func() { - if !veleroCfg.Debug { + if !CurrentSpecReport().Failed() || !veleroCfg.FailFast { for i := 0; i < testCaseNum; i++ { curResource := fmt.Sprintf("rockband%ds", i) curGroup := fmt.Sprintf("%s.%d", group, i) @@ -101,6 +101,8 @@ func APIGroupVersionsTest() { Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).NotTo(HaveOccurred()) }) } + } else { + fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.") } }) diff --git a/test/e2e/basic/namespace-mapping.go b/test/e2e/basic/namespace-mapping.go index 479c37a226..ad0b4aac7a 100644 --- a/test/e2e/basic/namespace-mapping.go +++ b/test/e2e/basic/namespace-mapping.go @@ -116,7 +116,7 @@ func (n *NamespaceMapping) Verify() error { } func (n *NamespaceMapping) Clean() error { - if !n.VeleroCfg.Debug { + if !CurrentSpecReport().Failed() || !n.VeleroCfg.FailFast { if err := DeleteStorageClass(context.Background(), n.Client, KibishiiStorageClassName); err != nil { return err } @@ -127,6 +127,9 @@ func (n *NamespaceMapping) Clean() error { } return n.GetTestCase().Clean() + } else { + fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.") } + return nil } diff --git a/test/e2e/basic/pvc-selected-node-changing.go b/test/e2e/basic/pvc-selected-node-changing.go index d3e22b8922..bb7b37e490 100644 --- a/test/e2e/basic/pvc-selected-node-changing.go +++ b/test/e2e/basic/pvc-selected-node-changing.go @@ -149,11 +149,14 @@ func (p *PVCSelectedNodeChanging) Verify() error { } func (p *PVCSelectedNodeChanging) Clean() error { - if !p.VeleroCfg.Debug { + if !CurrentSpecReport().Failed() || !p.VeleroCfg.FailFast { p.TestCase.Clean() By(fmt.Sprintf("Clean namespace with prefix %s after test", p.mappedNS), func() { CleanupNamespaces(p.Ctx, p.Client, p.mappedNS) }) + } else { + fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.") } + return nil } diff --git a/test/e2e/basic/resources-check/rbac.go b/test/e2e/basic/resources-check/rbac.go index 7afedd13c6..9b3086a2d7 100644 --- a/test/e2e/basic/resources-check/rbac.go +++ b/test/e2e/basic/resources-check/rbac.go @@ -36,6 +36,7 @@ import ( "fmt" "strings" + . "github.com/onsi/ginkgo/v2" "github.com/pkg/errors" . "github.com/vmware-tanzu/velero/test/e2e/test" @@ -176,8 +177,11 @@ func (r *RBACCase) Destroy() error { } func (r *RBACCase) Clean() error { - if !r.VeleroCfg.Debug { + if !CurrentSpecReport().Failed() || !r.VeleroCfg.FailFast { return r.Destroy() + } else { + fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.") } + return nil } diff --git a/test/e2e/basic/storage-class-changing.go b/test/e2e/basic/storage-class-changing.go index d61e0b4353..9344c556f3 100644 --- a/test/e2e/basic/storage-class-changing.go +++ b/test/e2e/basic/storage-class-changing.go @@ -142,7 +142,7 @@ func (s *StorageClasssChanging) Verify() error { } func (s *StorageClasssChanging) Clean() error { - if !s.VeleroCfg.Debug { + if !CurrentSpecReport().Failed() || !s.VeleroCfg.FailFast { By(fmt.Sprintf("Start to destroy namespace %s......", s.CaseBaseName), func() { Expect(CleanupNamespacesWithPoll(s.Ctx, s.Client, s.CaseBaseName)).To(Succeed(), fmt.Sprintf("Failed to delete namespace %s", s.CaseBaseName)) @@ -150,6 +150,9 @@ func (s *StorageClasssChanging) Clean() error { DeleteConfigmap(s.Client.ClientGo, s.VeleroCfg.VeleroNamespace, s.configmaptName) DeleteStorageClass(s.Ctx, s.Client, s.desStorageClass) s.TestCase.Clean() + } else { + fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.") } + return nil } diff --git a/test/e2e/bsl-mgmt/deletion.go b/test/e2e/bsl-mgmt/deletion.go index 35037efa37..3b57833374 100644 --- a/test/e2e/bsl-mgmt/deletion.go +++ b/test/e2e/bsl-mgmt/deletion.go @@ -74,7 +74,7 @@ func BslDeletionTest(useVolumeSnapshots bool) { }) AfterEach(func() { - if !veleroCfg.Debug { + if !CurrentSpecReport().Failed() || !veleroCfg.FailFast { By("Clean backups after test", func() { veleroCfg.ClientToInstallVelero = veleroCfg.DefaultClient DeleteAllBackups(context.Background(), &veleroCfg) @@ -84,6 +84,8 @@ func BslDeletionTest(useVolumeSnapshots bool) { true)).To(Succeed(), fmt.Sprintf("failed to delete the namespace %q", bslDeletionTestNs)) }) + } else { + fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.") } }) diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 1031958084..e14eb42a4a 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -85,7 +85,6 @@ func init() { flag.StringVar(&VeleroCfg.AdditionalBSLConfig, "additional-bsl-config", "", "configuration to use for the additional backup storage location. Format is key1=value1,key2=value2") flag.StringVar(&VeleroCfg.AdditionalBSLCredentials, "additional-bsl-credentials-file", "", "file containing credentials for additional backup storage location provider. Required if testing multiple credentials support.") flag.StringVar(&VeleroCfg.Features, "features", "", "comma-separated list of features to enable for this Velero process.") - flag.BoolVar(&VeleroCfg.Debug, "debug-e2e-test", false, "A Switch for enable or disable test data cleaning action.") flag.StringVar(&VeleroCfg.GCFrequency, "garbage-collection-frequency", "", "frequency of garbage collection.") flag.StringVar(&VeleroCfg.DefaultClusterContext, "default-cluster-context", "", "default cluster's kube config context, it's for migration test.") flag.StringVar(&VeleroCfg.StandbyClusterContext, "standby-cluster-context", "", "standby cluster's kube config context, it's for migration test.") @@ -103,6 +102,7 @@ func init() { flag.StringVar(&VeleroCfg.EKSPolicyARN, "eks-policy-arn", "", "EKS plicy ARN for creating AWS IAM service account.") flag.StringVar(&VeleroCfg.DefaultCLSServiceAccountName, "default-cls-service-account-name", "", "default cluster service account name.") flag.StringVar(&VeleroCfg.StandbyCLSServiceAccountName, "standby-cls-service-account-name", "", "standby cluster service account name.") + flag.BoolVar(&VeleroCfg.FailFast, "fail-fast", true, "a switch for failing fast on meeting error.") } // Add label [SkipVanillaZfs]: @@ -264,6 +264,8 @@ func GetKubeconfigContext() error { return nil } +var testSuitePassed bool + func TestE2e(t *testing.T) { // Skip running E2E tests when running only "short" tests because: // 1. E2E tests are long running tests involving installation of Velero and performing backup and restore operations. @@ -291,7 +293,7 @@ func TestE2e(t *testing.T) { } RegisterFailHandler(Fail) - RunSpecs(t, "E2e Suite") + testSuitePassed = RunSpecs(t, "E2e Suite") } var _ = BeforeSuite(func() { @@ -302,7 +304,9 @@ var _ = BeforeSuite(func() { }) var _ = AfterSuite(func() { - if InstallVelero && !VeleroCfg.Debug { + // If the Velero is installed during test, and the FailFast is not enabled, + // uninstall Velero. If not, either Velero is not installed, or kept it for debug on failure. + if InstallVelero && (testSuitePassed || !VeleroCfg.FailFast) { By("release test resources after testing") ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5) defer ctxCancel() diff --git a/test/e2e/migration/migration.go b/test/e2e/migration/migration.go index 0cf96f1bfd..c805737e35 100644 --- a/test/e2e/migration/migration.go +++ b/test/e2e/migration/migration.go @@ -81,7 +81,12 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) } }) AfterEach(func() { - if !veleroCfg.Debug { + By(fmt.Sprintf("Switch to default kubeconfig context %s", veleroCfg.DefaultClusterContext), func() { + Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed()) + veleroCfg.ClientToInstallVelero = veleroCfg.DefaultClient + veleroCfg.ClusterToInstallVelero = veleroCfg.DefaultClusterName + }) + if !CurrentSpecReport().Failed() || !veleroCfg.FailFast { By(fmt.Sprintf("Uninstall Velero on cluster %s", veleroCfg.DefaultClusterContext), func() { ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5) defer ctxCancel() @@ -105,11 +110,8 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) DeleteNamespace(context.Background(), *veleroCfg.StandbyClient, migrationNamespace, true) }) } - By(fmt.Sprintf("Switch to default kubeconfig context %s", veleroCfg.DefaultClusterContext), func() { - Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed()) - veleroCfg.ClientToInstallVelero = veleroCfg.DefaultClient - veleroCfg.ClusterToInstallVelero = veleroCfg.DefaultClusterName - }) + } else { + fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.") } }) When("kibishii is the sample workload", func() { diff --git a/test/e2e/resourcemodifiers/resource_modifiers.go b/test/e2e/resourcemodifiers/resource_modifiers.go index 745963c48b..6bbef7b677 100644 --- a/test/e2e/resourcemodifiers/resource_modifiers.go +++ b/test/e2e/resourcemodifiers/resource_modifiers.go @@ -131,7 +131,7 @@ func (r *ResourceModifiersCase) Verify() error { func (r *ResourceModifiersCase) Clean() error { // If created some resources which is not in current test namespace, we NEED to override the base Clean function - if !r.VeleroCfg.Debug { + if !r.VeleroCfg.FailFast { if err := DeleteConfigmap(r.Client.ClientGo, r.VeleroCfg.VeleroNamespace, r.cmName); err != nil { return err } diff --git a/test/e2e/resourcepolicies/resource_policies.go b/test/e2e/resourcepolicies/resource_policies.go index 62d495abb3..6dc632989a 100644 --- a/test/e2e/resourcepolicies/resource_policies.go +++ b/test/e2e/resourcepolicies/resource_policies.go @@ -178,7 +178,7 @@ func (r *ResourcePoliciesCase) Verify() error { func (r *ResourcePoliciesCase) Clean() error { // If created some resources which is not in current test namespace, we NEED to override the base Clean function - if !r.VeleroCfg.Debug { + if !CurrentSpecReport().Failed() || !r.VeleroCfg.FailFast { if err := r.deleteTestStorageClassList([]string{StorageClassName, StorageClassName2}); err != nil { return err } @@ -188,6 +188,8 @@ func (r *ResourcePoliciesCase) Clean() error { } return r.GetTestCase().Clean() // only clean up resources in test namespace + } else { + fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.") } return nil } diff --git a/test/e2e/schedule/ordered_resources.go b/test/e2e/schedule/ordered_resources.go index b6a6a6e428..5ba568f1ce 100644 --- a/test/e2e/schedule/ordered_resources.go +++ b/test/e2e/schedule/ordered_resources.go @@ -153,10 +153,13 @@ func (o *OrderedResources) Verify() error { } func (o *OrderedResources) Clean() error { - if !o.VeleroCfg.Debug { + if !CurrentSpecReport().Failed() || !o.VeleroCfg.FailFast { Expect(VeleroScheduleDelete(o.Ctx, o.VeleroCfg.VeleroCLI, o.VeleroCfg.VeleroNamespace, o.ScheduleName)).To(Succeed()) Expect(o.TestCase.Clean()).To(Succeed()) + } else { + fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.") } + return nil } diff --git a/test/e2e/schedule/schedule-backup-creation.go b/test/e2e/schedule/schedule-backup-creation.go index 8ece31292d..058bd28728 100644 --- a/test/e2e/schedule/schedule-backup-creation.go +++ b/test/e2e/schedule/schedule-backup-creation.go @@ -32,63 +32,63 @@ type ScheduleBackupCreation struct { var ScheduleBackupCreationTest func() = TestFunc(&ScheduleBackupCreation{}) -func (n *ScheduleBackupCreation) Init() error { - n.TestCase.Init() - n.CaseBaseName = "schedule-backup-creation-test" + n.UUIDgen - n.ScheduleName = "schedule-" + n.CaseBaseName - n.namespace = n.GetTestCase().CaseBaseName - n.Period = 3 // Unit is minute - n.verifyTimes = 5 // More larger verify times more confidence we have +func (s *ScheduleBackupCreation) Init() error { + s.TestCase.Init() + s.CaseBaseName = "schedule-backup-creation-test" + s.UUIDgen + s.ScheduleName = "schedule-" + s.CaseBaseName + s.namespace = s.GetTestCase().CaseBaseName + s.Period = 3 // Unit is minute + s.verifyTimes = 5 // More larger verify times more confidence we have podSleepDurationStr := "300s" - n.podSleepDuration, _ = time.ParseDuration(podSleepDurationStr) - n.TestMsg = &TestMSG{ + s.podSleepDuration, _ = time.ParseDuration(podSleepDurationStr) + s.TestMsg = &TestMSG{ Desc: "Schedule controller wouldn't create a new backup when it still has pending or InProgress backup", FailedMSG: "Failed to verify schedule back creation behavior", Text: "Schedule controller wouldn't create a new backup when it still has pending or InProgress backup", } - n.podAnn = map[string]string{ - "pre.hook.backup.velero.io/container": n.podName, + s.podAnn = map[string]string{ + "pre.hook.backup.velero.io/container": s.podName, "pre.hook.backup.velero.io/command": "[\"sleep\", \"" + podSleepDurationStr + "\"]", "pre.hook.backup.velero.io/timeout": "600s", } - n.volume = "volume-1" - n.podName = "pod-1" - n.pvcName = "pvc-1" - n.ScheduleArgs = []string{ - "--include-namespaces", n.namespace, - "--schedule=*/" + fmt.Sprintf("%v", n.Period) + " * * * *", + s.volume = "volume-1" + s.podName = "pod-1" + s.pvcName = "pvc-1" + s.ScheduleArgs = []string{ + "--include-namespaces", s.namespace, + "--schedule=*/" + fmt.Sprintf("%v", s.Period) + " * * * *", } - Expect(n.Period).To(BeNumerically("<", 30)) + Expect(s.Period).To(BeNumerically("<", 30)) return nil } -func (p *ScheduleBackupCreation) CreateResources() error { - By(fmt.Sprintf("Create namespace %s", p.namespace), func() { - Expect(CreateNamespace(p.Ctx, p.Client, p.namespace)).To(Succeed(), - fmt.Sprintf("Failed to create namespace %s", p.namespace)) +func (s *ScheduleBackupCreation) CreateResources() error { + By(fmt.Sprintf("Create namespace %s", s.namespace), func() { + Expect(CreateNamespace(s.Ctx, s.Client, s.namespace)).To(Succeed(), + fmt.Sprintf("Failed to create namespace %s", s.namespace)) }) - By(fmt.Sprintf("Create pod %s in namespace %s", p.podName, p.namespace), func() { - _, err := CreatePod(p.Client, p.namespace, p.podName, "default", p.pvcName, []string{p.volume}, nil, p.podAnn) + By(fmt.Sprintf("Create pod %s in namespace %s", s.podName, s.namespace), func() { + _, err := CreatePod(s.Client, s.namespace, s.podName, "default", s.pvcName, []string{s.volume}, nil, s.podAnn) Expect(err).To(Succeed()) - err = WaitForPods(p.Ctx, p.Client, p.namespace, []string{p.podName}) + err = WaitForPods(s.Ctx, s.Client, s.namespace, []string{s.podName}) Expect(err).To(Succeed()) }) return nil } -func (n *ScheduleBackupCreation) Backup() error { +func (s *ScheduleBackupCreation) Backup() error { // Wait until the beginning of the given period to create schedule, it will give us // a predictable period to wait for the first scheduled backup, and verify no immediate // scheduled backup was created between schedule creation and first scheduled backup. - By(fmt.Sprintf("Creating schedule %s ......\n", n.ScheduleName), func() { - for i := 0; i < n.Period*60/30; i++ { + By(fmt.Sprintf("Creating schedule %s ......\n", s.ScheduleName), func() { + for i := 0; i < s.Period*60/30; i++ { time.Sleep(30 * time.Second) now := time.Now().Minute() - triggerNow := now % n.Period + triggerNow := now % s.Period if triggerNow == 0 { - Expect(VeleroScheduleCreate(n.Ctx, n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace, n.ScheduleName, n.ScheduleArgs)).To(Succeed(), func() string { - RunDebug(context.Background(), n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace, "", "") + Expect(VeleroScheduleCreate(s.Ctx, s.VeleroCfg.VeleroCLI, s.VeleroCfg.VeleroNamespace, s.ScheduleName, s.ScheduleArgs)).To(Succeed(), func() string { + RunDebug(context.Background(), s.VeleroCfg.VeleroCLI, s.VeleroCfg.VeleroNamespace, "", "") return "Fail to create schedule" }) break @@ -100,13 +100,13 @@ func (n *ScheduleBackupCreation) Backup() error { time.Sleep(1 * time.Minute) }) - By(fmt.Sprintf("Get backups every %d minute, and backups count should increase 1 more step in the same pace\n", n.Period), func() { - for i := 1; i <= n.verifyTimes; i++ { - fmt.Printf("Start to sleep %d minute #%d time...\n", n.podSleepDuration, i) + By(fmt.Sprintf("Get backups every %d minute, and backups count should increase 1 more step in the same pace\n", s.Period), func() { + for i := 1; i <= s.verifyTimes; i++ { + fmt.Printf("Start to sleep %d minute #%d time...\n", s.podSleepDuration, i) mi, _ := time.ParseDuration("60s") - time.Sleep(n.podSleepDuration + mi) + time.Sleep(s.podSleepDuration + mi) bMap := make(map[string]string) - backupsInfo, err := GetScheduledBackupsCreationTime(n.Ctx, n.VeleroCfg.VeleroCLI, "default", n.ScheduleName) + backupsInfo, err := GetScheduledBackupsCreationTime(s.Ctx, s.VeleroCfg.VeleroCLI, "default", s.ScheduleName) Expect(err).To(Succeed()) Expect(backupsInfo).To(HaveLen(i)) for index, bi := range backupsInfo { @@ -116,19 +116,22 @@ func (n *ScheduleBackupCreation) Backup() error { _, err := time.Parse("2006-01-02 15:04:05 -0700 MST", bList[1]) Expect(err).To(Succeed()) } - if i == n.verifyTimes-1 { + if i == s.verifyTimes-1 { backupInfo := backupsInfo[rand.Intn(len(backupsInfo))] - n.randBackupName = strings.Split(backupInfo, ",")[0] + s.randBackupName = strings.Split(backupInfo, ",")[0] } } }) return nil } -func (n *ScheduleBackupCreation) Clean() error { - if !n.VeleroCfg.Debug { - Expect(VeleroScheduleDelete(n.Ctx, n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace, n.ScheduleName)).To(Succeed()) - Expect(n.TestCase.Clean()).To(Succeed()) +func (s *ScheduleBackupCreation) Clean() error { + if !CurrentSpecReport().Failed() || !s.VeleroCfg.FailFast { + Expect(VeleroScheduleDelete(s.Ctx, s.VeleroCfg.VeleroCLI, s.VeleroCfg.VeleroNamespace, s.ScheduleName)).To(Succeed()) + Expect(s.TestCase.Clean()).To(Succeed()) + } else { + fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.") } + return nil } diff --git a/test/e2e/schedule/schedule.go b/test/e2e/schedule/schedule.go index b728623aa7..ec1d4c05d1 100644 --- a/test/e2e/schedule/schedule.go +++ b/test/e2e/schedule/schedule.go @@ -203,9 +203,12 @@ func (n *ScheduleBackup) Verify() error { } func (n *ScheduleBackup) Clean() error { - if !n.VeleroCfg.Debug { + if !CurrentSpecReport().Failed() || !n.VeleroCfg.FailFast { Expect(VeleroScheduleDelete(n.Ctx, n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace, n.ScheduleName)).To(Succeed()) Expect(n.TestCase.Clean()).To(Succeed()) + } else { + fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.") } + return nil } diff --git a/test/e2e/test/test.go b/test/e2e/test/test.go index f46a147609..0aabbb8ed3 100644 --- a/test/e2e/test/test.go +++ b/test/e2e/test/test.go @@ -182,7 +182,7 @@ func (t *TestCase) Start() error { func (t *TestCase) Clean() error { veleroCfg := t.GetTestCase().VeleroCfg - if !veleroCfg.Debug { + if !CurrentSpecReport().Failed() || !veleroCfg.FailFast { By(fmt.Sprintf("Clean namespace with prefix %s after test", t.CaseBaseName), func() { if err := CleanupNamespaces(t.Ctx, t.Client, t.CaseBaseName); err != nil { fmt.Println("Fail to cleanup namespaces: ", err) @@ -194,6 +194,8 @@ func (t *TestCase) Clean() error { fmt.Println("Fail to clean backups after test: ", err) } }) + } else { + fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.") } return nil } diff --git a/test/e2e/upgrade/upgrade.go b/test/e2e/upgrade/upgrade.go index 0f33d2eff0..bcc86308a5 100644 --- a/test/e2e/upgrade/upgrade.go +++ b/test/e2e/upgrade/upgrade.go @@ -84,7 +84,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC } }) AfterEach(func() { - if !veleroCfg.Debug { + if !CurrentSpecReport().Failed() || !veleroCfg.FailFast { By("Clean backups after test", func() { DeleteAllBackups(context.Background(), &veleroCfg) }) @@ -99,6 +99,8 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC veleroCfg.VeleroNamespace)).To(Succeed()) }) } + } else { + fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.") } }) When("kibishii is the sample workload", func() { diff --git a/test/perf/e2e_suite_test.go b/test/perf/e2e_suite_test.go index e047749c22..5239a50e43 100644 --- a/test/perf/e2e_suite_test.go +++ b/test/perf/e2e_suite_test.go @@ -69,7 +69,6 @@ func init() { //vmware-tanzu-experiments flag.StringVar(&VeleroCfg.Features, "features", "", "Comma-separated list of features to enable for this Velero process.") flag.StringVar(&VeleroCfg.DefaultClusterContext, "default-cluster-context", "", "Default cluster context for migration test.") - flag.BoolVar(&VeleroCfg.Debug, "debug-e2e-test", true, "Switch to control namespace cleaning.") flag.StringVar(&VeleroCfg.UploaderType, "uploader-type", "kopia", "Identify persistent volume backup uploader.") flag.BoolVar(&VeleroCfg.VeleroServerDebugMode, "velero-server-debug-mode", false, "Identify persistent volume backup uploader.") flag.StringVar(&VeleroCfg.NFSServerPath, "nfs-server-path", "", "the path of nfs server") @@ -77,6 +76,7 @@ func init() { flag.StringVar(&VeleroCfg.BackupForRestore, "backup-for-restore", "", "the name of backup for restore") flag.BoolVar(&VeleroCfg.DeleteClusterResource, "delete-cluster-resource", false, "delete cluster resource after test") flag.BoolVar(&VeleroCfg.DebugVeleroPodRestart, "debug-velero-pod-restart", false, "Switch for debugging velero pod restart.") + flag.BoolVar(&VeleroCfg.FailFast, "fail-fast", true, "a switch for failing fast on meeting error.") } func initConfig() error { @@ -128,8 +128,14 @@ var _ = BeforeSuite(func() { var _ = AfterSuite(func() { Expect(report.GenerateYamlReport()).To(Succeed()) - if InstallVelero && !VeleroCfg.Debug { + // If the Velero is installed during test, and the FailFast is not enabled, + // uninstall Velero. If not, either Velero is not installed, or kept it for debug. + if InstallVelero && (!CurrentSpecReport().Failed() || !VeleroCfg.FailFast) { By("release test resources after testing") - Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To(Succeed()) + ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5) + defer ctxCancel() + Expect(VeleroUninstall(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To(Succeed()) + } else { + fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.") } }) diff --git a/test/perf/test/test.go b/test/perf/test/test.go index 36f46bbf54..1bc8a3fc06 100644 --- a/test/perf/test/test.go +++ b/test/perf/test/test.go @@ -161,7 +161,7 @@ func (t *TestCase) Verify() error { } func (t *TestCase) Clean() error { - if !VeleroCfg.Debug || VeleroCfg.DeleteClusterResource { + if (!CurrentSpecReport().Failed() || !VeleroCfg.FailFast) || VeleroCfg.DeleteClusterResource { By("Clean backups and restore after test", func() { if len(t.BackupArgs) != 0 { if err := VeleroBackupDelete(t.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, t.BackupName); err != nil { diff --git a/test/types.go b/test/types.go index dc1b100d92..083fcdbbad 100644 --- a/test/types.go +++ b/test/types.go @@ -84,7 +84,6 @@ type VeleroConfig struct { Plugins string AddBSLPlugins string KibishiiDirectory string - Debug bool GCFrequency string DefaultClusterContext string StandbyClusterContext string @@ -110,6 +109,7 @@ type VeleroConfig struct { StandbyCLSServiceAccountName string ServiceAccountNameToInstall string EKSPolicyARN string + FailFast bool } type VeleroCfgInPerf struct { diff --git a/test/util/kibishii/kibishii_utils.go b/test/util/kibishii/kibishii_utils.go index 2fe223ea96..a749538d72 100644 --- a/test/util/kibishii/kibishii_utils.go +++ b/test/util/kibishii/kibishii_utils.go @@ -23,6 +23,7 @@ import ( "strings" "time" + . "github.com/onsi/ginkgo/v2" "github.com/pkg/errors" "golang.org/x/net/context" "k8s.io/apimachinery/pkg/util/wait" @@ -86,7 +87,7 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", kibishiiNamespace) } defer func() { - if !veleroCfg.Debug { + if !CurrentSpecReport().Failed() || !veleroCfg.FailFast { if err := DeleteNamespace(context.Background(), client, kibishiiNamespace, true); err != nil { fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", kibishiiNamespace)) } diff --git a/test/util/velero/velero_utils.go b/test/util/velero/velero_utils.go index e9f96060b0..cbb95a849c 100644 --- a/test/util/velero/velero_utils.go +++ b/test/util/velero/velero_utils.go @@ -98,19 +98,19 @@ var pluginsMatrix = map[string]map[string][]string{ "csi": {"velero/velero-plugin-for-csi:v0.6.0"}, }, "v1.13": { - "aws": {"velero/velero-plugin-for-aws:v1.9.0"}, - "azure": {"velero/velero-plugin-for-microsoft-azure:v1.9.0"}, + "aws": {"velero/velero-plugin-for-aws:v1.9.2"}, + "azure": {"velero/velero-plugin-for-microsoft-azure:v1.9.2"}, "vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.2"}, - "gcp": {"velero/velero-plugin-for-gcp:v1.9.0"}, - "csi": {"velero/velero-plugin-for-csi:v0.7.0"}, - "datamover": {"velero/velero-plugin-for-aws:v1.9.0"}, + "gcp": {"velero/velero-plugin-for-gcp:v1.9.2"}, + "csi": {"velero/velero-plugin-for-csi:v0.7.1"}, + "datamover": {"velero/velero-plugin-for-aws:v1.9.2"}, }, "v1.14": { - "aws": {"velero/velero-plugin-for-aws:v1.10.0"}, - "azure": {"velero/velero-plugin-for-microsoft-azure:v1.10.0"}, + "aws": {"velero/velero-plugin-for-aws:v1.10.1"}, + "azure": {"velero/velero-plugin-for-microsoft-azure:v1.10.1"}, "vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.2"}, - "gcp": {"velero/velero-plugin-for-gcp:v1.10.0"}, - "datamover": {"velero/velero-plugin-for-aws:v1.10.0"}, + "gcp": {"velero/velero-plugin-for-gcp:v1.10.1"}, + "datamover": {"velero/velero-plugin-for-aws:v1.10.1"}, }, "main": { "aws": {"velero/velero-plugin-for-aws:main"},