Merge pull request #2371 from hickeyma/feat/upg-e2e-tst-k8s-sup

🌱  Upgrade e2e test for latest Kubernetes versions

test/common.sh

@@ -27,7 +27,8 @@ if [ -n "$TRACE" ]; then
   set -x
 fi
 
-k8s_version=1.16.4
+k8s_version=1.19.2
+kind_version=0.11.1
 goarch=amd64
 
 if [[ "$OSTYPE" == "linux-gnu" ]]; then
@@ -105,14 +106,31 @@ function fetch_tools {
   export KUBEBUILDER_ASSETS=$kb_root_dir/bin/
 }
 
+# Check if version of tool is less than or equal a required version
+function is_ver_lessthanequal {
+    [  "$1" = "`echo -e \"$1\n$2\" | sort -V | head -n1`" ]
+}
+
+# Check if version of tool is less than a required version
+function is_ver_lessthan {
+    [ "$1" = "$2" ] && return 1 || is_ver_lessthanequal $1 $2
+}
+
 # Installing kind in a temporal dir if no previously installed
 function install_kind {
   header_text "Checking if kind is installed"
   if ! is_installed kind ; then
     header_text "Kind not found, installing kind"
     pushd $(mktemp -d)
-    GO111MODULE=on go get sigs.k8s.io/kind@v0.7.0
+    GO111MODULE=on go get sigs.k8s.io/kind@v$kind_version
     popd
+  else
+    if is_ver_lessthan `kind version -q` $kind_version ; then
+      header_text "Kind version less than v$kind_version, updating kind"
+      pushd $(mktemp -d)
+      GO111MODULE=on go get sigs.k8s.io/kind@v$kind_version
+      popd
+    fi
   fi
 }
 
@@ -123,3 +141,4 @@ function is_installed {
   fi
   return 1
 }
+

test/e2e/local.sh

@@ -18,7 +18,7 @@ source "$(dirname "$0")/../common.sh"
 source "$(dirname "$0")/setup.sh"
 
 export KIND_CLUSTER="local-kubebuilder-e2e"
-create_cluster ${KIND_K8S_VERSION:-v1.18.15}
+create_cluster ${KIND_K8S_VERSION:-v1.22.1}
 if [ -z "${SKIP_KIND_CLEANUP:-}" ]; then
   trap delete_cluster EXIT
 fi

test/e2e/utils/test_context.go

@@ -106,7 +106,8 @@ func (t *TestContext) Prepare() error {
 
 const (
 	certmanagerVersionWithv1beta2CRs = "v0.11.0"
-	certmanagerVersion               = "v1.0.4"
+	certmanagerLegacyVersion         = "v1.0.4"
+	certmanagerVersion               = "v1.5.3"
 
 	certmanagerURLTmplLegacy = "https://github.com/jetstack/cert-manager/releases/download/%s/cert-manager-legacy.yaml"
 	certmanagerURLTmpl       = "https://github.com/jetstack/cert-manager/releases/download/%s/cert-manager.yaml"
@@ -122,7 +123,7 @@ func (t *TestContext) makeCertManagerURL(hasv1beta1CRs bool) string {
 	// Determine which URL to use for a manifest bundle with v1 CRs.
 	// The most up-to-date bundle uses v1 CRDs, which were introduced in k8s v1.16.
 	if ver := t.K8sVersion.ServerVersion; ver.GetMajorInt() <= 1 && ver.GetMinorInt() < 16 {
-		return fmt.Sprintf(certmanagerURLTmplLegacy, certmanagerVersion)
+		return fmt.Sprintf(certmanagerURLTmplLegacy, certmanagerLegacyVersion)
 	}
 	return fmt.Sprintf(certmanagerURLTmpl, certmanagerVersion)
 }
@@ -158,20 +159,33 @@ func (t *TestContext) UninstallCertManager(hasv1beta1CRs bool) {
 }
 
 const (
-	prometheusOperatorVersion = "0.33"
-	prometheusOperatorURL     = "https://raw.githubusercontent.com/coreos/prometheus-operator/release-%s/bundle.yaml"
+	prometheusOperatorLegacyVersion = "0.33"
+	prometheusOperatorLegacyURL     = "https://raw.githubusercontent.com/coreos/prometheus-operator/release-%s/bundle.yaml"
+	prometheusOperatorVersion       = "0.51"
+	prometheusOperatorURL           = "https://raw.githubusercontent.com/prometheus-operator/" +
+		"prometheus-operator/release-%s/bundle.yaml"
 )
 
 // InstallPrometheusOperManager installs the prometheus manager bundle.
 func (t *TestContext) InstallPrometheusOperManager() error {
-	url := fmt.Sprintf(prometheusOperatorURL, prometheusOperatorVersion)
+	var url string
+	if ver := t.K8sVersion.ServerVersion; ver.GetMajorInt() <= 1 && ver.GetMinorInt() < 16 {
+		url = fmt.Sprintf(prometheusOperatorLegacyURL, prometheusOperatorLegacyVersion)
+	} else {
+		url = fmt.Sprintf(prometheusOperatorURL, prometheusOperatorVersion)
+	}
 	_, err := t.Kubectl.Apply(false, "-f", url)
 	return err
 }
 
 // UninstallPrometheusOperManager uninstalls the prometheus manager bundle.
 func (t *TestContext) UninstallPrometheusOperManager() {
-	url := fmt.Sprintf(prometheusOperatorURL, prometheusOperatorVersion)
+	var url string
+	if ver := t.K8sVersion.ServerVersion; ver.GetMajorInt() <= 1 && ver.GetMinorInt() < 16 {
+		url = fmt.Sprintf(prometheusOperatorLegacyURL, prometheusOperatorLegacyVersion)
+	} else {
+		url = fmt.Sprintf(prometheusOperatorURL, prometheusOperatorVersion)
+	}
 	if _, err := t.Kubectl.Delete(false, "-f", url); err != nil {
 		fmt.Fprintf(GinkgoWriter, "error when running kubectl delete during cleaning up prometheus bundle: %v\n", err)
 	}

test/e2e/v2/plugin_cluster_test.go

@@ -46,6 +46,12 @@ var _ = Describe("kubebuilder", func() {
 			Expect(err).NotTo(HaveOccurred())
 			Expect(kbc.Prepare()).To(Succeed())
 
+			// Skip if cluster version >= 1.22 because pre v1 CRDs and webhooks no longer exist.
+			if srvVer := kbc.K8sVersion.ServerVersion; srvVer.GetMajorInt() >= 1 && srvVer.GetMinorInt() >= 22 {
+				Skip(fmt.Sprintf("cluster version %s does not support "+
+					"pre v1 CRDs or webhooks", srvVer.GitVersion))
+			}
+
 			// Install cert-manager with v1beta2 CRs.
 			By("installing cert manager bundle")
 			Expect(kbc.InstallCertManager(true)).To(Succeed())
@@ -58,12 +64,15 @@ var _ = Describe("kubebuilder", func() {
 			By("clean up created API objects during test process")
 			kbc.CleanupManifests(filepath.Join("config", "default"))
 
-			By("uninstalling prometheus manager bundle")
-			kbc.UninstallPrometheusOperManager()
+			// Skip if cluster version >= 1.22 because pre v1 CRDs and webhooks no longer exist.
+			if srvVer := kbc.K8sVersion.ServerVersion; srvVer.GetMajorInt() <= 1 && srvVer.GetMinorInt() < 22 {
+				By("uninstalling prometheus manager bundle")
+				kbc.UninstallPrometheusOperManager()
 
-			// Uninstall cert-manager with v1beta2 CRs.
-			By("uninstalling cert manager bundle")
-			kbc.UninstallCertManager(true)
+				// Uninstall cert-manager with v1beta2 CRs.
+				By("uninstalling cert manager bundle")
+				kbc.UninstallCertManager(true)
+			}
 
 			By("remove container image and work dir")
 			kbc.Destroy()
@@ -218,8 +227,9 @@ var _ = Describe("kubebuilder", func() {
 
 			By("creating a pod with curl image")
 			cmdOpts := []string{
-				"run", "--generator=run-pod/v1", "curl", "--image=curlimages/curl:7.68.0", "--restart=OnFailure", "--",
-				"curl", "-v", "-k", "-H", fmt.Sprintf(`Authorization: Bearer %s`, token),
+				"run", "curl", "--image=curlimages/curl:7.68.0", "--restart=OnFailure", "--",
+				"curl", "-v", "-k", "-H",
+				fmt.Sprintf(`Authorization: Bearer %s`, token),
 				fmt.Sprintf("https://e2e-%v-controller-manager-metrics-service.e2e-%v-system.svc:8443/metrics",
 					kbc.TestSuffix, kbc.TestSuffix),
 			}

test/e2e/v3/plugin_cluster_test.go

@@ -68,10 +68,18 @@ var _ = Describe("kubebuilder", func() {
 		Context("plugin go.kubebuilder.io/v2", func() {
 			// Use cert-manager with v1beta2 CRs.
 			BeforeEach(func() {
+				// Skip if cluster version >= 1.22 because pre v1 CRDs and webhooks no longer exist.
+				if srvVer := kbc.K8sVersion.ServerVersion; srvVer.GetMajorInt() >= 1 && srvVer.GetMinorInt() >= 22 {
+					Skip(fmt.Sprintf("cluster version %s does not support pre v1 CRDs or webhooks", srvVer.GitVersion))
+				}
 				By("installing the v1beta2 cert-manager bundle")
 				Expect(kbc.InstallCertManager(true)).To(Succeed())
 			})
 			AfterEach(func() {
+				// Skip if cluster version >= 1.22 because pre v1 CRDs and webhooks no longer exist.
+				if srvVer := kbc.K8sVersion.ServerVersion; srvVer.GetMajorInt() >= 1 && srvVer.GetMinorInt() >= 22 {
+					Skip(fmt.Sprintf("cluster version %s does not support pre v1 CRDs or webhooks", srvVer.GitVersion))
+				}
 				By("uninstalling the v1beta2 cert-manager bundle")
 				kbc.UninstallCertManager(true)
 			})
@@ -109,7 +117,9 @@ var _ = Describe("kubebuilder", func() {
 			})
 			It("should generate a runnable project with v1beta1 CRDs and Webhooks", func() {
 				// Skip if cluster version < 1.15, when `.spec.preserveUnknownFields` was not a v1beta1 CRD field.
-				if srvVer := kbc.K8sVersion.ServerVersion; srvVer.GetMajorInt() <= 1 && srvVer.GetMinorInt() < 16 {
+				// Skip if cluster version >= 1.22 because pre v1 CRDs and webhooks no longer exist.
+				if srvVer := kbc.K8sVersion.ServerVersion; srvVer.GetMajorInt() <= 1 && srvVer.GetMinorInt() < 16 ||
+					srvVer.GetMajorInt() <= 1 && srvVer.GetMinorInt() >= 22 {
 					Skip(fmt.Sprintf("cluster version %s does not support project defaults", srvVer.GitVersion))
 				}
 
@@ -307,7 +317,7 @@ func curlMetrics(kbc *utils.TestContext) string {
 
 	By("creating a curl pod")
 	cmdOpts := []string{
-		"run", "--generator=run-pod/v1", "curl", "--image=curlimages/curl:7.68.0", "--restart=OnFailure",
+		"run", "curl", "--image=curlimages/curl:7.68.0", "--restart=OnFailure",
 		"--serviceaccount=" + kbc.Kubectl.ServiceAccount, "--",
 		"curl", "-v", "-k", "-H", fmt.Sprintf(`Authorization: Bearer %s`, token),
 		fmt.Sprintf("https://e2e-%s-controller-manager-metrics-service.%s.svc:8443/metrics",
@@ -328,7 +338,7 @@ func curlMetrics(kbc *utils.TestContext) string {
 		}
 		return nil
 	}
-	EventuallyWithOffset(2, verifyCurlUp, 30*time.Second, time.Second).Should(Succeed())
+	EventuallyWithOffset(2, verifyCurlUp, 240*time.Second, time.Second).Should(Succeed())
 
 	By("validating that the metrics endpoint is serving as expected")
 	var metricsOutput string