Upgrade e2e test for latest Kubernetes versions

Signed-off-by: Martin Hickey <[email protected]>

test/common.sh

@@ -27,7 +27,7 @@ if [ -n "$TRACE" ]; then
   set -x
 fi
 
-k8s_version=1.16.4
+k8s_version=1.19.2
 goarch=amd64
 
 if [[ "$OSTYPE" == "linux-gnu" ]]; then

test/e2e/local.sh

@@ -18,7 +18,7 @@ source "$(dirname "$0")/../common.sh"
 source "$(dirname "$0")/setup.sh"
 
 export KIND_CLUSTER="local-kubebuilder-e2e"
-create_cluster ${KIND_K8S_VERSION:-v1.18.15}
+create_cluster ${KIND_K8S_VERSION:-v1.22.1}
 if [ -z "${SKIP_KIND_CLEANUP:-}" ]; then
   trap delete_cluster EXIT
 fi

test/e2e/utils/test_context.go

@@ -106,7 +106,8 @@ func (t *TestContext) Prepare() error {
 
 const (
 	certmanagerVersionWithv1beta2CRs = "v0.11.0"
-	certmanagerVersion               = "v1.0.4"
+	certmanagerLegacyVersion         = "v1.0.4"
+	certmanagerVersion               = "v1.5.3"
 
 	certmanagerURLTmplLegacy = "https://github.com/jetstack/cert-manager/releases/download/%s/cert-manager-legacy.yaml"
 	certmanagerURLTmpl       = "https://github.com/jetstack/cert-manager/releases/download/%s/cert-manager.yaml"
@@ -122,7 +123,7 @@ func (t *TestContext) makeCertManagerURL(hasv1beta1CRs bool) string {
 	// Determine which URL to use for a manifest bundle with v1 CRs.
 	// The most up-to-date bundle uses v1 CRDs, which were introduced in k8s v1.16.
 	if ver := t.K8sVersion.ServerVersion; ver.GetMajorInt() <= 1 && ver.GetMinorInt() < 16 {
-		return fmt.Sprintf(certmanagerURLTmplLegacy, certmanagerVersion)
+		return fmt.Sprintf(certmanagerURLTmplLegacy, certmanagerLegacyVersion)
 	}
 	return fmt.Sprintf(certmanagerURLTmpl, certmanagerVersion)
 }
@@ -158,20 +159,33 @@ func (t *TestContext) UninstallCertManager(hasv1beta1CRs bool) {
 }
 
 const (
-	prometheusOperatorVersion = "0.33"
-	prometheusOperatorURL     = "https://raw.githubusercontent.com/coreos/prometheus-operator/release-%s/bundle.yaml"
+	prometheusOperatorLegacyVersion = "0.33"
+	prometheusOperatorLegacyURL     = "https://raw.githubusercontent.com/coreos/prometheus-operator/release-%s/bundle.yaml"
+	prometheusOperatorVersion       = "0.51"
+	prometheusOperatorURL           = "https://raw.githubusercontent.com/prometheus-operator/" +
+		"prometheus-operator/release-%s/bundle.yaml"
 )
 
 // InstallPrometheusOperManager installs the prometheus manager bundle.
 func (t *TestContext) InstallPrometheusOperManager() error {
-	url := fmt.Sprintf(prometheusOperatorURL, prometheusOperatorVersion)
+	var url string
+	if ver := t.K8sVersion.ServerVersion; ver.GetMajorInt() <= 1 && ver.GetMinorInt() < 16 {
+		url = fmt.Sprintf(prometheusOperatorLegacyURL, prometheusOperatorLegacyVersion)
+	} else {
+		url = fmt.Sprintf(prometheusOperatorURL, prometheusOperatorVersion)
+	}
 	_, err := t.Kubectl.Apply(false, "-f", url)
 	return err
 }
 
 // UninstallPrometheusOperManager uninstalls the prometheus manager bundle.
 func (t *TestContext) UninstallPrometheusOperManager() {
-	url := fmt.Sprintf(prometheusOperatorURL, prometheusOperatorVersion)
+	var url string
+	if ver := t.K8sVersion.ServerVersion; ver.GetMajorInt() <= 1 && ver.GetMinorInt() < 16 {
+		url = fmt.Sprintf(prometheusOperatorLegacyURL, prometheusOperatorLegacyVersion)
+	} else {
+		url = fmt.Sprintf(prometheusOperatorURL, prometheusOperatorVersion)
+	}
 	if _, err := t.Kubectl.Delete(false, "-f", url); err != nil {
 		fmt.Fprintf(GinkgoWriter, "error when running kubectl delete during cleaning up prometheus bundle: %v\n", err)
 	}

test/e2e/v2/plugin_cluster_test.go

@@ -46,30 +46,41 @@ var _ = Describe("kubebuilder", func() {
 			Expect(err).NotTo(HaveOccurred())
 			Expect(kbc.Prepare()).To(Succeed())
 
-			// Install cert-manager with v1beta2 CRs.
-			By("installing cert manager bundle")
-			Expect(kbc.InstallCertManager(true)).To(Succeed())
-
-			By("installing prometheus operator")
-			Expect(kbc.InstallPrometheusOperManager()).To(Succeed())
+			// Skip if cluster version >= 1.22 because pre v1 CRDs and webhooks no longer exist.
+			if srvVer := kbc.K8sVersion.ServerVersion; srvVer.GetMajorInt() <= 1 && srvVer.GetMinorInt() < 22 {
+				// Install cert-manager with v1beta2 CRs.
+				By("installing cert manager bundle")
+				Expect(kbc.InstallCertManager(true)).To(Succeed())
+
+				By("installing prometheus operator")
+				Expect(kbc.InstallPrometheusOperManager()).To(Succeed())
+			}
 		})
 
 		AfterEach(func() {
 			By("clean up created API objects during test process")
 			kbc.CleanupManifests(filepath.Join("config", "default"))
 
-			By("uninstalling prometheus manager bundle")
-			kbc.UninstallPrometheusOperManager()
+			// Skip if cluster version >= 1.22 because pre v1 CRDs and webhooks no longer exist.
+			if srvVer := kbc.K8sVersion.ServerVersion; srvVer.GetMajorInt() <= 1 && srvVer.GetMinorInt() < 22 {
+				By("uninstalling prometheus manager bundle")
+				kbc.UninstallPrometheusOperManager()
 
-			// Uninstall cert-manager with v1beta2 CRs.
-			By("uninstalling cert manager bundle")
-			kbc.UninstallCertManager(true)
+				// Uninstall cert-manager with v1beta2 CRs.
+				By("uninstalling cert manager bundle")
+				kbc.UninstallCertManager(true)
+			}
 
 			By("remove container image and work dir")
 			kbc.Destroy()
 		})
 
 		It("should generate a runnable project", func() {
+			// Skip if cluster version >= 1.22, when pre v1 CRDs and webhooks did not exist.
+			if srvVer := kbc.K8sVersion.ServerVersion; srvVer.GetMajorInt() <= 1 && srvVer.GetMinorInt() >= 22 {
+				Skip(fmt.Sprintf("cluster version %s does not support "+
+					"pre v1 CRDs or webhooks", srvVer.GitVersion))
+			}
 			var controllerPodName string
 			By("init v2 project")
 			err := kbc.Init(
@@ -218,8 +229,9 @@ var _ = Describe("kubebuilder", func() {
 
 			By("creating a pod with curl image")
 			cmdOpts := []string{
-				"run", "--generator=run-pod/v1", "curl", "--image=curlimages/curl:7.68.0", "--restart=OnFailure", "--",
-				"curl", "-v", "-k", "-H", fmt.Sprintf(`Authorization: Bearer %s`, token),
+				"run", "curl", "--image=curlimages/curl:7.68.0", "--restart=Never", "--",
+				"curl", "-v", "-k", "-H",
+				fmt.Sprintf(`Authorization: Bearer %s`, token),
 				fmt.Sprintf("https://e2e-%v-controller-manager-metrics-service.e2e-%v-system.svc:8443/metrics",
 					kbc.TestSuffix, kbc.TestSuffix),
 			}

test/e2e/v3/plugin_cluster_test.go

@@ -68,15 +68,26 @@ var _ = Describe("kubebuilder", func() {
 		Context("plugin go.kubebuilder.io/v2", func() {
 			// Use cert-manager with v1beta2 CRs.
 			BeforeEach(func() {
-				By("installing the v1beta2 cert-manager bundle")
-				Expect(kbc.InstallCertManager(true)).To(Succeed())
+				// Skip if cluster version >= 1.22 because pre v1 CRDs and webhooks no longer exist.
+				if srvVer := kbc.K8sVersion.ServerVersion; srvVer.GetMajorInt() <= 1 && srvVer.GetMinorInt() < 22 {
+					By("installing the v1beta2 cert-manager bundle")
+					Expect(kbc.InstallCertManager(true)).To(Succeed())
+				}
 			})
 			AfterEach(func() {
-				By("uninstalling the v1beta2 cert-manager bundle")
-				kbc.UninstallCertManager(true)
+				// Skip if cluster version >= 1.22 because pre v1 CRDs and webhooks no longer exist.
+				if srvVer := kbc.K8sVersion.ServerVersion; srvVer.GetMajorInt() <= 1 && srvVer.GetMinorInt() < 22 {
+					By("uninstalling the v1beta2 cert-manager bundle")
+					kbc.UninstallCertManager(true)
+				}
 			})
 
 			It("should generate a runnable project", func() {
+				// Skip if cluster version >= 1.22 because pre v1 CRDs and webhooks no longer exist.
+				if srvVer := kbc.K8sVersion.ServerVersion; srvVer.GetMajorInt() <= 1 && srvVer.GetMinorInt() >= 22 {
+					Skip(fmt.Sprintf("cluster version %s does not support pre v1 CRDs or webhooks", srvVer.GitVersion))
+				}
+
 				// go/v3 uses a unqiue-per-project service account name,
 				// while go/v2 still uses "default".
 				tmp := kbc.Kubectl.ServiceAccount
@@ -109,7 +120,9 @@ var _ = Describe("kubebuilder", func() {
 			})
 			It("should generate a runnable project with v1beta1 CRDs and Webhooks", func() {
 				// Skip if cluster version < 1.15, when `.spec.preserveUnknownFields` was not a v1beta1 CRD field.
-				if srvVer := kbc.K8sVersion.ServerVersion; srvVer.GetMajorInt() <= 1 && srvVer.GetMinorInt() < 16 {
+				// Skip if cluster version >= 1.22 because pre v1 CRDs and webhooks no longer exist.
+				if srvVer := kbc.K8sVersion.ServerVersion; srvVer.GetMajorInt() <= 1 && srvVer.GetMinorInt() < 16 ||
+					srvVer.GetMajorInt() <= 1 && srvVer.GetMinorInt() >= 22 {
 					Skip(fmt.Sprintf("cluster version %s does not support project defaults", srvVer.GitVersion))
 				}
 
@@ -307,7 +320,7 @@ func curlMetrics(kbc *utils.TestContext) string {
 
 	By("creating a curl pod")
 	cmdOpts := []string{
-		"run", "--generator=run-pod/v1", "curl", "--image=curlimages/curl:7.68.0", "--restart=OnFailure",
+		"run", "curl", "--image=curlimages/curl:7.68.0", "--restart=Never",
 		"--serviceaccount=" + kbc.Kubectl.ServiceAccount, "--",
 		"curl", "-v", "-k", "-H", fmt.Sprintf(`Authorization: Bearer %s`, token),
 		fmt.Sprintf("https://e2e-%s-controller-manager-metrics-service.%s.svc:8443/metrics",
@@ -328,7 +341,7 @@ func curlMetrics(kbc *utils.TestContext) string {
 		}
 		return nil
 	}
-	EventuallyWithOffset(2, verifyCurlUp, 30*time.Second, time.Second).Should(Succeed())
+	EventuallyWithOffset(2, verifyCurlUp, 240*time.Second, time.Second).Should(Succeed())
 
 	By("validating that the metrics endpoint is serving as expected")
 	var metricsOutput string