Add detection for cert-manager CRDs v1beta1 and v1 (#1710)

* Add support to detect cert manager APIs in version v1beta1 and v1

Signed-off-by: Andreas Neumann <[email protected]>

pkg/kudoctl/cmd/init_test.go

@@ -325,7 +325,9 @@ func MockCRD(client *kube.Client, crdName string, apiVersion string) {
 					Spec: extv1beta1.CustomResourceDefinitionSpec{
 						Versions: []extv1beta1.CustomResourceDefinitionVersion{
 							{
-								Name: apiVersion,
+								Name:    apiVersion,
+								Served:  true,
+								Storage: true,
 							},
 						},
 					},

pkg/kudoctl/kudoinit/prereq/webhook.go

@@ -3,6 +3,7 @@ package prereq
 import (
 	"context"
 	"fmt"
+	"sort"
 	"strings"
 
 	"github.com/thoas/go-funk"
@@ -14,6 +15,7 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/version"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
 	clientv1beta1 "k8s.io/client-go/kubernetes/typed/admissionregistration/v1beta1"
@@ -52,11 +54,19 @@ const (
 var (
 	// Cert-Manager APIs that we can detect
 	certManagerAPIs = []certManagerVersion{
-		{group: "cert-manager.io", versions: []string{"v1alpha2", "v1alpha3"}}, // 0.11.0+
-		{group: "certmanager.k8s.io", versions: []string{"v1alpha1"}},          // 0.10.1
+		{group: "cert-manager.io", versions: []string{"v1", "v1beta1", "v1alpha3", "v1alpha2"}}, // 0.11.0+
+		{group: "certmanager.k8s.io", versions: []string{"v1alpha1"}},                           // 0.10.1
 	}
 )
 
+type kubeLikeVersions []string
+
+func (a kubeLikeVersions) Len() int      { return len(a) }
+func (a kubeLikeVersions) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a kubeLikeVersions) Less(i, j int) bool {
+	return version.CompareKubeAwareVersionStrings(a[i], a[j]) < 0
+}
+
 func NewWebHookInitializer(options kudoinit.Options) *KudoWebHook {
 	return &KudoWebHook{
 		opts: options,
@@ -253,9 +263,22 @@ func detectCertManagerCRD(extClient clientset.Interface, api certManagerVersion)
 	clog.V(4).Printf("Try to retrieve cert-manager CRD %s", testCRD)
 	crd, err := extClient.ApiextensionsV1beta1().CustomResourceDefinitions().Get(context.TODO(), testCRD, metav1.GetOptions{})
 	if err == nil {
-		// crd.Spec.Versions[0] must be the one that is stored and served, we should use that one
-		clog.V(4).Printf("Got CRD. Group: %s, Version: %s", api.group, crd.Spec.Versions[0].Name)
-		return api.group, crd.Spec.Versions[0].Name, nil
+		servedVersions := kubeLikeVersions{}
+		// Look through the versions and find the one that is the stored one
+		for _, v := range crd.Spec.Versions {
+			v := v
+			if v.Served {
+				servedVersions = append(servedVersions, v.Name)
+			}
+		}
+		if len(servedVersions) == 0 {
+			return "", "", fmt.Errorf("failed to detect cert manager CRD %s: no served version found", testCRD)
+		}
+		sort.Sort(servedVersions)
+		bestVersion := servedVersions[len(servedVersions)-1]
+
+		clog.V(4).Printf("Got CRD. Group: %s, Version: %s", api.group, bestVersion)
+		return api.group, bestVersion, nil
 	}
 	if !kerrors.IsNotFound(err) {
 		return "", "", fmt.Errorf("failed to detect cert manager CRD %s: %v", testCRD, err)
@@ -334,12 +357,20 @@ func validateCrdVersion(extClient clientset.Interface, crdName string, expectedV
 		}
 		return err
 	}
-	crdVersion := certCRD.Spec.Versions[0].Name
-
-	if crdVersion != expectedVersion {
-		result.AddErrors(fmt.Sprintf("invalid CRD version found for '%s': %s instead of %s", crdName, crdVersion, expectedVersion))
+	allVersions := []string{}
+	for _, v := range certCRD.Spec.Versions {
+		if v.Name == expectedVersion {
+			if v.Served {
+				clog.V(2).Printf("CRD %s is served with version %s", crdName, v.Name)
+				return nil
+			}
+			result.AddErrors(fmt.Sprintf("outdated CRD version found for '%s': %s is known but not served", crdName, v.Name))
+			return nil
+		}
+		allVersions = append(allVersions, v.Name)
 	}
-	clog.V(2).Printf("CRD %s is installed with version %s", crdName, crdVersion)
+
+	result.AddErrors(fmt.Sprintf("CRD versions for '%s' are %v, did not find expected version %s or it is not served", crdName, allVersions, expectedVersion))
 	return nil
 }
 

pkg/kudoctl/kudoinit/prereq/webhook_test.go

@@ -51,7 +51,7 @@ func TestPrereq_Fail_PreValidate_Webhook_WrongCertificateVersion(t *testing.T) {
 	_ = init.PreInstallVerify(client, &result)
 
 	assert.EqualValues(t, verifier.NewWarning(
-		"Detected cert-manager CRDs with version v0, only versions [v1alpha2 v1alpha3] are fully supported. Certificates for webhooks may not work.",
+		"Detected cert-manager CRDs with version v0, only versions [v1 v1beta1 v1alpha3 v1alpha2] are fully supported. Certificates for webhooks may not work.",
 	), result)
 }
 
@@ -67,7 +67,7 @@ func TestPrereq_Fail_PreValidate_Webhook_WrongCertManagerInstallation(t *testing
 	_ = init.PreInstallVerify(client, &result)
 
 	assert.EqualValues(t, verifier.NewError(
-		"invalid CRD version found for 'issuers.cert-manager.io': v0 instead of v1alpha2",
+		"CRD versions for 'issuers.cert-manager.io' are [v0], did not find expected version v1alpha2 or it is not served",
 	), result)
 }
 
@@ -95,7 +95,7 @@ func TestPrereq_Fail_PreValidate_Webhook_WrongIssuerVersion(t *testing.T) {
 	result := verifier.NewResult()
 	_ = init.PreInstallVerify(client, &result)
 
-	assert.EqualValues(t, verifier.NewError("invalid CRD version found for 'issuers.cert-manager.io': v0 instead of v1alpha2"), result)
+	assert.EqualValues(t, verifier.NewError("CRD versions for 'issuers.cert-manager.io' are [v0], did not find expected version v1alpha2 or it is not served"), result)
 }
 
 func TestPrereq_Ok_PreValidate_Webhook_CertManager_v1alpha2(t *testing.T) {
@@ -126,32 +126,81 @@ func TestPrereq_Ok_PreValidate_Webhook_CertManager_v1alpha1(t *testing.T) {
 	assert.Equal(t, 0, len(result.Errors))
 }
 
-func mockCRD(client *kube.Client, crdName string, apiVersion string) {
-	client.ExtClient.(*apiextensionsfake.Clientset).Fake.PrependReactor("get", "customresourcedefinitions", func(action testing2.Action) (handled bool, ret runtime.Object, err error) {
+func TestPrereq_Ok_PreValidate_Webhook_CertManager_MultipleVersions(t *testing.T) {
+	client := getFakeClient()
+
+	cert := createCrd("certificates.certmanager.k8s.io", "v1alpha1")
+	cert.Spec.Versions = append(cert.Spec.Versions, apiextensions.CustomResourceDefinitionVersion{
+		Name:    "v1beta1",
+		Served:  true,
+		Storage: false,
+	})
+	cert.Spec.Versions = append(cert.Spec.Versions, apiextensions.CustomResourceDefinitionVersion{
+		Name:    "v1beta2",
+		Served:  true,
+		Storage: false,
+	})
+
+	issuer := createCrd("issuers.certmanager.k8s.io", "v1alpha1")
+	issuer.Spec.Versions = append(cert.Spec.Versions, apiextensions.CustomResourceDefinitionVersion{
+		Name:    "v1beta1",
+		Served:  true,
+		Storage: false,
+	})
+	issuer.Spec.Versions = append(cert.Spec.Versions, apiextensions.CustomResourceDefinitionVersion{
+		Name:    "v1beta2",
+		Served:  true,
+		Storage: false,
+	})
 
+	mockFullCRD(client, cert)
+	mockFullCRD(client, issuer)
+
+	init := NewWebHookInitializer(kudoinit.NewOptions("", "", "", false, false))
+
+	result := verifier.NewResult()
+	_ = init.PreInstallVerify(client, &result)
+
+	assert.Equal(t, init.certManagerAPIVersion, "v1beta2")
+
+	assert.Equal(t, 0, len(result.Errors))
+}
+
+func mockFullCRD(client *kube.Client, definition *apiextensions.CustomResourceDefinition) {
+	client.ExtClient.(*apiextensionsfake.Clientset).Fake.PrependReactor("get", "customresourcedefinitions", func(action testing2.Action) (handled bool, ret runtime.Object, err error) {
 		getAction, _ := action.(testing2.GetAction)
 		if getAction != nil {
-			if getAction.GetName() == crdName {
-				crd := &apiextensions.CustomResourceDefinition{
-					TypeMeta: metav1.TypeMeta{
-						APIVersion: apiVersion,
-					},
-					ObjectMeta: metav1.ObjectMeta{
-						Name: crdName,
-					},
-					Spec: apiextensions.CustomResourceDefinitionSpec{
-						Versions: []apiextensions.CustomResourceDefinitionVersion{
-							{
-								Name: apiVersion,
-							},
-						},
-					},
-					Status: apiextensions.CustomResourceDefinitionStatus{},
-				}
-				return true, crd, nil
+			if getAction.GetName() == definition.Name {
+				return true, definition, nil
 			}
 		}
-
 		return false, nil, nil
 	})
 }
+
+func createCrd(crdName string, apiVersion string) *apiextensions.CustomResourceDefinition {
+	return &apiextensions.CustomResourceDefinition{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: apiVersion,
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: crdName,
+		},
+		Spec: apiextensions.CustomResourceDefinitionSpec{
+			Versions: []apiextensions.CustomResourceDefinitionVersion{
+				{
+					Name:    apiVersion,
+					Served:  true,
+					Storage: true,
+				},
+			},
+		},
+		Status: apiextensions.CustomResourceDefinitionStatus{},
+	}
+}
+
+func mockCRD(client *kube.Client, crdName string, apiVersion string) {
+	def := createCrd(crdName, apiVersion)
+	mockFullCRD(client, def)
+
+}

test/upgrade/cert-manager-detection/00-assert.yaml

@@ -0,0 +1,23 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: issuers.cert-manager.io
+spec:
+  versions:
+    - name: v1alpha2
+    - name: v1alpha3
+    - name: v1beta1
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cert-manager-webhook
+  namespace: cert-manager
+status:
+  readyReplicas: 1
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: cert-manager-webhook-ca
+  namespace: cert-manager

test/upgrade/cert-manager-detection/00-install-cert-manager-0-16-0.yaml

@@ -0,0 +1,4 @@
+apiVersion: kudo.dev/v1beta1
+kind: TestStep
+commands:
+  - command: kubectl apply --validate=false -f cert-manager/cert-manager-0.16.0.yaml

test/upgrade/cert-manager-detection/01-assert.yaml

@@ -0,0 +1,11 @@
+apiVersion: cert-manager.io/v1beta1
+kind: Issuer
+metadata:
+  name: selfsigned-issuer
+  namespace: kudo-system
+---
+apiVersion: cert-manager.io/v1beta1
+kind: Certificate
+metadata:
+  name: kudo-webhook-server-certificate
+  namespace: kudo-system

test/upgrade/cert-manager-detection/01-install-kudo.yaml

@@ -0,0 +1,5 @@
+apiVersion: kudo.dev/v1beta1
+kind: TestStep
+commands:
+  - command: sleep 10
+  - command: kubectl kudo init -v 4 --kudo-image kudobuilder/controller:test --kudo-image-pull-policy IfNotPresent --wait

test/upgrade/cert-manager-detection/02-errors.yaml

@@ -0,0 +1,26 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: instances.kudo.dev
+spec:
+  group: kudo.dev
+  names:
+    kind: Instance
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: operators.kudo.dev
+spec:
+  group: kudo.dev
+  names:
+    kind: Operator
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: operatorversions.kudo.dev
+spec:
+  group: kudo.dev
+  names:
+    kind: OperatorVersion

test/upgrade/cert-manager-detection/02-remove-kudo.yaml

@@ -0,0 +1,4 @@
+apiVersion: kudo.dev/v1beta1
+kind: TestStep
+commands:
+  - script: kubectl kudo init --upgrade --kudo-image kudobuilder/controller:test --kudo-image-pull-policy IfNotPresent --dry-run --output yaml | tee output.log | kubectl delete -f -

test/upgrade/cert-manager-detection/03-errors.yaml

@@ -0,0 +1,23 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: issuers.cert-manager.io
+spec:
+  versions:
+    - name: v1alpha2
+    - name: v1alpha3
+    - name: v1beta1
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cert-manager-webhook
+  namespace: cert-manager
+status:
+  readyReplicas: 1
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: cert-manager-webhook-ca
+  namespace: cert-manager

test/upgrade/cert-manager-detection/03-uninstall-cert-manager-0-16-0.yaml

@@ -0,0 +1,4 @@
+apiVersion: kudo.dev/v1beta1
+kind: TestStep
+commands:
+  - command: kubectl delete -f cert-manager/cert-manager-0.16.0.yaml

test/upgrade/cert-manager-detection/05-assert.yaml

@@ -0,0 +1,24 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: issuers.cert-manager.io
+spec:
+  versions:
+    - name: v1alpha2
+    - name: v1alpha3
+    - name: v1beta1
+    - name: v1
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cert-manager-webhook
+  namespace: cert-manager
+status:
+  readyReplicas: 1
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: cert-manager-webhook-ca
+  namespace: cert-manager

test/upgrade/cert-manager-detection/05-install-cert-manager-1-0-3.yaml

@@ -0,0 +1,4 @@
+apiVersion: kudo.dev/v1beta1
+kind: TestStep
+commands:
+  - command: kubectl apply --validate=false -f cert-manager/cert-manager-1.0.3.yaml

test/upgrade/cert-manager-detection/06-assert.yaml

@@ -0,0 +1,11 @@
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: selfsigned-issuer
+  namespace: kudo-system
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: kudo-webhook-server-certificate
+  namespace: kudo-system

test/upgrade/cert-manager-detection/06-install-kudo.yaml

@@ -0,0 +1,5 @@
+apiVersion: kudo.dev/v1beta1
+kind: TestStep
+commands:
+  - command: sleep 10
+  - command: kubectl kudo init -v 4 --kudo-image kudobuilder/controller:test --kudo-image-pull-policy IfNotPresent --wait