Refine ClusterSet controller to support ClusterSet version upgrade

Refine ClusterSet controller to watch ClusterClaims to handle the
ClusterSet upgrade from v1alpha1 to v1alpha2. When the ClusterID
is not in ClusterSet spec, controller will try to get ClusterID from
ClusterClaims.

Signed-off-by: Lan Luo <[email protected]>

docs/multicluster/upgrade.md

@@ -37,6 +37,41 @@ for the feature in new version.
 It should have no impact during upgrade to those imported resources like Service, Endpoints
 or AntreaClusterNetworkPolicy.
 
+## Upgrade from a version prior to v1.13
+
+Prior to Antrea v1.13, the `ClusterClaim` CRD is used to define both the Cluster ID and
+ClusterSet ID. Since Antrea v1.13, the `ClusterClaim` CRD is removed, and the `ClusterSet`
+CRD solely defines a ClusterSet. The name of a `ClusterSet` CR must match the ClusterSet ID,
+and a new `clusterID` field specifies the local Cluster ID.
+
+After upgrading Antrea Multi-cluster Controller from a version older than v1.13, the new version
+Multi-cluster Controller can still recognize and work with the old version `ClusterClaim` and
+`ClusterSet` CRs. However, we still suggest updating the `ClusterSet` CR to the new version after
+upgrading Multi-cluster Controller. You just need to update the existing `ClusterSet` CR and add the
+right `clusterID` to the spec. An example `ClusterSet` CR is like the following:
+
+```yaml
+apiVersion: multicluster.crd.antrea.io/v1alpha2
+kind: ClusterSet
+metadata:
+  name: test-clusterset # This value must match the ClusterSet ID.
+  namespace: kube-system
+spec:
+  clusterID: test-cluster-north # The new added field since v1.13.
+  leaders:
+    - clusterID: test-cluster-north
+      secret: "member-north-token"
+      server: "https://172.18.0.1:6443"
+  namespace: antrea-multicluster
+```
+
+You may also delete the `ClusterClaim` CRD after the upgrade, and then all existing `ClusterClaim`
+CRs will be removed automatically after the CRD is deleted.
+
+```bash
+kubectl delete crds clusterclaims.multicluster.crd.antrea.io
+```
+
 ## APIs deprecation policy
 
 The Antrea Multi-cluster APIs are built using K8s CustomResourceDefinitions and we

docs/multicluster/user-guide.md

@@ -33,6 +33,10 @@ through tunnels among clusters. The ClusterNetworkPolicy replication feature is
 supported since Antrea v1.6.0, and Multi-cluster NetworkPolicy rules are
 supported since Antrea v1.10.0.
 
+Antrea v1.13 promoted the ClusterSet CRD version from v1alpha1 to v1alpha2. If you
+plan to upgrade from a previous version to v1.13 or later, please check
+the [upgrade guide](./upgrade.md#upgrade-from-a-version-prior-to-v113).
+
 ## Quick Start
 
 Please refer to the [Quick Start Guide](quick-start.md) to learn how to build a

multicluster/cmd/multicluster-controller/controller.go

@@ -18,7 +18,9 @@ import (
 	"fmt"
 	"time"
 
+	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
@@ -171,6 +173,14 @@ func setupManagerAndCertController(isLeader bool, o *Options) (manager.Manager,
 		o.EnableEndpointSlice = true
 	}
 
+	// ClusterClaim CRD is removed since v1.13. Check the existence of
+	// ClusterClaim API before using ClusterClaim API.
+	clusterClaimCRDAvailable, err := clusterClaimCRDAvailable(client)
+	if err != nil {
+		return nil, fmt.Errorf("error checking if ClusterClaim API is available")
+	}
+	o.ClusterCalimCRDAvailable = clusterClaimCRDAvailable
+
 	mgr, err := ctrl.NewManager(k8sConfig, o.options)
 	if err != nil {
 		return nil, fmt.Errorf("error starting manager: %v", err)
@@ -192,3 +202,21 @@ func setupManagerAndCertController(isLeader bool, o *Options) (manager.Manager,
 	}
 	return mgr, nil
 }
+
+func clusterClaimCRDAvailable(k8sClient clientset.Interface) (bool, error) {
+	groupVersion := mcv1alpha2.SchemeGroupVersion.String()
+	resources, err := k8sClient.Discovery().ServerResourcesForGroupVersion(groupVersion)
+	if err != nil {
+		// The group version doesn't exist.
+		if errors.IsNotFound(err) {
+			return false, nil
+		}
+		return false, fmt.Errorf("error getting server resources for GroupVersion %s: %v", groupVersion, err)
+	}
+	for _, resource := range resources.APIResources {
+		if resource.Kind == "ClusterClaim" {
+			return true, nil
+		}
+	}
+	return false, nil
+}

multicluster/cmd/multicluster-controller/controller_test.go

@@ -23,8 +23,12 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes/fake"
 	"k8s.io/client-go/rest"
 
+	mcv1alpha2 "antrea.io/antrea/multicluster/apis/multicluster/v1alpha2"
 	"antrea.io/antrea/pkg/apiserver/certificate"
 )
 
@@ -93,3 +97,45 @@ func TestGetCAConfig(t *testing.T) {
 		})
 	}
 }
+
+func TestClusterClaimCRDAvailable(t *testing.T) {
+	groupVersion := mcv1alpha2.SchemeGroupVersion.String()
+	testCases := []struct {
+		name              string
+		resources         []*metav1.APIResourceList
+		expectedAvailable bool
+	}{
+		{
+			name:              "empty",
+			expectedAvailable: false,
+		},
+		{
+			name: "GroupVersion exists",
+			resources: []*metav1.APIResourceList{
+				{
+					GroupVersion: groupVersion,
+				},
+			},
+			expectedAvailable: false,
+		},
+		{
+			name: "API exists",
+			resources: []*metav1.APIResourceList{
+				{
+					GroupVersion: groupVersion,
+					APIResources: []metav1.APIResource{{Kind: "ClusterClaim"}},
+				},
+			},
+			expectedAvailable: true,
+		},
+	}
+	for _, tt := range testCases {
+		t.Run(tt.name, func(t *testing.T) {
+			k8sClient := fake.NewSimpleClientset()
+			k8sClient.Resources = tt.resources
+			available, err := clusterClaimCRDAvailable(k8sClient)
+			require.NoError(t, err)
+			assert.Equal(t, tt.expectedAvailable, available)
+		})
+	}
+}

multicluster/cmd/multicluster-controller/leader.go

@@ -77,9 +77,10 @@ func runLeader(o *Options) error {
 			namespace: env.GetPodNamespace()}})
 
 	clusterSetReconciler := &leader.LeaderClusterSetReconciler{
-		Client:        mgr.GetClient(),
-		Scheme:        mgr.GetScheme(),
-		StatusManager: memberClusterStatusManager,
+		Client:                   mgr.GetClient(),
+		Scheme:                   mgr.GetScheme(),
+		StatusManager:            memberClusterStatusManager,
+		ClusterCalimCRDAvailable: o.ClusterCalimCRDAvailable,
 	}
 	if err = clusterSetReconciler.SetupWithManager(mgr); err != nil {
 		return fmt.Errorf("error creating ClusterSet controller: %v", err)

multicluster/cmd/multicluster-controller/member.go

@@ -67,6 +67,7 @@ func runMember(o *Options) error {
 		mgr.GetScheme(),
 		env.GetPodNamespace(),
 		o.EnableStretchedNetworkPolicy,
+		o.ClusterCalimCRDAvailable,
 	)
 	if err = clusterSetReconciler.SetupWithManager(mgr); err != nil {
 		return fmt.Errorf("error creating ClusterSet controller: %v", err)

multicluster/cmd/multicluster-controller/options.go

@@ -46,6 +46,9 @@ type Options struct {
 	EnableStretchedNetworkPolicy bool
 	// Watch EndpointSlice API for exported Service if EndpointSlice API is available.
 	EnableEndpointSlice bool
+	// ClusterCalimCRDAvailable indicates if the ClusterClaim CRD is available or not
+	// in the cluster.
+	ClusterCalimCRDAvailable bool
 }
 
 func newOptions() *Options {

multicluster/controllers/multicluster/common/controller_utils.go

@@ -22,7 +22,10 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	mcv1alpha2 "antrea.io/antrea/multicluster/apis/multicluster/v1alpha2"
 )
 
 // DiscoverServiceCIDRByInvalidServiceCreation creates an invalid Service to get returned error, and analyzes
@@ -78,3 +81,40 @@ func parseServiceCIDRFromError(msg string) (string, error) {
 func NewClusterInfoResourceExportName(clusterID string) string {
 	return clusterID + "-clusterinfo"
 }
+
+func getClusterIDFromClusterClaim(c client.Client, clusterSet *mcv1alpha2.ClusterSet) (ClusterID, error) {
+	configNamespace := clusterSet.GetNamespace()
+
+	clusterClaimList := &mcv1alpha2.ClusterClaimList{}
+	if err := c.List(context.TODO(), clusterClaimList, client.InNamespace(configNamespace)); err != nil {
+		return "", err
+	}
+	if len(clusterClaimList.Items) == 0 {
+		return "", fmt.Errorf("ClusterClaim is not configured for the cluster")
+	}
+
+	for _, clusterClaim := range clusterClaimList.Items {
+		if clusterClaim.Name == mcv1alpha2.WellKnownClusterClaimID {
+			return ClusterID(clusterClaim.Value), nil
+		}
+	}
+
+	return "", fmt.Errorf("ClusterClaim not configured for Name=%s",
+		mcv1alpha2.WellKnownClusterClaimID)
+}
+
+func GetClusterID(clusterCalimCRDAvailable bool, req ctrl.Request, client client.Client, clusterSet *mcv1alpha2.ClusterSet) (ClusterID, error) {
+	if clusterSet.Spec.ClusterID == "" {
+		// ClusterID is a required feild, and the empty value case should only happen
+		// when Antrea Multi-cluster is upgraded from an old version prior to v1.13.
+		// Here we try to get the ClusterID from ClusterClaim before returning any error.
+		if clusterCalimCRDAvailable {
+			clusterID, err := getClusterIDFromClusterClaim(client, clusterSet)
+			if err == nil {
+				return clusterID, nil
+			}
+		}
+		return "", fmt.Errorf("'clusterID' is not set in the ClusterSet %s spec", req.NamespacedName)
+	}
+	return ClusterID(clusterSet.Spec.ClusterID), nil
+}