Enhance ClusterSet controller to support ClusterSet version upgrade

docs/multicluster/upgrade.md

@@ -37,6 +37,41 @@ for the feature in new version.
 It should have no impact during upgrade to those imported resources like Service, Endpoints
 or AntreaClusterNetworkPolicy.
 
+## Upgrade from a version prior to v1.13
+
+Prior to Antrea v1.13, the `ClusterClaim` CRD is used to define both the local Cluster ID and
+the ClusterSet ID. Since Antrea v1.13, the `ClusterClaim` CRD is removed, and the `ClusterSet`
+CRD solely defines a ClusterSet. The name of a `ClusterSet` CR must match the ClusterSet ID,
+and a new `clusterID` field specifies the local Cluster ID.
+
+After upgrading Antrea Multi-cluster Controller from a version older than v1.13, the new version
+Multi-cluster Controller can still recognize and work with the old version `ClusterClaim` and
+`ClusterSet` CRs. However, we still suggest updating the `ClusterSet` CR to the new version after
+upgrading Multi-cluster Controller. You just need to update the existing `ClusterSet` CR and add the
+right `clusterID` to the spec. An example `ClusterSet` CR is like the following:
+
+```yaml
+apiVersion: multicluster.crd.antrea.io/v1alpha2
+kind: ClusterSet
+metadata:
+  name: test-clusterset # This value must match the ClusterSet ID.
+  namespace: kube-system
+spec:
+  clusterID: test-cluster-north # The new added field since v1.13.
+  leaders:
+    - clusterID: test-cluster-north
+      secret: "member-north-token"
+      server: "https://172.18.0.1:6443"
+  namespace: antrea-multicluster
+```
+
+You may also delete the `ClusterClaim` CRD after the upgrade, and then all existing `ClusterClaim`
+CRs will be removed automatically after the CRD is deleted.
+
+```bash
+kubectl delete crds clusterclaims.multicluster.crd.antrea.io
+```
+
 ## APIs deprecation policy
 
 The Antrea Multi-cluster APIs are built using K8s CustomResourceDefinitions and we

docs/multicluster/user-guide.md

@@ -33,6 +33,10 @@ through tunnels among clusters. The ClusterNetworkPolicy replication feature is
 supported since Antrea v1.6.0, and Multi-cluster NetworkPolicy rules are
 supported since Antrea v1.10.0.
 
+Antrea v1.13 promoted the ClusterSet CRD version from v1alpha1 to v1alpha2. If you
+plan to upgrade from a previous version to v1.13 or later, please check
+the [upgrade guide](./upgrade.md#upgrade-from-a-version-prior-to-v113).
+
 ## Quick Start
 
 Please refer to the [Quick Start Guide](quick-start.md) to learn how to build a

multicluster/cmd/multicluster-controller/controller.go

@@ -18,7 +18,9 @@ import (
 	"fmt"
 	"time"
 
+	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
@@ -171,6 +173,14 @@ func setupManagerAndCertController(isLeader bool, o *Options) (manager.Manager,
 		o.EnableEndpointSlice = true
 	}
 
+	// ClusterClaim CRD is removed since v1.13. Check the existence of
+	// ClusterClaim API before using ClusterClaim API.
+	clusterClaimCRDAvailable, err := clusterClaimCRDAvailable(client)
+	if err != nil {
+		return nil, fmt.Errorf("error checking if ClusterClaim API is available")
+	}
+	o.ClusterCalimCRDAvailable = clusterClaimCRDAvailable
+
 	mgr, err := ctrl.NewManager(k8sConfig, o.options)
 	if err != nil {
 		return nil, fmt.Errorf("error starting manager: %v", err)
@@ -192,3 +202,21 @@ func setupManagerAndCertController(isLeader bool, o *Options) (manager.Manager,
 	}
 	return mgr, nil
 }
+
+func clusterClaimCRDAvailable(k8sClient clientset.Interface) (bool, error) {
+	groupVersion := mcv1alpha2.SchemeGroupVersion.String()
+	resources, err := k8sClient.Discovery().ServerResourcesForGroupVersion(groupVersion)
+	if err != nil {
+		// The group version doesn't exist.
+		if errors.IsNotFound(err) {
+			return false, nil
+		}
+		return false, fmt.Errorf("error getting server resources for GroupVersion %s: %v", groupVersion, err)
+	}
+	for _, resource := range resources.APIResources {
+		if resource.Kind == "ClusterClaim" {
+			return true, nil
+		}
+	}
+	return false, nil
+}

multicluster/cmd/multicluster-controller/controller_test.go

@@ -23,8 +23,12 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes/fake"
 	"k8s.io/client-go/rest"
 
+	mcv1alpha2 "antrea.io/antrea/multicluster/apis/multicluster/v1alpha2"
 	"antrea.io/antrea/pkg/apiserver/certificate"
 )
 
@@ -93,3 +97,45 @@ func TestGetCAConfig(t *testing.T) {
 		})
 	}
 }
+
+func TestClusterClaimCRDAvailable(t *testing.T) {
+	groupVersion := mcv1alpha2.SchemeGroupVersion.String()
+	testCases := []struct {
+		name              string
+		resources         []*metav1.APIResourceList
+		expectedAvailable bool
+	}{
+		{
+			name:              "empty",
+			expectedAvailable: false,
+		},
+		{
+			name: "GroupVersion exists",
+			resources: []*metav1.APIResourceList{
+				{
+					GroupVersion: groupVersion,
+				},
+			},
+			expectedAvailable: false,
+		},
+		{
+			name: "API exists",
+			resources: []*metav1.APIResourceList{
+				{
+					GroupVersion: groupVersion,
+					APIResources: []metav1.APIResource{{Kind: "ClusterClaim"}},
+				},
+			},
+			expectedAvailable: true,
+		},
+	}
+	for _, tt := range testCases {
+		t.Run(tt.name, func(t *testing.T) {
+			k8sClient := fake.NewSimpleClientset()
+			k8sClient.Resources = tt.resources
+			available, err := clusterClaimCRDAvailable(k8sClient)
+			require.NoError(t, err)
+			assert.Equal(t, tt.expectedAvailable, available)
+		})
+	}
+}

multicluster/cmd/multicluster-controller/leader.go

@@ -77,9 +77,10 @@ func runLeader(o *Options) error {
 			namespace: env.GetPodNamespace()}})
 
 	clusterSetReconciler := &leader.LeaderClusterSetReconciler{
-		Client:        mgr.GetClient(),
-		Scheme:        mgr.GetScheme(),
-		StatusManager: memberClusterStatusManager,
+		Client:                   mgr.GetClient(),
+		Scheme:                   mgr.GetScheme(),
+		StatusManager:            memberClusterStatusManager,
+		ClusterCalimCRDAvailable: o.ClusterCalimCRDAvailable,
 	}
 	if err = clusterSetReconciler.SetupWithManager(mgr); err != nil {
 		return fmt.Errorf("error creating ClusterSet controller: %v", err)

multicluster/cmd/multicluster-controller/member.go

@@ -67,6 +67,7 @@ func runMember(o *Options) error {
 		mgr.GetScheme(),
 		env.GetPodNamespace(),
 		o.EnableStretchedNetworkPolicy,
+		o.ClusterCalimCRDAvailable,
 	)
 	if err = clusterSetReconciler.SetupWithManager(mgr); err != nil {
 		return fmt.Errorf("error creating ClusterSet controller: %v", err)

multicluster/cmd/multicluster-controller/options.go

@@ -46,6 +46,9 @@ type Options struct {
 	EnableStretchedNetworkPolicy bool
 	// Watch EndpointSlice API for exported Service if EndpointSlice API is available.
 	EnableEndpointSlice bool
+	// ClusterCalimCRDAvailable indicates if the ClusterClaim CRD is available or not
+	// in the cluster.
+	ClusterCalimCRDAvailable bool
 }
 
 func newOptions() *Options {

multicluster/controllers/multicluster/common/controller_utils.go

@@ -22,7 +22,10 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	mcv1alpha2 "antrea.io/antrea/multicluster/apis/multicluster/v1alpha2"
 )
 
 // DiscoverServiceCIDRByInvalidServiceCreation creates an invalid Service to get returned error, and analyzes
@@ -78,3 +81,40 @@ func parseServiceCIDRFromError(msg string) (string, error) {
 func NewClusterInfoResourceExportName(clusterID string) string {
 	return clusterID + "-clusterinfo"
 }
+
+func getClusterIDFromClusterClaim(c client.Client, clusterSet *mcv1alpha2.ClusterSet) (ClusterID, error) {
+	configNamespace := clusterSet.GetNamespace()
+
+	clusterClaimList := &mcv1alpha2.ClusterClaimList{}
+	if err := c.List(context.TODO(), clusterClaimList, client.InNamespace(configNamespace)); err != nil {
+		return "", err
+	}
+	if len(clusterClaimList.Items) == 0 {
+		return "", fmt.Errorf("ClusterClaim is not configured for the cluster")
+	}
+
+	for _, clusterClaim := range clusterClaimList.Items {
+		if clusterClaim.Name == mcv1alpha2.WellKnownClusterClaimID {
+			return ClusterID(clusterClaim.Value), nil
+		}
+	}
+
+	return "", fmt.Errorf("ClusterClaim not configured for Name=%s",
+		mcv1alpha2.WellKnownClusterClaimID)
+}
+
+func GetClusterID(clusterCalimCRDAvailable bool, req ctrl.Request, client client.Client, clusterSet *mcv1alpha2.ClusterSet) (ClusterID, error) {
+	if clusterSet.Spec.ClusterID == "" {
+		// ClusterID is a required feild, and the empty value case should only happen
+		// when Antrea Multi-cluster is upgraded from an old version prior to v1.13.
+		// Here we try to get the ClusterID from ClusterClaim before returning any error.
+		if clusterCalimCRDAvailable {
+			clusterID, err := getClusterIDFromClusterClaim(client, clusterSet)
+			if err == nil {
+				return clusterID, nil
+			}
+		}
+		return "", fmt.Errorf("'clusterID' is not set in the ClusterSet %s spec", req.NamespacedName)
+	}
+	return ClusterID(clusterSet.Spec.ClusterID), nil
+}