Refine ClusterSet controller to support ClusterSet version upgrade

Refine ClusterSet controller to watch ClusterClaims to handle the
ClusterSet upgrade from v1alpha1 to v1alpha2. When the ClusterID
is not in ClusterSet spec, controller will try to get ClusterID from
ClusterClaims.

Signed-off-by: Lan Luo <[email protected]>

docs/multicluster/upgrade.md

@@ -37,6 +37,43 @@ for the feature in new version.
 It should have no impact during upgrade to those imported resources like Service, Endpoints
 or AntreaClusterNetworkPolicy.
 
+## Upgrade from a version prior to v1.13
+
+Prior to Antrea v1.13, the `ClusterClaim` CRD is used to define both the Cluster ID and
+ClusterSet ID for a ClusterSet. Since Antrea v1.13, the `ClusterClaim` CRD is removed, and
+the `ClusterSet` CRD solely defines a ClusterSet. The name of a ClusterSet CR must match the
+ClusterSet ID, and a new `clusterID` field specifies the Cluster ID.
+
+To upgrade from a version older than v1.13, once you apply the latest manifests for an
+existing ClusterSet, you need to update the existing `ClusterSet` CR with the right `clusterID`
+in the spec. A sample ClusterSet CR is like following:
+
+```yaml
+apiVersion: multicluster.crd.antrea.io/v1alpha2
+kind: ClusterSet
+metadata:
+  name: test-clusterset # This value must match the ClusterSet ID.
+  namespace: kube-system
+spec:
+  clusterID: test-cluster-north # The new added field since v1.13.
+  leaders:
+    - clusterID: test-cluster-north
+      secret: "member-north-token"
+      server: "https://172.18.0.1:6443"
+  namespace: antrea-multicluster
+```
+
+Note: No action is required if there is already a Cluster ID in a `ClusterClaim` CR. The new
+version of Antrea Multi-cluster controller will update the existing `ClusterSet` CR with
+the valid Cluster ID. You can verify this via `kubectl get clusterclaim id.k8s.io -o json -n kube-system | jq -r '.value'`.
+
+Once you finish the cluster upgrade, you should also delete the `ClusterClaim` CRD
+manually, all `ClusterClaim` CRs will be removed automatically after the CRD is deleted.
+
+```bash
+kubectl delete crds clusterclaims.multicluster.crd.antrea.io
+```
+
 ## APIs deprecation policy
 
 The Antrea Multi-cluster APIs are built using K8s CustomResourceDefinitions and we

docs/multicluster/user-guide.md

@@ -33,6 +33,10 @@ through tunnels among clusters. The ClusterNetworkPolicy replication feature is
 supported since Antrea v1.6.0, and Multi-cluster NetworkPolicy rules are
 supported since Antrea v1.10.0.
 
+Antrea v1.13 promoted the ClusterSet CRD version from v1alpha1 to v1alpha2. If you
+plan to upgrade from a previous version to v1.13 or later, please check
+the [upgrade guide](./upgrade.md#upgrade-from-a-version-prior-to-v113).
+
 ## Quick Start
 
 Please refer to the [Quick Start Guide](quick-start.md) to learn how to build a

multicluster/cmd/multicluster-controller/controller.go

@@ -18,7 +18,9 @@ import (
 	"fmt"
 	"time"
 
+	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
@@ -171,6 +173,14 @@ func setupManagerAndCertController(isLeader bool, o *Options) (manager.Manager,
 		o.EnableEndpointSlice = true
 	}
 
+	// ClusterClaim CRD is removed since v1.13. Check the existence of
+	// ClusterClaim API before using ClusterClaim API.
+	clusterClaimCRDAvailable, err := clusterClaimCRDAvailable(client)
+	if err != nil {
+		return nil, fmt.Errorf("error checking if ClusterClaim API is available")
+	}
+	o.ClusterCalimCRDAvailable = clusterClaimCRDAvailable
+
 	mgr, err := ctrl.NewManager(k8sConfig, o.options)
 	if err != nil {
 		return nil, fmt.Errorf("error starting manager: %v", err)
@@ -192,3 +202,21 @@ func setupManagerAndCertController(isLeader bool, o *Options) (manager.Manager,
 	}
 	return mgr, nil
 }
+
+func clusterClaimCRDAvailable(k8sClient clientset.Interface) (bool, error) {
+	groupVersion := mcv1alpha2.SchemeGroupVersion.String()
+	resources, err := k8sClient.Discovery().ServerResourcesForGroupVersion(groupVersion)
+	if err != nil {
+		// The group version doesn't exist.
+		if errors.IsNotFound(err) {
+			return false, nil
+		}
+		return false, fmt.Errorf("error getting server resources for GroupVersion %s: %v", groupVersion, err)
+	}
+	for _, resource := range resources.APIResources {
+		if resource.Kind == "ClusterClaim" {
+			return true, nil
+		}
+	}
+	return false, nil
+}

multicluster/cmd/multicluster-controller/controller_test.go

@@ -23,8 +23,12 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes/fake"
 	"k8s.io/client-go/rest"
 
+	mcv1alpha2 "antrea.io/antrea/multicluster/apis/multicluster/v1alpha2"
 	"antrea.io/antrea/pkg/apiserver/certificate"
 )
 
@@ -93,3 +97,45 @@ func TestGetCAConfig(t *testing.T) {
 		})
 	}
 }
+
+func TestClusterClaimCRDAvailable(t *testing.T) {
+	groupVersion := mcv1alpha2.SchemeGroupVersion.String()
+	testCases := []struct {
+		name              string
+		resources         []*metav1.APIResourceList
+		expectedAvailable bool
+	}{
+		{
+			name:              "empty",
+			expectedAvailable: false,
+		},
+		{
+			name: "GroupVersion exists",
+			resources: []*metav1.APIResourceList{
+				{
+					GroupVersion: groupVersion,
+				},
+			},
+			expectedAvailable: false,
+		},
+		{
+			name: "API exists",
+			resources: []*metav1.APIResourceList{
+				{
+					GroupVersion: groupVersion,
+					APIResources: []metav1.APIResource{{Kind: "ClusterClaim"}},
+				},
+			},
+			expectedAvailable: true,
+		},
+	}
+	for _, tt := range testCases {
+		t.Run(tt.name, func(t *testing.T) {
+			k8sClient := fake.NewSimpleClientset()
+			k8sClient.Resources = tt.resources
+			available, err := clusterClaimCRDAvailable(k8sClient)
+			require.NoError(t, err)
+			assert.Equal(t, tt.expectedAvailable, available)
+		})
+	}
+}

multicluster/cmd/multicluster-controller/leader.go

@@ -77,9 +77,10 @@ func runLeader(o *Options) error {
 			namespace: env.GetPodNamespace()}})
 
 	clusterSetReconciler := &leader.LeaderClusterSetReconciler{
-		Client:        mgr.GetClient(),
-		Scheme:        mgr.GetScheme(),
-		StatusManager: memberClusterStatusManager,
+		Client:                   mgr.GetClient(),
+		Scheme:                   mgr.GetScheme(),
+		StatusManager:            memberClusterStatusManager,
+		ClusterCalimCRDAvailable: o.ClusterCalimCRDAvailable,
 	}
 	if err = clusterSetReconciler.SetupWithManager(mgr); err != nil {
 		return fmt.Errorf("error creating ClusterSet controller: %v", err)

multicluster/cmd/multicluster-controller/member.go

@@ -67,6 +67,7 @@ func runMember(o *Options) error {
 		mgr.GetScheme(),
 		env.GetPodNamespace(),
 		o.EnableStretchedNetworkPolicy,
+		o.ClusterCalimCRDAvailable,
 	)
 	if err = clusterSetReconciler.SetupWithManager(mgr); err != nil {
 		return fmt.Errorf("error creating ClusterSet controller: %v", err)

multicluster/cmd/multicluster-controller/options.go

@@ -46,6 +46,9 @@ type Options struct {
 	EnableStretchedNetworkPolicy bool
 	// Watch EndpointSlice API for exported Service if EndpointSlice API is available.
 	EnableEndpointSlice bool
+	// ClusterCalimCRDAvailable indicates if the ClusterClaim CRD is available or not
+	// in the cluster.
+	ClusterCalimCRDAvailable bool
 }
 
 func newOptions() *Options {

multicluster/controllers/multicluster/common/controller_utils.go

@@ -22,7 +22,10 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	mcv1alpha2 "antrea.io/antrea/multicluster/apis/multicluster/v1alpha2"
 )
 
 // DiscoverServiceCIDRByInvalidServiceCreation creates an invalid Service to get returned error, and analyzes
@@ -78,3 +81,40 @@ func parseServiceCIDRFromError(msg string) (string, error) {
 func NewClusterInfoResourceExportName(clusterID string) string {
 	return clusterID + "-clusterinfo"
 }
+
+func getClusterIDFromClusterClaim(c client.Client, clusterSet *mcv1alpha2.ClusterSet) (ClusterID, error) {
+	configNamespace := clusterSet.GetNamespace()
+
+	clusterClaimList := &mcv1alpha2.ClusterClaimList{}
+	if err := c.List(context.TODO(), clusterClaimList, client.InNamespace(configNamespace)); err != nil {
+		return "", err
+	}
+	if len(clusterClaimList.Items) == 0 {
+		return "", fmt.Errorf("ClusterClaim is not configured for the cluster")
+	}
+
+	for _, clusterClaim := range clusterClaimList.Items {
+		if clusterClaim.Name == mcv1alpha2.WellKnownClusterClaimID {
+			return ClusterID(clusterClaim.Value), nil
+		}
+	}
+
+	return "", fmt.Errorf("ClusterClaim not configured for Name=%s",
+		mcv1alpha2.WellKnownClusterClaimID)
+}
+
+func GetClusterID(clusterCalimCRDAvailable bool, req ctrl.Request, client client.Client, clusterSet *mcv1alpha2.ClusterSet) (ClusterID, error) {
+	if clusterSet.Spec.ClusterID == "" {
+		// ClusterID is a required feild, and the empty value case should only happen
+		// when Antrea Multi-cluster is upgraded from an old version prior to v1.13.
+		// Here we try to get the ClusterID from ClusterClaim before returning any error.
+		if clusterCalimCRDAvailable {
+			clusterID, err := getClusterIDFromClusterClaim(client, clusterSet)
+			if err == nil {
+				return clusterID, nil
+			}
+		}
+		return "", fmt.Errorf("'clusterID' is not set in the ClusterSet %s spec", req.NamespacedName)
+	}
+	return ClusterID(clusterSet.Spec.ClusterID), nil
+}