Support Pod to Pod connectivity

Signed-off-by: hujiajing <[email protected]>
antrea-io · Sep 27, 2022 · 24d2c07 · 24d2c07
1 parent 0f77529
commit 24d2c07
Show file tree

Hide file tree

Showing 18 changed files with 188 additions and 3 deletions.
diff --git a/multicluster/apis/multicluster/v1alpha1/gateway_types.go b/multicluster/apis/multicluster/v1alpha1/gateway_types.go
@@ -50,6 +50,8 @@ type ClusterInfo struct {
 	ServiceCIDR string `json:"serviceCIDR,omitempty"`
 	// GatewayInfos has information of Gateways
 	GatewayInfos []GatewayInfo `json:"gatewayInfos,omitempty"`
+	// PodCIDRs is the Pod IP address CIDRs.
+	PodCIDRs []string `json:"podCIDRs,omitempty"`
 }
 
 //+kubebuilder:object:root=true

diff --git a/multicluster/apis/multicluster/v1alpha1/multiclusterconfig_types.go b/multicluster/apis/multicluster/v1alpha1/multiclusterconfig_types.go
@@ -41,6 +41,8 @@ type MultiClusterConfig struct {
 	config.ControllerManagerConfigurationSpec `json:",inline"`
 	// ServiceCIDR allows user to set the ClusterIP range of the cluster manually.
 	ServiceCIDR string `json:"serviceCIDR,omitempty"`
+	// PodCIDRs is the Pod IP address CIDRs.
+	PodCIDRs []string `json:"podCIDRs,omitempty"`
 	// The precedence about which IP address (internal or external IP) of Node is preferred to
 	// be used as the cross-cluster tunnel endpoint. if not specified, internal IP will be chosen.
 	GatewayIPPrecedence Precedence `json:"gatewayIPPrecedence,omitempty"`

diff --git a/multicluster/build/yamls/antrea-multicluster-leader-global.yml b/multicluster/build/yamls/antrea-multicluster-leader-global.yml
@@ -379,6 +379,11 @@ spec:
                           type: string
                       type: object
                     type: array
+                  podCIDRs:
+                    description: PodCIDRs is the Pod IP address CIDRs.
+                    items:
+                      type: string
+                    type: array
                   serviceCIDR:
                     description: ServiceCIDR is the IP ranges used by Service ClusterIP.
                     type: string
@@ -3268,6 +3273,11 @@ spec:
                           type: string
                       type: object
                     type: array
+                  podCIDRs:
+                    description: PodCIDRs is the Pod IP address CIDRs.
+                    items:
+                      type: string
+                    type: array
                   serviceCIDR:
                     description: ServiceCIDR is the IP ranges used by Service ClusterIP.
                     type: string

diff --git a/multicluster/build/yamls/antrea-multicluster-leader-namespaced.yml b/multicluster/build/yamls/antrea-multicluster-leader-namespaced.yml
@@ -322,6 +322,8 @@ data:
     leaderElection:
       leaderElect: false
     serviceCIDR: ""
+    podCIDRs:
+      - ""
     gatewayIPPrecedence: "private"
     endpointIPType: "ClusterIP"
 kind: ConfigMap

diff --git a/multicluster/build/yamls/antrea-multicluster-member.yml b/multicluster/build/yamls/antrea-multicluster-member.yml
@@ -111,6 +111,11 @@ spec:
                       type: string
                   type: object
                 type: array
+              podCIDRs:
+                description: PodCIDRs is the Pod IP address CIDRs.
+                items:
+                  type: string
+                type: array
               serviceCIDR:
                 description: ServiceCIDR is the IP ranges used by Service ClusterIP.
                 type: string
@@ -946,6 +951,8 @@ data:
     leaderElection:
       leaderElect: false
     serviceCIDR: ""
+    podCIDRs:
+      - ""
     gatewayIPPrecedence: "private"
     endpointIPType: "ClusterIP"
 kind: ConfigMap

diff --git a/multicluster/cmd/multicluster-controller/member.go b/multicluster/cmd/multicluster-controller/member.go
@@ -84,6 +84,7 @@ func runMember(o *Options) error {
 		mgr.GetScheme(),
 		env.GetPodNamespace(),
 		opts.ServiceCIDR,
+		opts.PodCIDRs,
 		commonAreaGetter)
 	if err = gwReconciler.SetupWithManager(mgr); err != nil {
 		return fmt.Errorf("error creating Gateway controller: %v", err)

diff --git a/multicluster/cmd/multicluster-controller/options.go b/multicluster/cmd/multicluster-controller/options.go
@@ -32,6 +32,8 @@ type Options struct {
 	options        ctrl.Options
 	// The Service ClusterIP range used in the member cluster.
 	ServiceCIDR string
+	// PodCIDRs is the Pod IP address CIDRs of the member cluster.
+	PodCIDRs []string
 	// The precedence about which IP (private or public one) of Node is preferred to
 	// be used as tunnel endpoint. If not specified, private IP will be chosen.
 	GatewayIPPrecedence mcsv1alpha1.Precedence
@@ -64,7 +66,17 @@ func (o *Options) complete(args []string) error {
 				return fmt.Errorf("failed to parse serviceCIDR, invalid CIDR string %s", ctrlConfig.ServiceCIDR)
 			}
 		}
+		cidrs := []string{}
+		for _, cidr := range ctrlConfig.PodCIDRs {
+			if _, _, err := net.ParseCIDR(cidr); err != nil && cidr != "" {
+				return fmt.Errorf("failed to parse podCIDRs, invalid CIDR string %s", cidr)
+			}
+			if cidr != "" {
+				cidrs = append(cidrs, cidr)
+			}
+		}
 		o.ServiceCIDR = ctrlConfig.ServiceCIDR
+		o.PodCIDRs = cidrs
 		o.GatewayIPPrecedence = ctrlConfig.GatewayIPPrecedence
 		if ctrlConfig.EndpointIPType == "" {
 			o.EndpointIPType = "ClusterIP"

diff --git a/multicluster/cmd/multicluster-controller/options_test.go b/multicluster/cmd/multicluster-controller/options_test.go
@@ -0,0 +1,79 @@
+// Copyright 2022 Antrea Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	ctrl "sigs.k8s.io/controller-runtime"
+)
+
+func TestComplete(t *testing.T) {
+	testCases := []struct {
+		name       string
+		o          Options
+		configFile string
+		exceptdErr error
+	}{
+		{
+			name: "options with valid PodCIDRs",
+			o: Options{
+				configFile:          "./testdata/antrea-mc-config-with-valid-podcidrs.yml",
+				SelfSignedCert:      false,
+				options:             ctrl.Options{},
+				ServiceCIDR:         "",
+				PodCIDRs:            nil,
+				GatewayIPPrecedence: "",
+				EndpointIPType:      "",
+			},
+			exceptdErr: nil,
+		},
+		{
+			name: "options with empty PodCIDRs",
+			o: Options{
+				configFile:          "./testdata/antrea-mc-config-with-empty-podcidrs.yml",
+				SelfSignedCert:      false,
+				options:             ctrl.Options{},
+				ServiceCIDR:         "",
+				PodCIDRs:            nil,
+				GatewayIPPrecedence: "",
+				EndpointIPType:      "",
+			},
+			exceptdErr: nil,
+		},
+		{
+			name: "options without PodCIDRs",
+			o: Options{
+				configFile:          "./testdata/antrea-mc-config-with-invalid-podcidrs.yml",
+				SelfSignedCert:      false,
+				options:             ctrl.Options{},
+				ServiceCIDR:         "10.100.0.0/16",
+				PodCIDRs:            nil,
+				GatewayIPPrecedence: "",
+				EndpointIPType:      "",
+			},
+			exceptdErr: fmt.Errorf("failed to parse podCIDRs, invalid CIDR string 10.10a.0.0/16"),
+		},
+	}
+
+	for _, tt := range testCases {
+		t.Run(tt.name, func(t *testing.T) {
+			err := tt.o.complete(nil)
+			assert.Equal(t, tt.exceptdErr, err)
+		})
+	}
+}
diff --git a/multicluster/cmd/multicluster-controller/testdata/antrea-mc-config-with-empty-podcidrs.yml b/multicluster/cmd/multicluster-controller/testdata/antrea-mc-config-with-empty-podcidrs.yml
@@ -0,0 +1,15 @@
+apiVersion: multicluster.crd.antrea.io/v1alpha1
+kind: MultiClusterConfig
+health:
+  healthProbeBindAddress: :8080
+metrics:
+  bindAddress: "0"
+webhook:
+  port: 9443
+leaderElection:
+  leaderElect: false
+serviceCIDR: ""
+podCIDRs:
+  - ""
+gatewayIPPrecedence: "private"
+endpointIPType: "ClusterIP"
diff --git a/multicluster/cmd/multicluster-controller/testdata/antrea-mc-config-with-invalid-podcidrs.yml b/multicluster/cmd/multicluster-controller/testdata/antrea-mc-config-with-invalid-podcidrs.yml
@@ -0,0 +1,16 @@
+apiVersion: multicluster.crd.antrea.io/v1alpha1
+kind: MultiClusterConfig
+health:
+  healthProbeBindAddress: :8080
+metrics:
+  bindAddress: "0"
+webhook:
+  port: 9443
+leaderElection:
+  leaderElect: false
+serviceCIDR: "10.100.0.0/16"
+podCIDRs:
+  - "10.10a.0.0/16"
+  - ""
+gatewayIPPrecedence: "private"
+endpointIPType: "ClusterIP"
diff --git a/multicluster/cmd/multicluster-controller/testdata/antrea-mc-config-with-valid-podcidrs.yml b/multicluster/cmd/multicluster-controller/testdata/antrea-mc-config-with-valid-podcidrs.yml
@@ -0,0 +1,16 @@
+apiVersion: multicluster.crd.antrea.io/v1alpha1
+kind: MultiClusterConfig
+health:
+  healthProbeBindAddress: :8080
+metrics:
+  bindAddress: "0"
+webhook:
+  port: 9443
+leaderElection:
+  leaderElect: false
+serviceCIDR: ""
+podCIDRs:
+  - "10.10.0.0/16"
+  - ""
+gatewayIPPrecedence: "private"
+endpointIPType: "ClusterIP"
diff --git a/multicluster/config/crd/bases/multicluster.crd.antrea.io_clusterinfoimports.yaml b/multicluster/config/crd/bases/multicluster.crd.antrea.io_clusterinfoimports.yaml
@@ -57,6 +57,11 @@ spec:
                       type: string
                   type: object
                 type: array
+              podCIDRs:
+                description: PodCIDRs is the Pod IP address CIDRs.
+                items:
+                  type: string
+                type: array
               serviceCIDR:
                 description: ServiceCIDR is the IP ranges used by Service ClusterIP.
                 type: string

diff --git a/multicluster/config/crd/bases/multicluster.crd.antrea.io_resourceexports.yaml b/multicluster/config/crd/bases/multicluster.crd.antrea.io_resourceexports.yaml
@@ -74,6 +74,11 @@ spec:
                           type: string
                       type: object
                     type: array
+                  podCIDRs:
+                    description: PodCIDRs is the Pod IP address CIDRs.
+                    items:
+                      type: string
+                    type: array
                   serviceCIDR:
                     description: ServiceCIDR is the IP ranges used by Service ClusterIP.
                     type: string

diff --git a/multicluster/config/crd/bases/multicluster.crd.antrea.io_resourceimports.yaml b/multicluster/config/crd/bases/multicluster.crd.antrea.io_resourceimports.yaml
@@ -72,6 +72,11 @@ spec:
                           type: string
                       type: object
                     type: array
+                  podCIDRs:
+                    description: PodCIDRs is the Pod IP address CIDRs.
+                    items:
+                      type: string
+                    type: array
                   serviceCIDR:
                     description: ServiceCIDR is the IP ranges used by Service ClusterIP.
                     type: string

diff --git a/multicluster/config/default/configmap/controller_manager_config.yaml b/multicluster/config/default/configmap/controller_manager_config.yaml
@@ -9,5 +9,7 @@ webhook:
 leaderElection:
   leaderElect: false
 serviceCIDR: ""
+podCIDRs:
+  - ""
 gatewayIPPrecedence: "private"
 endpointIPType: "ClusterIP"
diff --git a/multicluster/controllers/multicluster/gateway_controller.go b/multicluster/controllers/multicluster/gateway_controller.go
@@ -44,6 +44,7 @@ type (
 		namespace        string
 		localClusterID   string
 		serviceCIDR      string
+		podCIDRs         []string
 		leaderNamespace  string
 	}
 )
@@ -55,12 +56,14 @@ func NewGatewayReconciler(
 	scheme *runtime.Scheme,
 	namespace string,
 	serviceCIDR string,
+	podCIDRs []string,
 	commonAreaGetter RemoteCommonAreaGetter) *GatewayReconciler {
 	reconciler := &GatewayReconciler{
 		Client:           client,
 		Scheme:           scheme,
 		namespace:        namespace,
 		serviceCIDR:      serviceCIDR,
+		podCIDRs:         podCIDRs,
 		commonAreaGetter: commonAreaGetter,
 	}
 	return reconciler
@@ -144,6 +147,7 @@ func (r *GatewayReconciler) updateResourceExport(ctx context.Context, req ctrl.R
 	resExportSpec.ClusterInfo = &mcsv1alpha1.ClusterInfo{
 		ClusterID:    r.localClusterID,
 		ServiceCIDR:  r.serviceCIDR,
+		PodCIDRs:     r.podCIDRs,
 		GatewayInfos: []mcsv1alpha1.GatewayInfo{*gwInfo},
 	}
 	if reflect.DeepEqual(existingResExport.Spec, resExportSpec) {
@@ -171,6 +175,7 @@ func (r *GatewayReconciler) createResourceExport(ctx context.Context, req ctrl.R
 	resExportSpec.ClusterInfo = &mcsv1alpha1.ClusterInfo{
 		ClusterID:   r.localClusterID,
 		ServiceCIDR: r.serviceCIDR,
+		PodCIDRs:    r.podCIDRs,
 		GatewayInfos: []mcsv1alpha1.GatewayInfo{
 			{
 				GatewayIP: gatewayIP,

diff --git a/multicluster/controllers/multicluster/gateway_controller_test.go b/multicluster/controllers/multicluster/gateway_controller_test.go
@@ -141,7 +141,7 @@ func TestGatewayReconciler(t *testing.T) {
 		mcReconciler := NewMemberClusterSetReconciler(fakeClient, scheme, "default")
 		mcReconciler.SetRemoteCommonArea(commonArea)
 		commonAreaGatter := mcReconciler
-		r := NewGatewayReconciler(fakeClient, scheme, "default", "10.96.0.0/12", commonAreaGatter)
+		r := NewGatewayReconciler(fakeClient, scheme, "default", "10.96.0.0/12", []string{"10.200.1.1/16"}, commonAreaGatter)
 		t.Run(tt.name, func(t *testing.T) {
 			req := ctrl.Request{NamespacedName: tt.namespacedName}
 			if _, err := r.Reconcile(ctx, req); err != nil {

diff --git a/pkg/agent/multicluster/mc_route_controller.go b/pkg/agent/multicluster/mc_route_controller.go
@@ -331,15 +331,16 @@ func (c *MCRouteController) addMCFlowsForSingleCIImp(activeGW *mcv1alpha1.Gatewa
 
 	if installedCIImp != nil {
 		oldTunnelPeerIPToRemoteGW := getPeerGatewayIP(installedCIImp.Spec)
-		if oldTunnelPeerIPToRemoteGW.Equal(tunnelPeerIPToRemoteGW) && installedCIImp.Spec.ServiceCIDR == ciImport.Spec.ServiceCIDR {
+		if oldTunnelPeerIPToRemoteGW.Equal(tunnelPeerIPToRemoteGW) && installedCIImp.Spec.ServiceCIDR == ciImport.Spec.ServiceCIDR &&
+			sets.NewString(installedCIImp.Spec.PodCIDRs...).Equal(sets.NewString(ciImport.Spec.PodCIDRs...)) {
 			klog.V(2).InfoS("No difference between new and installed ClusterInfoImports, skip updating", "clusterinfoimport", ciImport.Name)
 			return nil
 		}
 	}
 
 	klog.InfoS("Adding/updating remote Gateway Node flows for Multi-cluster", "gateway", klog.KObj(activeGW),
 		"node", c.nodeConfig.Name, "peer", tunnelPeerIPToRemoteGW)
-	allCIDRs := []string{ciImport.Spec.ServiceCIDR}
+	allCIDRs := append([]string{ciImport.Spec.ServiceCIDR}, ciImport.Spec.PodCIDRs...)
 	peerConfigs, err := generatePeerConfigs(allCIDRs, tunnelPeerIPToRemoteGW)
 	if err != nil {
 		klog.ErrorS(err, "Parse error for serviceCIDR from remote cluster", "clusterinfoimport", ciImport.Name, "gateway", activeGW.Name)