[IPv6] Add support for dual-stack when using kube-proxy for Service (a…

…ntrea-io#1200) 1. Add a config item for IPv6 Serivce CIDR if using kube-proxy to provide Service functions. 2. Output IPv6 traffic from host gateway if its destination is a Service address. 3. Use ct_mark to identify Service traffic and output the reply packet to the host gateway to ensure the DNAT processing in iptables.
lzhecheng · Nov 2, 2020 · c8b0c6a · c8b0c6a
1 parent db24513
commit c8b0c6a
Show file tree

Hide file tree

Showing 19 changed files with 367 additions and 179 deletions.
diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml
@@ -1041,7 +1041,8 @@ data:
     featureGates:
     # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent.
     # It should be enabled on Windows, otherwise NetworkPolicy will not take effect on
-    # Service traffic.
+    # Service traffic. Antrea proxy doesn't support an IPv6 only cluster or a Dual-Stack cluster
+    # before PR #1102[https://github.com/vmware-tanzu/antrea/pull/1102] is merged.
       AntreaProxy: true
 
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
@@ -1109,6 +1110,12 @@ data:
     # for the GRE tunnel type.
     #enableIPSecTunnel: false
 
+    # ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack
+    # cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by
+    # --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed.
+    # No default value for this field.
+    #serviceCIDRv6:
+
     # The port for the antrea-agent APIServer to serve on.
     # Note that if it's set to another value, the `containerPort` of the `api` port of the
     # `antrea-agent` container must be set to the same value.
@@ -1187,7 +1194,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-2fm49f65tg
+  name: antrea-config-hmttgfbf78
   namespace: kube-system
 ---
 apiVersion: v1
@@ -1294,7 +1301,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-2fm49f65tg
+          name: antrea-config-hmttgfbf78
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -1543,7 +1550,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-2fm49f65tg
+          name: antrea-config-hmttgfbf78
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml
@@ -1041,7 +1041,8 @@ data:
     featureGates:
     # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent.
     # It should be enabled on Windows, otherwise NetworkPolicy will not take effect on
-    # Service traffic.
+    # Service traffic. Antrea proxy doesn't support an IPv6 only cluster or a Dual-Stack cluster
+    # before PR #1102[https://github.com/vmware-tanzu/antrea/pull/1102] is merged.
       AntreaProxy: true
 
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
@@ -1109,6 +1110,12 @@ data:
     # for the GRE tunnel type.
     #enableIPSecTunnel: false
 
+    # ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack
+    # cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by
+    # --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed.
+    # No default value for this field.
+    #serviceCIDRv6:
+
     # The port for the antrea-agent APIServer to serve on.
     # Note that if it's set to another value, the `containerPort` of the `api` port of the
     # `antrea-agent` container must be set to the same value.
@@ -1187,7 +1194,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-2fm49f65tg
+  name: antrea-config-hmttgfbf78
   namespace: kube-system
 ---
 apiVersion: v1
@@ -1294,7 +1301,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-2fm49f65tg
+          name: antrea-config-hmttgfbf78
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -1545,7 +1552,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-2fm49f65tg
+          name: antrea-config-hmttgfbf78
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml
@@ -1041,7 +1041,8 @@ data:
     featureGates:
     # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent.
     # It should be enabled on Windows, otherwise NetworkPolicy will not take effect on
-    # Service traffic.
+    # Service traffic. Antrea proxy doesn't support an IPv6 only cluster or a Dual-Stack cluster
+    # before PR #1102[https://github.com/vmware-tanzu/antrea/pull/1102] is merged.
       AntreaProxy: true
 
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
@@ -1109,6 +1110,12 @@ data:
     # for the GRE tunnel type.
     #enableIPSecTunnel: false
 
+    # ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack
+    # cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by
+    # --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed.
+    # No default value for this field.
+    #serviceCIDRv6:
+
     # The port for the antrea-agent APIServer to serve on.
     # Note that if it's set to another value, the `containerPort` of the `api` port of the
     # `antrea-agent` container must be set to the same value.
@@ -1187,7 +1194,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-m742b78ct4
+  name: antrea-config-8bc4m9g22g
   namespace: kube-system
 ---
 apiVersion: v1
@@ -1294,7 +1301,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-m742b78ct4
+          name: antrea-config-8bc4m9g22g
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -1543,7 +1550,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-m742b78ct4
+          name: antrea-config-8bc4m9g22g
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml
@@ -1041,7 +1041,8 @@ data:
     featureGates:
     # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent.
     # It should be enabled on Windows, otherwise NetworkPolicy will not take effect on
-    # Service traffic.
+    # Service traffic. Antrea proxy doesn't support an IPv6 only cluster or a Dual-Stack cluster
+    # before PR #1102[https://github.com/vmware-tanzu/antrea/pull/1102] is merged.
     #  AntreaProxy: false
 
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
@@ -1114,6 +1115,12 @@ data:
     # AntreaProxy is enabled, this parameter is not needed and will be ignored if provided.
     #serviceCIDR: 10.96.0.0/12
 
+    # ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack
+    # cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by
+    # --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed.
+    # No default value for this field.
+    #serviceCIDRv6:
+
     # The port for the antrea-agent APIServer to serve on.
     # Note that if it's set to another value, the `containerPort` of the `api` port of the
     # `antrea-agent` container must be set to the same value.
@@ -1192,7 +1199,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-2gdh849dhk
+  name: antrea-config-kgd27dftgd
   namespace: kube-system
 ---
 apiVersion: v1
@@ -1308,7 +1315,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-2gdh849dhk
+          name: antrea-config-kgd27dftgd
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -1592,7 +1599,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-2gdh849dhk
+          name: antrea-config-kgd27dftgd
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml
@@ -1041,7 +1041,8 @@ data:
     featureGates:
     # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent.
     # It should be enabled on Windows, otherwise NetworkPolicy will not take effect on
-    # Service traffic.
+    # Service traffic. Antrea proxy doesn't support an IPv6 only cluster or a Dual-Stack cluster
+    # before PR #1102[https://github.com/vmware-tanzu/antrea/pull/1102] is merged.
     #  AntreaProxy: false
 
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
@@ -1114,6 +1115,12 @@ data:
     # AntreaProxy is enabled, this parameter is not needed and will be ignored if provided.
     #serviceCIDR: 10.96.0.0/12
 
+    # ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack
+    # cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by
+    # --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed.
+    # No default value for this field.
+    #serviceCIDRv6:
+
     # The port for the antrea-agent APIServer to serve on.
     # Note that if it's set to another value, the `containerPort` of the `api` port of the
     # `antrea-agent` container must be set to the same value.
@@ -1192,7 +1199,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-t5k2d655gm
+  name: antrea-config-2hk276fdf4
   namespace: kube-system
 ---
 apiVersion: v1
@@ -1299,7 +1306,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-t5k2d655gm
+          name: antrea-config-2hk276fdf4
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -1548,7 +1555,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-t5k2d655gm
+          name: antrea-config-2hk276fdf4
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/base/conf/antrea-agent.conf b/build/yamls/base/conf/antrea-agent.conf
@@ -2,7 +2,8 @@
 featureGates:
 # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent.
 # It should be enabled on Windows, otherwise NetworkPolicy will not take effect on
-# Service traffic.
+# Service traffic. Antrea proxy doesn't support an IPv6 only cluster or a Dual-Stack cluster
+# before PR #1102[https://github.com/vmware-tanzu/antrea/pull/1102] is merged.
 #  AntreaProxy: false
 
 # Enable traceflow which provides packet tracing feature to diagnose network issue.
@@ -75,6 +76,12 @@ featureGates:
 # AntreaProxy is enabled, this parameter is not needed and will be ignored if provided.
 #serviceCIDR: 10.96.0.0/12
 
+# ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack
+# cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by
+# --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed.
+# No default value for this field.
+#serviceCIDRv6:
+
 # The port for the antrea-agent APIServer to serve on.
 # Note that if it's set to another value, the `containerPort` of the `api` port of the
 # `antrea-agent` container must be set to the same value.

diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go
@@ -101,6 +101,12 @@ func run(o *Options) error {
 	}
 
 	_, serviceCIDRNet, _ := net.ParseCIDR(o.config.ServiceCIDR)
+	var serviceCIDRNetv6 *net.IPNet
+	// Todo: use FeatureGate to check if IPv6 is enabled and then read configuration item "ServiceCIDRv6".
+	if o.config.ServiceCIDRv6 != "" {
+		_, serviceCIDRNetv6, _ = net.ParseCIDR(o.config.ServiceCIDRv6)
+	}
+
 	_, encapMode := config.GetTrafficEncapModeFromStr(o.config.TrafficEncapMode)
 	networkConfig := &config.NetworkConfig{
 		TunnelType:        ovsconfig.TunnelType(o.config.TunnelType),
@@ -126,6 +132,7 @@ func run(o *Options) error {
 		o.config.HostGateway,
 		o.config.DefaultMTU,
 		serviceCIDRNet,
+		serviceCIDRNetv6,
 		networkConfig,
 		features.DefaultFeatureGate.Enabled(features.AntreaProxy))
 	err = agentInitializer.Initialize()

diff --git a/cmd/antrea-agent/config.go b/cmd/antrea-agent/config.go
@@ -84,6 +84,11 @@ type AgentConfig struct {
 	// AntreaProxy is enabled, this parameter is not needed and will be ignored if provided.
 	// Default is 10.96.0.0/12
 	ServiceCIDR string `yaml:"serviceCIDR,omitempty"`
+	// ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack
+	// cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by
+	// --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed.
+	// No default value for this field.
+	ServiceCIDRv6 string `yaml:"serviceCIDRv6,omitempty"`
 	// Whether or not to enable IPSec (ESP) encryption for Pod traffic across Nodes. IPSec encryption
 	// is supported only for the GRE tunnel type. Antrea uses Preshared Key (PSK) for IKE
 	// authentication. When IPSec tunnel is enabled, the PSK value must be passed to Antrea Agent

diff --git a/cmd/antrea-agent/options.go b/cmd/antrea-agent/options.go
@@ -84,7 +84,13 @@ func (o *Options) validate(args []string) error {
 	// Validate service CIDR configuration
 	_, _, err := net.ParseCIDR(o.config.ServiceCIDR)
 	if err != nil {
-		return fmt.Errorf("service CIDR %s is invalid", o.config.ServiceCIDR)
+		return fmt.Errorf("Service CIDR %s is invalid", o.config.ServiceCIDR)
+	}
+	if o.config.ServiceCIDRv6 != "" {
+		_, _, err := net.ParseCIDR(o.config.ServiceCIDRv6)
+		if err != nil {
+			return fmt.Errorf("Service CIDR v6 %s is invalid", o.config.ServiceCIDRv6)
+		}
 	}
 	if o.config.TunnelType != ovsconfig.VXLANTunnel && o.config.TunnelType != ovsconfig.GeneveTunnel &&
 		o.config.TunnelType != ovsconfig.GRETunnel && o.config.TunnelType != ovsconfig.STTTunnel {

diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go
@@ -64,6 +64,7 @@ type Initializer struct {
 	hostGateway     string // name of gateway port on the OVS bridge
 	mtu             int
 	serviceCIDR     *net.IPNet // K8s Service ClusterIP CIDR
+	serviceCIDRv6   *net.IPNet // K8s Service ClusterIP CIDR in IPv6
 	networkConfig   *config.NetworkConfig
 	nodeConfig      *config.NodeConfig
 	enableProxy     bool
@@ -79,6 +80,7 @@ func NewInitializer(
 	hostGateway string,
 	mtu int,
 	serviceCIDR *net.IPNet,
+	serviceCIDRv6 *net.IPNet,
 	networkConfig *config.NetworkConfig,
 	enableProxy bool) *Initializer {
 	return &Initializer{
@@ -91,6 +93,7 @@ func NewInitializer(
 		hostGateway:     hostGateway,
 		mtu:             mtu,
 		serviceCIDR:     serviceCIDR,
+		serviceCIDRv6:   serviceCIDRv6,
 		networkConfig:   networkConfig,
 		enableProxy:     enableProxy,
 	}
@@ -314,8 +317,8 @@ func (i *Initializer) initOpenFlowPipeline() error {
 		// from local Pods to any Service address can be forwarded to the host gateway interface
 		// correctly. Otherwise packets might be dropped by egress rules before they are DNATed to
 		// backend Pods.
-		if err := i.ofClient.InstallClusterServiceCIDRFlows(i.serviceCIDR, gatewayOFPort); err != nil {
-			klog.Errorf("Failed to setup openflow entries for Cluster Service CIDR %s: %v", i.serviceCIDR, err)
+		if err := i.ofClient.InstallClusterServiceCIDRFlows([]*net.IPNet{i.serviceCIDR, i.serviceCIDRv6}, gatewayOFPort); err != nil {
+			klog.Errorf("Failed to setup OpenFlow entries for Service CIDRs: %v", err)
 			return err
 		}
 	} else {