[IPv6] Add support for dual-stack when using kube-proxy for Service (#…

…1200) 1. Add a config item for IPv6 Serivce CIDR if using kube-proxy to provide Service functions. 2. Output IPv6 traffic from host gateway if its destination is a Service address. 3. Use ct_mark to identify Service traffic and output the reply packet to the host gateway to ensure the DNAT processing in iptables.
antrea-io · Oct 12, 2020 · 3c98b56 · 3c98b56
1 parent 773fb03
commit 3c98b56
Show file tree

Hide file tree

Showing 19 changed files with 366 additions and 178 deletions.
diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml
@@ -1030,7 +1030,8 @@ data:
     featureGates:
     # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent.
     # It should be enabled on Windows, otherwise NetworkPolicy will not take effect on
-    # Service traffic.
+    # Service traffic. Antrea proxy doesn't support an IPv6 only cluster or a Dual-Stack cluster
+    # before PR #1102[https://github.com/vmware-tanzu/antrea/pull/1102] is merged.
       AntreaProxy: true
 
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
@@ -1078,6 +1079,12 @@ data:
     # for the GRE tunnel type.
     #enableIPSecTunnel: false
 
+    # ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack
+    # cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by
+    # --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed.
+    # No default value for this field.
+    #serviceCIDRv6:
+
     # Determines how traffic is encapsulated. It has the following options
     # encap(default): Inter-node Pod traffic is always encapsulated and Pod to outbound traffic is masqueraded.
     # noEncap: Inter-node Pod traffic is not encapsulated, but Pod to outbound traffic is masqueraded.
@@ -1161,7 +1168,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-8hk5mt9mg7
+  name: antrea-config-8964cgbht9
   namespace: kube-system
 ---
 apiVersion: v1
@@ -1268,7 +1275,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-8hk5mt9mg7
+          name: antrea-config-8964cgbht9
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -1517,7 +1524,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-8hk5mt9mg7
+          name: antrea-config-8964cgbht9
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml
@@ -1030,7 +1030,8 @@ data:
     featureGates:
     # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent.
     # It should be enabled on Windows, otherwise NetworkPolicy will not take effect on
-    # Service traffic.
+    # Service traffic. Antrea proxy doesn't support an IPv6 only cluster or a Dual-Stack cluster
+    # before PR #1102[https://github.com/vmware-tanzu/antrea/pull/1102] is merged.
       AntreaProxy: true
 
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
@@ -1078,6 +1079,12 @@ data:
     # for the GRE tunnel type.
     #enableIPSecTunnel: false
 
+    # ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack
+    # cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by
+    # --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed.
+    # No default value for this field.
+    #serviceCIDRv6:
+
     # Determines how traffic is encapsulated. It has the following options
     # encap(default): Inter-node Pod traffic is always encapsulated and Pod to outbound traffic is masqueraded.
     # noEncap: Inter-node Pod traffic is not encapsulated, but Pod to outbound traffic is masqueraded.
@@ -1161,7 +1168,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-8hk5mt9mg7
+  name: antrea-config-8964cgbht9
   namespace: kube-system
 ---
 apiVersion: v1
@@ -1268,7 +1275,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-8hk5mt9mg7
+          name: antrea-config-8964cgbht9
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -1519,7 +1526,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-8hk5mt9mg7
+          name: antrea-config-8964cgbht9
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml
@@ -1030,7 +1030,8 @@ data:
     featureGates:
     # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent.
     # It should be enabled on Windows, otherwise NetworkPolicy will not take effect on
-    # Service traffic.
+    # Service traffic. Antrea proxy doesn't support an IPv6 only cluster or a Dual-Stack cluster
+    # before PR #1102[https://github.com/vmware-tanzu/antrea/pull/1102] is merged.
       AntreaProxy: true
 
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
@@ -1078,6 +1079,12 @@ data:
     # for the GRE tunnel type.
     #enableIPSecTunnel: false
 
+    # ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack
+    # cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by
+    # --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed.
+    # No default value for this field.
+    #serviceCIDRv6:
+
     # Determines how traffic is encapsulated. It has the following options
     # encap(default): Inter-node Pod traffic is always encapsulated and Pod to outbound traffic is masqueraded.
     # noEncap: Inter-node Pod traffic is not encapsulated, but Pod to outbound traffic is masqueraded.
@@ -1161,7 +1168,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-26mthf882c
+  name: antrea-config-m4tmcc92d4
   namespace: kube-system
 ---
 apiVersion: v1
@@ -1268,7 +1275,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-26mthf882c
+          name: antrea-config-m4tmcc92d4
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -1517,7 +1524,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-26mthf882c
+          name: antrea-config-m4tmcc92d4
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml
@@ -1030,7 +1030,8 @@ data:
     featureGates:
     # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent.
     # It should be enabled on Windows, otherwise NetworkPolicy will not take effect on
-    # Service traffic.
+    # Service traffic. Antrea proxy doesn't support an IPv6 only cluster or a Dual-Stack cluster
+    # before PR #1102[https://github.com/vmware-tanzu/antrea/pull/1102] is merged.
     #  AntreaProxy: false
 
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
@@ -1083,6 +1084,12 @@ data:
     # AntreaProxy is enabled, this parameter is not needed and will be ignored if provided.
     #serviceCIDR: 10.96.0.0/12
 
+    # ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack
+    # cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by
+    # --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed.
+    # No default value for this field.
+    #serviceCIDRv6:
+
     # Determines how traffic is encapsulated. It has the following options
     # encap(default): Inter-node Pod traffic is always encapsulated and Pod to outbound traffic is masqueraded.
     # noEncap: Inter-node Pod traffic is not encapsulated, but Pod to outbound traffic is masqueraded.
@@ -1166,7 +1173,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-799khkd457
+  name: antrea-config-922fhdk74d
   namespace: kube-system
 ---
 apiVersion: v1
@@ -1282,7 +1289,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-799khkd457
+          name: antrea-config-922fhdk74d
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -1566,7 +1573,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-799khkd457
+          name: antrea-config-922fhdk74d
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml
@@ -1030,7 +1030,8 @@ data:
     featureGates:
     # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent.
     # It should be enabled on Windows, otherwise NetworkPolicy will not take effect on
-    # Service traffic.
+    # Service traffic. Antrea proxy doesn't support an IPv6 only cluster or a Dual-Stack cluster
+    # before PR #1102[https://github.com/vmware-tanzu/antrea/pull/1102] is merged.
     #  AntreaProxy: false
 
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
@@ -1083,6 +1084,12 @@ data:
     # AntreaProxy is enabled, this parameter is not needed and will be ignored if provided.
     #serviceCIDR: 10.96.0.0/12
 
+    # ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack
+    # cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by
+    # --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed.
+    # No default value for this field.
+    #serviceCIDRv6:
+
     # Determines how traffic is encapsulated. It has the following options
     # encap(default): Inter-node Pod traffic is always encapsulated and Pod to outbound traffic is masqueraded.
     # noEncap: Inter-node Pod traffic is not encapsulated, but Pod to outbound traffic is masqueraded.
@@ -1166,7 +1173,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-f4kt4bdh8t
+  name: antrea-config-742754m4hc
   namespace: kube-system
 ---
 apiVersion: v1
@@ -1273,7 +1280,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-f4kt4bdh8t
+          name: antrea-config-742754m4hc
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -1522,7 +1529,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-f4kt4bdh8t
+          name: antrea-config-742754m4hc
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/base/conf/antrea-agent.conf b/build/yamls/base/conf/antrea-agent.conf
@@ -2,7 +2,8 @@
 featureGates:
 # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent.
 # It should be enabled on Windows, otherwise NetworkPolicy will not take effect on
-# Service traffic.
+# Service traffic. Antrea proxy doesn't support an IPv6 only cluster or a Dual-Stack cluster
+# before PR #1102[https://github.com/vmware-tanzu/antrea/pull/1102] is merged.
 #  AntreaProxy: false
 
 # Enable traceflow which provides packet tracing feature to diagnose network issue.
@@ -55,6 +56,12 @@ featureGates:
 # AntreaProxy is enabled, this parameter is not needed and will be ignored if provided.
 #serviceCIDR: 10.96.0.0/12
 
+# ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack
+# cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by
+# --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed.
+# No default value for this field.
+#serviceCIDRv6:
+
 # Determines how traffic is encapsulated. It has the following options
 # encap(default): Inter-node Pod traffic is always encapsulated and Pod to outbound traffic is masqueraded.
 # noEncap: Inter-node Pod traffic is not encapsulated, but Pod to outbound traffic is masqueraded.

diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go
@@ -104,6 +104,12 @@ func run(o *Options) error {
 	}
 
 	_, serviceCIDRNet, _ := net.ParseCIDR(o.config.ServiceCIDR)
+	var serviceCIDRNetv6 *net.IPNet
+	// Todo: use FeatureGate to check if IPv6 is enabled and then read configuration item "ServiceCIDRv6".
+	if o.config.ServiceCIDRv6 != "" {
+		_, serviceCIDRNetv6, _ = net.ParseCIDR(o.config.ServiceCIDRv6)
+	}
+
 	_, encapMode := config.GetTrafficEncapModeFromStr(o.config.TrafficEncapMode)
 	networkConfig := &config.NetworkConfig{
 		TunnelType:        ovsconfig.TunnelType(o.config.TunnelType),
@@ -129,6 +135,7 @@ func run(o *Options) error {
 		o.config.HostGateway,
 		o.config.DefaultMTU,
 		serviceCIDRNet,
+		serviceCIDRNetv6,
 		networkConfig,
 		features.DefaultFeatureGate.Enabled(features.AntreaProxy))
 	err = agentInitializer.Initialize()

diff --git a/cmd/antrea-agent/config.go b/cmd/antrea-agent/config.go
@@ -68,6 +68,11 @@ type AgentConfig struct {
 	// AntreaProxy is enabled, this parameter is not needed and will be ignored if provided.
 	// Default is 10.96.0.0/12
 	ServiceCIDR string `yaml:"serviceCIDR,omitempty"`
+	// ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack
+	// cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by
+	// --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed.
+	// No default value for this field.
+	ServiceCIDRv6 string `yaml:"serviceCIDRv6,omitempty"`
 	// Whether or not to enable IPSec (ESP) encryption for Pod traffic across Nodes. IPSec encryption
 	// is supported only for the GRE tunnel type. Antrea uses Preshared Key (PSK) for IKE
 	// authentication. When IPSec tunnel is enabled, the PSK value must be passed to Antrea Agent

diff --git a/cmd/antrea-agent/options.go b/cmd/antrea-agent/options.go
@@ -84,7 +84,13 @@ func (o *Options) validate(args []string) error {
 	// Validate service CIDR configuration
 	_, _, err := net.ParseCIDR(o.config.ServiceCIDR)
 	if err != nil {
-		return fmt.Errorf("service CIDR %s is invalid", o.config.ServiceCIDR)
+		return fmt.Errorf("Service CIDR %s is invalid", o.config.ServiceCIDR)
+	}
+	if o.config.ServiceCIDRv6 != "" {
+		_, _, err := net.ParseCIDR(o.config.ServiceCIDRv6)
+		if err != nil {
+			return fmt.Errorf("Service CIDR v6 %s is invalid", o.config.ServiceCIDRv6)
+		}
 	}
 	if o.config.TunnelType != ovsconfig.VXLANTunnel && o.config.TunnelType != ovsconfig.GeneveTunnel &&
 		o.config.TunnelType != ovsconfig.GRETunnel && o.config.TunnelType != ovsconfig.STTTunnel {

diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go
@@ -64,6 +64,7 @@ type Initializer struct {
 	hostGateway     string // name of gateway port on the OVS bridge
 	mtu             int
 	serviceCIDR     *net.IPNet // K8s Service ClusterIP CIDR
+	serviceCIDRv6   *net.IPNet // K8s Service ClusterIP CIDR in IPv6
 	networkConfig   *config.NetworkConfig
 	nodeConfig      *config.NodeConfig
 	enableProxy     bool
@@ -79,6 +80,7 @@ func NewInitializer(
 	hostGateway string,
 	mtu int,
 	serviceCIDR *net.IPNet,
+	serviceCIDRv6 *net.IPNet,
 	networkConfig *config.NetworkConfig,
 	enableProxy bool) *Initializer {
 	return &Initializer{
@@ -91,6 +93,7 @@ func NewInitializer(
 		hostGateway:     hostGateway,
 		mtu:             mtu,
 		serviceCIDR:     serviceCIDR,
+		serviceCIDRv6:   serviceCIDRv6,
 		networkConfig:   networkConfig,
 		enableProxy:     enableProxy,
 	}
@@ -313,8 +316,8 @@ func (i *Initializer) initOpenFlowPipeline() error {
 		// from local Pods to any Service address can be forwarded to the host gateway interface
 		// correctly. Otherwise packets might be dropped by egress rules before they are DNATed to
 		// backend Pods.
-		if err := i.ofClient.InstallClusterServiceCIDRFlows(i.serviceCIDR, gatewayOFPort); err != nil {
-			klog.Errorf("Failed to setup openflow entries for Cluster Service CIDR %s: %v", i.serviceCIDR, err)
+		if err := i.ofClient.InstallClusterServiceCIDRFlows([]*net.IPNet{i.serviceCIDR, i.serviceCIDRv6}, gatewayOFPort); err != nil {
+			klog.Errorf("Failed to setup OpenFlow entries for Service CIDRs: %v", err)
 			return err
 		}
 	} else {