[IPv6] Add support for dual-stack when using kube-proxy for Service (#…

…1200) 1. Add a config item for IPv6 Serivce CIDR if using kube-proxy to provide Service functions. 2. Output IPv6 traffic from host gateway if its destination is a Service address. 3. Use ct_mark to identify Service traffic and output the reply packet to the host gateway to ensure the DNAT processing in iptables.
antrea-io · Sep 21, 2020 · 8cb642f · 8cb642f
1 parent abffe7a
commit 8cb642f
Show file tree

Hide file tree

Showing 19 changed files with 329 additions and 179 deletions.
diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml
@@ -961,7 +961,8 @@ data:
     featureGates:
     # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent.
     # It should be enabled on Windows, otherwise NetworkPolicy will not take effect on
-    # Service traffic.
+    # Service traffic. Antrea proxy doesn't support an IPv6 only cluster or a Dual-Stack cluster
+    # before PR #1102[https://github.com/vmware-tanzu/antrea/pull/1102] is merged.
       AntreaProxy: true
 
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
@@ -1086,7 +1087,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-c5t898htbb
+  name: antrea-config-4c8bb4f97d
   namespace: kube-system
 ---
 apiVersion: v1
@@ -1193,7 +1194,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-c5t898htbb
+          name: antrea-config-4c8bb4f97d
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -1424,7 +1425,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-c5t898htbb
+          name: antrea-config-4c8bb4f97d
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml
@@ -961,7 +961,8 @@ data:
     featureGates:
     # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent.
     # It should be enabled on Windows, otherwise NetworkPolicy will not take effect on
-    # Service traffic.
+    # Service traffic. Antrea proxy doesn't support an IPv6 only cluster or a Dual-Stack cluster
+    # before PR #1102[https://github.com/vmware-tanzu/antrea/pull/1102] is merged.
       AntreaProxy: true
 
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
@@ -1086,7 +1087,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-c5t898htbb
+  name: antrea-config-4c8bb4f97d
   namespace: kube-system
 ---
 apiVersion: v1
@@ -1193,7 +1194,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-c5t898htbb
+          name: antrea-config-4c8bb4f97d
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -1426,7 +1427,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-c5t898htbb
+          name: antrea-config-4c8bb4f97d
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml
@@ -961,7 +961,8 @@ data:
     featureGates:
     # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent.
     # It should be enabled on Windows, otherwise NetworkPolicy will not take effect on
-    # Service traffic.
+    # Service traffic. Antrea proxy doesn't support an IPv6 only cluster or a Dual-Stack cluster
+    # before PR #1102[https://github.com/vmware-tanzu/antrea/pull/1102] is merged.
       AntreaProxy: true
 
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
@@ -1086,7 +1087,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-t47k4gbhkb
+  name: antrea-config-g86thkg7c9
   namespace: kube-system
 ---
 apiVersion: v1
@@ -1193,7 +1194,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-t47k4gbhkb
+          name: antrea-config-g86thkg7c9
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -1424,7 +1425,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-t47k4gbhkb
+          name: antrea-config-g86thkg7c9
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml
@@ -961,7 +961,8 @@ data:
     featureGates:
     # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent.
     # It should be enabled on Windows, otherwise NetworkPolicy will not take effect on
-    # Service traffic.
+    # Service traffic. Antrea proxy doesn't support an IPv6 only cluster or a Dual-Stack cluster
+    # before PR #1102[https://github.com/vmware-tanzu/antrea/pull/1102] is merged.
     #  AntreaProxy: false
 
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
@@ -1091,7 +1092,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-4k2khg5c24
+  name: antrea-config-mcd9664ghf
   namespace: kube-system
 ---
 apiVersion: v1
@@ -1207,7 +1208,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-4k2khg5c24
+          name: antrea-config-mcd9664ghf
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -1473,7 +1474,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-4k2khg5c24
+          name: antrea-config-mcd9664ghf
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml
@@ -961,7 +961,8 @@ data:
     featureGates:
     # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent.
     # It should be enabled on Windows, otherwise NetworkPolicy will not take effect on
-    # Service traffic.
+    # Service traffic. Antrea proxy doesn't support an IPv6 only cluster or a Dual-Stack cluster
+    # before PR #1102[https://github.com/vmware-tanzu/antrea/pull/1102] is merged.
     #  AntreaProxy: false
 
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
@@ -1091,7 +1092,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-b8h2ghckdc
+  name: antrea-config-6mf8hc86cg
   namespace: kube-system
 ---
 apiVersion: v1
@@ -1198,7 +1199,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-b8h2ghckdc
+          name: antrea-config-6mf8hc86cg
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -1429,7 +1430,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-b8h2ghckdc
+          name: antrea-config-6mf8hc86cg
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/base/conf/antrea-agent.conf b/build/yamls/base/conf/antrea-agent.conf
@@ -2,7 +2,8 @@
 featureGates:
 # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent.
 # It should be enabled on Windows, otherwise NetworkPolicy will not take effect on
-# Service traffic.
+# Service traffic. Antrea proxy doesn't support an IPv6 only cluster or a Dual-Stack cluster
+# before PR #1102[https://github.com/vmware-tanzu/antrea/pull/1102] is merged.
 #  AntreaProxy: false
 
 # Enable traceflow which provides packet tracing feature to diagnose network issue.

diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go
@@ -96,6 +96,12 @@ func run(o *Options) error {
 		features.DefaultFeatureGate.Enabled(features.AntreaPolicy))
 
 	_, serviceCIDRNet, _ := net.ParseCIDR(o.config.ServiceCIDR)
+	var serviceCIDRNetv6 *net.IPNet
+	// Todo: use FeatureGate to check if IPv6 is enabled and then read configuration item "ServiceCIDRv6".
+	if o.config.ServiceCIDRv6 != "" {
+		_, serviceCIDRNetv6, _ = net.ParseCIDR(o.config.ServiceCIDRv6)
+	}
+
 	_, encapMode := config.GetTrafficEncapModeFromStr(o.config.TrafficEncapMode)
 	networkConfig := &config.NetworkConfig{
 		TunnelType:        ovsconfig.TunnelType(o.config.TunnelType),
@@ -121,6 +127,7 @@ func run(o *Options) error {
 		o.config.HostGateway,
 		o.config.DefaultMTU,
 		serviceCIDRNet,
+		serviceCIDRNetv6,
 		networkConfig,
 		features.DefaultFeatureGate.Enabled(features.AntreaProxy))
 	err = agentInitializer.Initialize()

diff --git a/cmd/antrea-agent/config.go b/cmd/antrea-agent/config.go
@@ -68,6 +68,11 @@ type AgentConfig struct {
 	// AntreaProxy is enabled, this parameter is not needed and will be ignored if provided.
 	// Default is 10.96.0.0/12
 	ServiceCIDR string `yaml:"serviceCIDR,omitempty"`
+	// ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack
+	// cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by
+	// --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed.
+	// No default value for this field.
+	ServiceCIDRv6 string `yaml:"serviceCIDRv6,omitempty"`
 	// Whether or not to enable IPSec (ESP) encryption for Pod traffic across Nodes. IPSec encryption
 	// is supported only for the GRE tunnel type. Antrea uses Preshared Key (PSK) for IKE
 	// authentication. When IPSec tunnel is enabled, the PSK value must be passed to Antrea Agent

diff --git a/cmd/antrea-agent/options.go b/cmd/antrea-agent/options.go
@@ -84,7 +84,13 @@ func (o *Options) validate(args []string) error {
 	// Validate service CIDR configuration
 	_, _, err := net.ParseCIDR(o.config.ServiceCIDR)
 	if err != nil {
-		return fmt.Errorf("service CIDR %s is invalid", o.config.ServiceCIDR)
+		return fmt.Errorf("Service CIDR %s is invalid", o.config.ServiceCIDR)
+	}
+	if o.config.ServiceCIDRv6 != "" {
+		_, _, err := net.ParseCIDR(o.config.ServiceCIDRv6)
+		if err != nil {
+			return fmt.Errorf("Service CIDR v6 %s is invalid", o.config.ServiceCIDRv6)
+		}
 	}
 	if o.config.TunnelType != ovsconfig.VXLANTunnel && o.config.TunnelType != ovsconfig.GeneveTunnel &&
 		o.config.TunnelType != ovsconfig.GRETunnel && o.config.TunnelType != ovsconfig.STTTunnel {

diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go
@@ -64,6 +64,7 @@ type Initializer struct {
 	hostGateway     string // name of gateway port on the OVS bridge
 	mtu             int
 	serviceCIDR     *net.IPNet // K8s Service ClusterIP CIDR
+	serviceCIDRv6   *net.IPNet // K8s Service ClusterIP CIDR in IPv6
 	networkConfig   *config.NetworkConfig
 	nodeConfig      *config.NodeConfig
 	enableProxy     bool
@@ -79,6 +80,7 @@ func NewInitializer(
 	hostGateway string,
 	mtu int,
 	serviceCIDR *net.IPNet,
+	serviceCIDRv6 *net.IPNet,
 	networkConfig *config.NetworkConfig,
 	enableProxy bool) *Initializer {
 	return &Initializer{
@@ -91,6 +93,7 @@ func NewInitializer(
 		hostGateway:     hostGateway,
 		mtu:             mtu,
 		serviceCIDR:     serviceCIDR,
+		serviceCIDRv6:   serviceCIDRv6,
 		networkConfig:   networkConfig,
 		enableProxy:     enableProxy,
 	}
@@ -313,8 +316,8 @@ func (i *Initializer) initOpenFlowPipeline() error {
 		// from local Pods to any Service address can be forwarded to the host gateway interface
 		// correctly. Otherwise packets might be dropped by egress rules before they are DNATed to
 		// backend Pods.
-		if err := i.ofClient.InstallClusterServiceCIDRFlows(i.serviceCIDR, gateway.MAC, gatewayOFPort); err != nil {
-			klog.Errorf("Failed to setup openflow entries for Cluster Service CIDR %s: %v", i.serviceCIDR, err)
+		if err := i.ofClient.InstallClusterServiceCIDRFlows([]*net.IPNet{i.serviceCIDR, i.serviceCIDRv6}, gateway.MAC, gatewayOFPort); err != nil {
+			klog.Errorf("Failed to setup OpenFlow entries for Service CIDRss: %v", err)
 			return err
 		}
 	} else {

diff --git a/pkg/agent/openflow/client.go b/pkg/agent/openflow/client.go
@@ -51,7 +51,7 @@ type Client interface {
 	// InstallClusterServiceCIDRFlows sets up the appropriate flows so that traffic can reach
 	// the different Services running in the Cluster. This method needs to be invoked once with
 	// the Cluster Service CIDR as a parameter.
-	InstallClusterServiceCIDRFlows(serviceNet *net.IPNet, gatewayMAC net.HardwareAddr, gatewayOFPort uint32) error
+	InstallClusterServiceCIDRFlows(serviceNets []*net.IPNet, gatewayMAC net.HardwareAddr, gatewayOFPort uint32) error
 
 	// InstallClusterServiceFlows sets up the appropriate flows so that traffic can reach
 	// the different Services running in the Cluster. This method needs to be invoked once.
@@ -470,23 +470,22 @@ func (c *client) InstallClusterServiceFlows() error {
 	return nil
 }
 
-func (c *client) InstallClusterServiceCIDRFlows(serviceNet *net.IPNet, gatewayMAC net.HardwareAddr, gatewayOFPort uint32) error {
-	flow := c.serviceCIDRDNATFlow(serviceNet, gatewayMAC, gatewayOFPort)
-	if err := c.ofEntryOperations.Add(flow); err != nil {
+func (c *client) InstallClusterServiceCIDRFlows(serviceNets []*net.IPNet, gatewayMAC net.HardwareAddr, gatewayOFPort uint32) error {
+	flows := c.serviceCIDRDNATFlows(serviceNets, gatewayMAC, gatewayOFPort)
+	if err := c.ofEntryOperations.AddAll(flows); err != nil {
 		return err
 	}
-	c.defaultServiceFlows = []binding.Flow{flow}
+	c.defaultServiceFlows = flows
 	return nil
 }
 
 func (c *client) InstallGatewayFlows(gatewayAddrs []net.IP, gatewayMAC net.HardwareAddr, gatewayOFPort uint32) error {
 	flows := []binding.Flow{
 		c.gatewayClassifierFlow(gatewayOFPort, cookie.Default),
-		c.ctRewriteDstMACFlow(gatewayMAC, cookie.Default),
 		c.l2ForwardCalcFlow(gatewayMAC, gatewayOFPort, cookie.Default),
 	}
-	hasIPv6Addr := util.ContainIPv6Addr(gatewayAddrs)
-	flows = append(flows, c.gatewayIPSpoofGuardFlows(gatewayOFPort, hasIPv6Addr, cookie.Default)...)
+	hasV4, hasV6 := util.CheckAddressFamilies(gatewayAddrs)
+	flows = append(flows, c.gatewayIPSpoofGuardFlows(gatewayOFPort, hasV4, hasV6, cookie.Default)...)
 
 	// Add ARP SpoofGuard flow for local gateway interface.
 	gwIPv4 := util.GetIPv4Addr(gatewayAddrs)
@@ -495,7 +494,7 @@ func (c *client) InstallGatewayFlows(gatewayAddrs []net.IP, gatewayMAC net.Hardw
 	}
 	// Add flow to ensure the liveness check packet could be forwarded correctly.
 	flows = append(flows, c.localProbeFlow(gatewayAddrs, cookie.Default)...)
-
+	flows = append(flows, c.ctRewriteDstMACFlow(gatewayMAC, hasV4, hasV6, cookie.Default)...)
 	// In NoEncap , no traffic from tunnel port
 	if c.encapMode.SupportsEncap() {
 		flows = append(flows, c.l3ToGatewayFlow(gatewayAddrs, gatewayMAC, cookie.Default)...)