Add NodePort support for Antrea Proxy

Signed-off-by: Weiqiang Tang <[email protected]>
antrea-io · Jan 15, 2021 · d734e2b · d734e2b
1 parent 53e407e
commit d734e2b
Show file tree

Hide file tree

Showing 30 changed files with 1,008 additions and 133 deletions.
diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml
@@ -1170,6 +1170,9 @@ data:
     # Service traffic.
     #  AntreaProxy: true
 
+    # Enable NodePort Service support in AntreaProxy in antrea-agent.
+    #  AntreaProxyNodePort: true
+
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
     #  Traceflow: true
 
@@ -1276,6 +1279,16 @@ data:
     # whenever a Pod's container defines a specific port to be exposed (each container can define a list of ports as pod.spec.containers[].ports),
     # and all Node traffic directed to that port will be forwarded to the Pod.
     #nplPortRange: 40000-41000
+
+    # The virtual IP for NodePort Service support. It must be a link-local IP otherwise the Agents will report error.
+    #nodePortVirtualIP: 169.254.169.110
+
+    # The virtual IPv6 for NodePort Service support. It must not be a link-local IP otherwise the Agents will report error.
+    #nodePortVirtualIPv6: fec0::ffee:ddcc:bbaa
+
+    # A string slice of values which specify the addresses to use for NodePorts. Values may be valid IP blocks
+    # (e.g. 1.2.3.0/24, 1.2.3.4/32). The default empty string slice ([]) means to use all local addresses.
+    #nodePortAddresses: []
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -1332,7 +1345,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-b8hh7hm486
+  name: antrea-config-dccfh2f227
   namespace: kube-system
 ---
 apiVersion: v1
@@ -1443,7 +1456,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-b8hh7hm486
+          name: antrea-config-dccfh2f227
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -1708,7 +1721,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-b8hh7hm486
+          name: antrea-config-dccfh2f227
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml
@@ -1170,6 +1170,9 @@ data:
     # Service traffic.
     #  AntreaProxy: true
 
+    # Enable NodePort Service support in AntreaProxy in antrea-agent.
+    #  AntreaProxyNodePort: true
+
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
     #  Traceflow: true
 
@@ -1276,6 +1279,16 @@ data:
     # whenever a Pod's container defines a specific port to be exposed (each container can define a list of ports as pod.spec.containers[].ports),
     # and all Node traffic directed to that port will be forwarded to the Pod.
     #nplPortRange: 40000-41000
+
+    # The virtual IP for NodePort Service support. It must be a link-local IP otherwise the Agents will report error.
+    #nodePortVirtualIP: 169.254.169.110
+
+    # The virtual IPv6 for NodePort Service support. It must not be a link-local IP otherwise the Agents will report error.
+    #nodePortVirtualIPv6: fec0::ffee:ddcc:bbaa
+
+    # A string slice of values which specify the addresses to use for NodePorts. Values may be valid IP blocks
+    # (e.g. 1.2.3.0/24, 1.2.3.4/32). The default empty string slice ([]) means to use all local addresses.
+    #nodePortAddresses: []
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -1332,7 +1345,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-b8hh7hm486
+  name: antrea-config-dccfh2f227
   namespace: kube-system
 ---
 apiVersion: v1
@@ -1443,7 +1456,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-b8hh7hm486
+          name: antrea-config-dccfh2f227
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -1710,7 +1723,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-b8hh7hm486
+          name: antrea-config-dccfh2f227
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml
@@ -1170,6 +1170,9 @@ data:
     # Service traffic.
     #  AntreaProxy: true
 
+    # Enable NodePort Service support in AntreaProxy in antrea-agent.
+    #  AntreaProxyNodePort: true
+
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
     #  Traceflow: true
 
@@ -1276,6 +1279,16 @@ data:
     # whenever a Pod's container defines a specific port to be exposed (each container can define a list of ports as pod.spec.containers[].ports),
     # and all Node traffic directed to that port will be forwarded to the Pod.
     #nplPortRange: 40000-41000
+
+    # The virtual IP for NodePort Service support. It must be a link-local IP otherwise the Agents will report error.
+    #nodePortVirtualIP: 169.254.169.110
+
+    # The virtual IPv6 for NodePort Service support. It must not be a link-local IP otherwise the Agents will report error.
+    #nodePortVirtualIPv6: fec0::ffee:ddcc:bbaa
+
+    # A string slice of values which specify the addresses to use for NodePorts. Values may be valid IP blocks
+    # (e.g. 1.2.3.0/24, 1.2.3.4/32). The default empty string slice ([]) means to use all local addresses.
+    #nodePortAddresses: []
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -1332,7 +1345,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-hhfkgg2fg5
+  name: antrea-config-ft2hkgk4g5
   namespace: kube-system
 ---
 apiVersion: v1
@@ -1443,7 +1456,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-hhfkgg2fg5
+          name: antrea-config-ft2hkgk4g5
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -1711,7 +1724,7 @@ spec:
           path: /home/kubernetes/bin
         name: host-cni-bin
       - configMap:
-          name: antrea-config-hhfkgg2fg5
+          name: antrea-config-ft2hkgk4g5
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml
@@ -1170,6 +1170,9 @@ data:
     # Service traffic.
     #  AntreaProxy: true
 
+    # Enable NodePort Service support in AntreaProxy in antrea-agent.
+    #  AntreaProxyNodePort: true
+
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
     #  Traceflow: true
 
@@ -1281,6 +1284,16 @@ data:
     # whenever a Pod's container defines a specific port to be exposed (each container can define a list of ports as pod.spec.containers[].ports),
     # and all Node traffic directed to that port will be forwarded to the Pod.
     #nplPortRange: 40000-41000
+
+    # The virtual IP for NodePort Service support. It must be a link-local IP otherwise the Agents will report error.
+    #nodePortVirtualIP: 169.254.169.110
+
+    # The virtual IPv6 for NodePort Service support. It must not be a link-local IP otherwise the Agents will report error.
+    #nodePortVirtualIPv6: fec0::ffee:ddcc:bbaa
+
+    # A string slice of values which specify the addresses to use for NodePorts. Values may be valid IP blocks
+    # (e.g. 1.2.3.0/24, 1.2.3.4/32). The default empty string slice ([]) means to use all local addresses.
+    #nodePortAddresses: []
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -1337,7 +1350,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-bdc66g4872
+  name: antrea-config-h2d64449d2
   namespace: kube-system
 ---
 apiVersion: v1
@@ -1457,7 +1470,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-bdc66g4872
+          name: antrea-config-h2d64449d2
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -1757,7 +1770,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-bdc66g4872
+          name: antrea-config-h2d64449d2
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml
@@ -1170,6 +1170,9 @@ data:
     # Service traffic.
     #  AntreaProxy: true
 
+    # Enable NodePort Service support in AntreaProxy in antrea-agent.
+    #  AntreaProxyNodePort: true
+
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
     #  Traceflow: true
 
@@ -1281,6 +1284,16 @@ data:
     # whenever a Pod's container defines a specific port to be exposed (each container can define a list of ports as pod.spec.containers[].ports),
     # and all Node traffic directed to that port will be forwarded to the Pod.
     #nplPortRange: 40000-41000
+
+    # The virtual IP for NodePort Service support. It must be a link-local IP otherwise the Agents will report error.
+    #nodePortVirtualIP: 169.254.169.110
+
+    # The virtual IPv6 for NodePort Service support. It must not be a link-local IP otherwise the Agents will report error.
+    #nodePortVirtualIPv6: fec0::ffee:ddcc:bbaa
+
+    # A string slice of values which specify the addresses to use for NodePorts. Values may be valid IP blocks
+    # (e.g. 1.2.3.0/24, 1.2.3.4/32). The default empty string slice ([]) means to use all local addresses.
+    #nodePortAddresses: []
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -1337,7 +1350,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-9964gfgbb4
+  name: antrea-config-td97b2gf56
   namespace: kube-system
 ---
 apiVersion: v1
@@ -1448,7 +1461,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-9964gfgbb4
+          name: antrea-config-td97b2gf56
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -1713,7 +1726,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-9964gfgbb4
+          name: antrea-config-td97b2gf56
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/base/conf/antrea-agent.conf b/build/yamls/base/conf/antrea-agent.conf
@@ -5,6 +5,9 @@ featureGates:
 # Service traffic.
 #  AntreaProxy: true
 
+# Enable NodePort Service support in AntreaProxy in antrea-agent.
+#  AntreaProxyNodePort: true
+
 # Enable traceflow which provides packet tracing feature to diagnose network issue.
 #  Traceflow: true
 
@@ -116,3 +119,13 @@ featureGates:
 # whenever a Pod's container defines a specific port to be exposed (each container can define a list of ports as pod.spec.containers[].ports),
 # and all Node traffic directed to that port will be forwarded to the Pod.
 #nplPortRange: 40000-41000
+
+# The virtual IP for NodePort Service support. It must be a link-local IP otherwise the Agents will report error.
+#nodePortVirtualIP: 169.254.169.110
+
+# The virtual IPv6 for NodePort Service support. It must not be a link-local IP otherwise the Agents will report error.
+#nodePortVirtualIPv6: fec0::ffee:ddcc:bbaa
+
+# A string slice of values which specify the addresses to use for NodePorts. Values may be valid IP blocks
+# (e.g. 1.2.3.0/24, 1.2.3.4/32). The default empty string slice ([]) means to use all local addresses.
+#nodePortAddresses: []
diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go
@@ -89,9 +89,13 @@ func run(o *Options) error {
 	}
 	defer ovsdbConnection.Close()
 
+	nodePortVirtualIP := net.ParseIP(o.config.NodePortVirtualIP)
+	nodePortVirtualIPv6 := net.ParseIP(o.config.NodePortVirtualIPv6)
 	ovsBridgeClient := ovsconfig.NewOVSBridge(o.config.OVSBridge, o.config.OVSDatapathType, ovsdbConnection)
 	ovsBridgeMgmtAddr := ofconfig.GetMgmtAddress(o.config.OVSRunDir, o.config.OVSBridge)
 	ofClient := openflow.NewClient(o.config.OVSBridge, ovsBridgeMgmtAddr,
+		nodePortVirtualIP,
+		nodePortVirtualIPv6,
 		features.DefaultFeatureGate.Enabled(features.AntreaProxy),
 		features.DefaultFeatureGate.Enabled(features.AntreaPolicy))
 
@@ -108,7 +112,7 @@ func run(o *Options) error {
 		TrafficEncapMode:  encapMode,
 		EnableIPSecTunnel: o.config.EnableIPSecTunnel}
 
-	routeClient, err := route.NewClient(serviceCIDRNet, networkConfig, o.config.NoSNAT)
+	routeClient, err := route.NewClient(nodePortVirtualIP, nodePortVirtualIPv6, serviceCIDRNet, networkConfig, o.config.NoSNAT, features.DefaultFeatureGate.Enabled(features.AntreaProxyNodePort))
 	if err != nil {
 		return fmt.Errorf("error creating route client: %v", err)
 	}
@@ -180,18 +184,28 @@ func run(o *Options) error {
 
 	var proxier k8sproxy.Provider
 	if features.DefaultFeatureGate.Enabled(features.AntreaProxy) {
+		var nodePortAddresses []*net.IPNet
+		for _, nodePortAddress := range o.config.NodePortAddresses {
+			_, ipNet, _ := net.ParseCIDR(nodePortAddress)
+			nodePortAddresses = append(nodePortAddresses, ipNet)
+		}
 		v4Enabled := config.IsIPv4Enabled(nodeConfig, networkConfig.TrafficEncapMode)
 		v6Enabled := config.IsIPv6Enabled(nodeConfig, networkConfig.TrafficEncapMode)
+		nodePortSupport := features.DefaultFeatureGate.Enabled(features.AntreaProxyNodePort)
+		var err error
 		switch {
 		case v4Enabled && v6Enabled:
-			proxier = proxy.NewDualStackProxier(nodeConfig.Name, informerFactory, ofClient)
+			proxier, err = proxy.NewDualStackProxier(nodePortVirtualIP, nodePortVirtualIPv6, nodePortAddresses, nodeConfig.Name, nodeConfig.PodIPv4CIDR, nodeConfig.PodIPv6CIDR, informerFactory, ofClient, routeClient, nodePortSupport)
 		case v4Enabled:
-			proxier = proxy.NewProxier(nodeConfig.Name, informerFactory, ofClient, false)
+			proxier, err = proxy.NewProxier(nodePortVirtualIP, nodePortAddresses, nodeConfig.Name, nodeConfig.PodIPv4CIDR, informerFactory, ofClient, routeClient, false, nodePortSupport)
 		case v6Enabled:
-			proxier = proxy.NewProxier(nodeConfig.Name, informerFactory, ofClient, true)
+			proxier, err = proxy.NewProxier(nodePortVirtualIP, nodePortAddresses, nodeConfig.Name, nodeConfig.PodIPv6CIDR, informerFactory, ofClient, routeClient, true, nodePortSupport)
 		default:
 			return fmt.Errorf("at least one of IPv4 or IPv6 should be enabled")
 		}
+		if err != nil {
+			return fmt.Errorf("error when creating Antrea Proxy: %w", err)
+		}
 	}
 
 	isChaining := false

diff --git a/cmd/antrea-agent/config.go b/cmd/antrea-agent/config.go
@@ -124,4 +124,11 @@ type AgentConfig struct {
 	// whenever a Pod's container defines a specific port to be exposed (each container can define a list of ports as pod.spec.containers[].ports),
 	// and all Node traffic directed to that port will be forwarded to the Pod.
 	NPLPortRange string `yaml:"nplPortRange,omitempty"`
+	// The virtual IP for NodePort Service support. It must be a link-local IP otherwise the Agents will report error.
+	NodePortVirtualIP string `yaml:"nodePortVirtualIP,omitempty"`
+	// The virtual IPv6 for NodePort Service support. It must not be a link-local IP otherwise the Agents will report error.
+	NodePortVirtualIPv6 string `yaml:"nodePortVirtualIPv6,omitempty"`
+	// A string slice of values which specify the addresses to use for NodePorts. Values may be valid IP blocks
+	// (e.g. 1.2.3.0/24, 1.2.3.4/32). The default empty string slice ([]) means to use all local addresses.
+	NodePortAddresses []string `yaml:"nodePortAddresses,omitempty"`
 }