NodePort support for Antrea Proxy on Linux

Resolves antrea-io#1463.
hongliangl · Apr 25, 2021 · 9e79899 · 9e79899
1 parent 3335e73
commit 9e79899
Show file tree

Hide file tree

Showing 30 changed files with 1,034 additions and 140 deletions.
diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml
@@ -2424,6 +2424,9 @@ data:
     # this flag will not take effect.
     #  EndpointSlice: false
 
+    # Enable NodePort Service support in AntreaProxy in antrea-agent.
+      AntreaProxyNodePort: true
+
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
     #  Traceflow: true
 
@@ -2558,6 +2561,10 @@ data:
 
     # TLS min version from: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13.
     #tlsMinVersion:
+
+    # A string slice of values which specify the addresses to use for NodePorts. Values may be valid IP blocks
+    # (e.g. 1.2.3.0/24, 1.2.3.4/32). The default empty string slice ([]) means to use all local addresses.
+    #nodePortAddresses: []
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -2640,7 +2647,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-gg4m728h98
+  name: antrea-config-f74gh4mtc2
   namespace: kube-system
 ---
 apiVersion: v1
@@ -2760,7 +2767,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-gg4m728h98
+          name: antrea-config-f74gh4mtc2
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -3071,7 +3078,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-gg4m728h98
+          name: antrea-config-f74gh4mtc2
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml
@@ -2424,6 +2424,9 @@ data:
     # this flag will not take effect.
     #  EndpointSlice: false
 
+    # Enable NodePort Service support in AntreaProxy in antrea-agent.
+      AntreaProxyNodePort: true
+
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
     #  Traceflow: true
 
@@ -2558,6 +2561,10 @@ data:
 
     # TLS min version from: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13.
     #tlsMinVersion:
+
+    # A string slice of values which specify the addresses to use for NodePorts. Values may be valid IP blocks
+    # (e.g. 1.2.3.0/24, 1.2.3.4/32). The default empty string slice ([]) means to use all local addresses.
+    #nodePortAddresses: []
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -2640,7 +2647,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-gg4m728h98
+  name: antrea-config-f74gh4mtc2
   namespace: kube-system
 ---
 apiVersion: v1
@@ -2760,7 +2767,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-gg4m728h98
+          name: antrea-config-f74gh4mtc2
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -3073,7 +3080,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-gg4m728h98
+          name: antrea-config-f74gh4mtc2
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml
@@ -2424,6 +2424,9 @@ data:
     # this flag will not take effect.
     #  EndpointSlice: false
 
+    # Enable NodePort Service support in AntreaProxy in antrea-agent.
+      AntreaProxyNodePort: true
+
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
     #  Traceflow: true
 
@@ -2558,6 +2561,10 @@ data:
 
     # TLS min version from: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13.
     #tlsMinVersion:
+
+    # A string slice of values which specify the addresses to use for NodePorts. Values may be valid IP blocks
+    # (e.g. 1.2.3.0/24, 1.2.3.4/32). The default empty string slice ([]) means to use all local addresses.
+    #nodePortAddresses: []
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -2640,7 +2647,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-6bb22hc7fg
+  name: antrea-config-gm7dh5f556
   namespace: kube-system
 ---
 apiVersion: v1
@@ -2760,7 +2767,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-6bb22hc7fg
+          name: antrea-config-gm7dh5f556
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -3074,7 +3081,7 @@ spec:
           path: /home/kubernetes/bin
         name: host-cni-bin
       - configMap:
-          name: antrea-config-6bb22hc7fg
+          name: antrea-config-gm7dh5f556
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml
@@ -2424,6 +2424,9 @@ data:
     # this flag will not take effect.
     #  EndpointSlice: false
 
+    # Enable NodePort Service support in AntreaProxy in antrea-agent.
+      AntreaProxyNodePort: true
+
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
     #  Traceflow: true
 
@@ -2563,6 +2566,10 @@ data:
 
     # TLS min version from: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13.
     #tlsMinVersion:
+
+    # A string slice of values which specify the addresses to use for NodePorts. Values may be valid IP blocks
+    # (e.g. 1.2.3.0/24, 1.2.3.4/32). The default empty string slice ([]) means to use all local addresses.
+    #nodePortAddresses: []
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -2645,7 +2652,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-f57t688chc
+  name: antrea-config-ckh7bt6f4t
   namespace: kube-system
 ---
 apiVersion: v1
@@ -2774,7 +2781,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-f57t688chc
+          name: antrea-config-ckh7bt6f4t
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -3120,7 +3127,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-f57t688chc
+          name: antrea-config-ckh7bt6f4t
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml
@@ -2424,6 +2424,9 @@ data:
     # this flag will not take effect.
     #  EndpointSlice: false
 
+    # Enable NodePort Service support in AntreaProxy in antrea-agent.
+      AntreaProxyNodePort: true
+
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
     #  Traceflow: true
 
@@ -2563,6 +2566,10 @@ data:
 
     # TLS min version from: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13.
     #tlsMinVersion:
+
+    # A string slice of values which specify the addresses to use for NodePorts. Values may be valid IP blocks
+    # (e.g. 1.2.3.0/24, 1.2.3.4/32). The default empty string slice ([]) means to use all local addresses.
+    #nodePortAddresses: []
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -2645,7 +2652,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-5ct9ktdt77
+  name: antrea-config-8t8hbfmd84
   namespace: kube-system
 ---
 apiVersion: v1
@@ -2765,7 +2772,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-5ct9ktdt77
+          name: antrea-config-8t8hbfmd84
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -3076,7 +3083,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-5ct9ktdt77
+          name: antrea-config-8t8hbfmd84
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/base/conf/antrea-agent.conf b/build/yamls/base/conf/antrea-agent.conf
@@ -10,6 +10,9 @@ featureGates:
 # this flag will not take effect.
 #  EndpointSlice: false
 
+# Enable NodePort Service support in AntreaProxy in antrea-agent.
+  AntreaProxyNodePort: true
+
 # Enable traceflow which provides packet tracing feature to diagnose network issue.
 #  Traceflow: true
 
@@ -149,3 +152,7 @@ featureGates:
 
 # TLS min version from: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13.
 #tlsMinVersion:
+
+# A string slice of values which specify the addresses to use for NodePorts. Values may be valid IP blocks
+# (e.g. 1.2.3.0/24, 1.2.3.4/32). The default empty string slice ([]) means to use all local addresses.
+#nodePortAddresses: []
diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go
@@ -99,6 +99,8 @@ func run(o *Options) error {
 	ovsBridgeClient := ovsconfig.NewOVSBridge(o.config.OVSBridge, ovsDatapathType, ovsdbConnection)
 	ovsBridgeMgmtAddr := ofconfig.GetMgmtAddress(o.config.OVSRunDir, o.config.OVSBridge)
 	ofClient := openflow.NewClient(o.config.OVSBridge, ovsBridgeMgmtAddr, ovsDatapathType,
+		o.nodePortVirtualIP,
+		o.nodePortVirtualIPv6,
 		features.DefaultFeatureGate.Enabled(features.AntreaProxy),
 		features.DefaultFeatureGate.Enabled(features.AntreaPolicy),
 		features.DefaultFeatureGate.Enabled(features.Egress))
@@ -116,7 +118,7 @@ func run(o *Options) error {
 		TrafficEncapMode:  encapMode,
 		EnableIPSecTunnel: o.config.EnableIPSecTunnel}
 
-	routeClient, err := route.NewClient(serviceCIDRNet, networkConfig, o.config.NoSNAT)
+	routeClient, err := route.NewClient(o.nodePortVirtualIP, o.nodePortVirtualIPv6, serviceCIDRNet, networkConfig, o.config.NoSNAT, features.DefaultFeatureGate.Enabled(features.AntreaProxyNodePort))
 	if err != nil {
 		return fmt.Errorf("error creating route client: %v", err)
 	}
@@ -202,15 +204,52 @@ func run(o *Options) error {
 	if features.DefaultFeatureGate.Enabled(features.AntreaProxy) {
 		v4Enabled := config.IsIPv4Enabled(nodeConfig, networkConfig.TrafficEncapMode)
 		v6Enabled := config.IsIPv6Enabled(nodeConfig, networkConfig.TrafficEncapMode)
+		var nodePortAddresses []*net.IPNet
+		nodePortSupport := features.DefaultFeatureGate.Enabled(features.AntreaProxyNodePort)
+		if nodePortSupport {
+			for _, nodePortAddress := range o.config.NodePortAddresses {
+				_, ipNet, _ := net.ParseCIDR(nodePortAddress)
+				nodePortAddresses = append(nodePortAddresses, ipNet)
+			}
+		}
+		var err error
 		switch {
 		case v4Enabled && v6Enabled:
-			proxier = proxy.NewDualStackProxier(nodeConfig.Name, informerFactory, ofClient)
+			proxier = proxy.NewDualStackProxier(o.nodePortVirtualIP,
+				o.nodePortVirtualIPv6,
+				nodePortAddresses,
+				nodeConfig.Name,
+				nodeConfig.PodIPv4CIDR,
+				nodeConfig.PodIPv6CIDR,
+				informerFactory,
+				ofClient,
+				routeClient,
+				nodePortSupport)
 		case v4Enabled:
-			proxier = proxy.NewProxier(nodeConfig.Name, informerFactory, ofClient, false)
+			proxier = proxy.NewProxier(o.nodePortVirtualIP,
+				nodePortAddresses,
+				nodeConfig.Name,
+				nodeConfig.PodIPv4CIDR,
+				informerFactory,
+				ofClient,
+				routeClient,
+				v6Enabled,
+				nodePortSupport)
 		case v6Enabled:
-			proxier = proxy.NewProxier(nodeConfig.Name, informerFactory, ofClient, true)
+			proxier = proxy.NewProxier(o.nodePortVirtualIPv6,
+				nodePortAddresses,
+				nodeConfig.Name,
+				nodeConfig.PodIPv4CIDR,
+				informerFactory,
+				ofClient,
+				routeClient,
+				v6Enabled,
+				nodePortSupport)
 		default:
-			return fmt.Errorf("at least one of IPv4 or IPv6 should be enabled")
+			err = fmt.Errorf("at least one of IPv4 or IPv6 should be enabled")
+		}
+		if err != nil {
+			return fmt.Errorf("error when creating Antrea Proxy: %w", err)
 		}
 	}
 

diff --git a/cmd/antrea-agent/config.go b/cmd/antrea-agent/config.go
@@ -143,4 +143,7 @@ type AgentConfig struct {
 	TLSCipherSuites string `yaml:"tlsCipherSuites,omitempty"`
 	// TLS min version.
 	TLSMinVersion string `yaml:"tlsMinVersion,omitempty"`
+	// A string slice of values which specify the addresses to use for NodePorts. Values may be valid IP blocks
+	// (e.g. 1.2.3.0/24, 1.2.3.4/32). The default empty string slice ([]) means to use all local addresses.
+	NodePortAddresses []string `yaml:"nodePortAddresses,omitempty"`
 }
diff --git a/cmd/antrea-agent/options.go b/cmd/antrea-agent/options.go
@@ -42,9 +42,12 @@ const (
 	defaultFlowCollectorTransport  = "tcp"
 	defaultFlowCollectorPort       = "4739"
 	defaultFlowPollInterval        = 5 * time.Second
+	defaultFlowExportFrequency    = 12
 	defaultActiveFlowExportTimeout = 60 * time.Second
 	defaultIdleFlowExportTimeout   = 15 * time.Second
 	defaultNPLPortRange            = "40000-41000"
+	defaultNodePortVirtualIP      = "169.254.169.110"
+	defaultNodePortVirtualIPv6    = "fec0::ffee:ddcc:bbaa"
 )
 
 type Options struct {
@@ -62,10 +65,14 @@ type Options struct {
 	activeFlowTimeout time.Duration
 	// Idle flow timeout to export records of inactive flows
 	idleFlowTimeout time.Duration
+	// The virtual IP for NodePort Service support.
+	nodePortVirtualIP, nodePortVirtualIPv6 net.IP
 }
 
 func newOptions() *Options {
 	return &Options{
+		nodePortVirtualIP:   net.ParseIP(defaultNodePortVirtualIP),
+		nodePortVirtualIPv6: net.ParseIP(defaultNodePortVirtualIPv6),
 		config: &AgentConfig{
 			EnablePrometheusMetrics:   true,
 			EnableTLSToFlowAggregator: true,
@@ -147,6 +154,9 @@ func (o *Options) validate(args []string) error {
 		// (but SNAT can be done by the primary CNI).
 		o.config.NoSNAT = true
 	}
+	if err := o.validateAntreaProxyConfig(); err != nil {
+		return fmt.Errorf("proxy config is invalid: %w", err)
+	}
 	if err := o.validateFlowExporterConfig(); err != nil {
 		return fmt.Errorf("failed to validate flow exporter config: %v", err)
 	}
@@ -216,6 +226,17 @@ func (o *Options) setDefaults() {
 	}
 }
 
+func (o *Options) validateAntreaProxyConfig() error {
+	if features.DefaultFeatureGate.Enabled(features.AntreaProxyNodePort) {
+		for _, nodePortAddress := range o.config.NodePortAddresses {
+			if _, _, err := net.ParseCIDR(nodePortAddress); err != nil {
+				return fmt.Errorf("NodePortAddress is not valid, can not parse `%s`: %w", nodePortAddress, err)
+			}
+		}
+	}
+	return nil
+}
+
 func (o *Options) validateFlowExporterConfig() error {
 	if features.DefaultFeatureGate.Enabled(features.FlowExporter) {
 		host, port, proto, err := flowexport.ParseFlowCollectorAddr(o.config.FlowCollectorAddr, defaultFlowCollectorPort, defaultFlowCollectorTransport)