Add NodePort support for Antrea Proxy on Linux

Resolves antrea-io#1463. Signed-off-by: Weiqiang Tang <[email protected]>
weiqiangt · Feb 24, 2021 · 0b6dcd4 · 0b6dcd4
1 parent 1f7775e
commit 0b6dcd4
Show file tree

Hide file tree

Showing 31 changed files with 1,010 additions and 145 deletions.
diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml
@@ -1312,6 +1312,9 @@ data:
     # this flag will not take effect.
     #  EndpointSlice: false
 
+    # Enable NodePort Service support in AntreaProxy in antrea-agent.
+      AntreaProxyNodePort: true
+
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
     #  Traceflow: true
 
@@ -1434,6 +1437,10 @@ data:
 
     # TLS min version from: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13.
     #tlsMinVersion:
+
+    # A string slice of values which specify the addresses to use for NodePorts. Values may be valid IP blocks
+    # (e.g. 1.2.3.0/24, 1.2.3.4/32). The default empty string slice ([]) means to use all local addresses.
+    #nodePortAddresses: []
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -1499,7 +1506,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-mc9bkf47f4
+  name: antrea-config-hdm8fm2tkk
   namespace: kube-system
 ---
 apiVersion: v1
@@ -1619,7 +1626,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-mc9bkf47f4
+          name: antrea-config-hdm8fm2tkk
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -1883,7 +1890,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-mc9bkf47f4
+          name: antrea-config-hdm8fm2tkk
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml
@@ -1312,6 +1312,9 @@ data:
     # this flag will not take effect.
     #  EndpointSlice: false
 
+    # Enable NodePort Service support in AntreaProxy in antrea-agent.
+      AntreaProxyNodePort: true
+
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
     #  Traceflow: true
 
@@ -1434,6 +1437,10 @@ data:
 
     # TLS min version from: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13.
     #tlsMinVersion:
+
+    # A string slice of values which specify the addresses to use for NodePorts. Values may be valid IP blocks
+    # (e.g. 1.2.3.0/24, 1.2.3.4/32). The default empty string slice ([]) means to use all local addresses.
+    #nodePortAddresses: []
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -1499,7 +1506,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-mc9bkf47f4
+  name: antrea-config-hdm8fm2tkk
   namespace: kube-system
 ---
 apiVersion: v1
@@ -1619,7 +1626,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-mc9bkf47f4
+          name: antrea-config-hdm8fm2tkk
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -1885,7 +1892,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-mc9bkf47f4
+          name: antrea-config-hdm8fm2tkk
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml
@@ -1312,6 +1312,9 @@ data:
     # this flag will not take effect.
     #  EndpointSlice: false
 
+    # Enable NodePort Service support in AntreaProxy in antrea-agent.
+      AntreaProxyNodePort: true
+
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
     #  Traceflow: true
 
@@ -1434,6 +1437,10 @@ data:
 
     # TLS min version from: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13.
     #tlsMinVersion:
+
+    # A string slice of values which specify the addresses to use for NodePorts. Values may be valid IP blocks
+    # (e.g. 1.2.3.0/24, 1.2.3.4/32). The default empty string slice ([]) means to use all local addresses.
+    #nodePortAddresses: []
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -1499,7 +1506,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-bkc6dfdhgt
+  name: antrea-config-f7b6g9b8k2
   namespace: kube-system
 ---
 apiVersion: v1
@@ -1619,7 +1626,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-bkc6dfdhgt
+          name: antrea-config-f7b6g9b8k2
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -1886,7 +1893,7 @@ spec:
           path: /home/kubernetes/bin
         name: host-cni-bin
       - configMap:
-          name: antrea-config-bkc6dfdhgt
+          name: antrea-config-f7b6g9b8k2
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml
@@ -1312,6 +1312,9 @@ data:
     # this flag will not take effect.
     #  EndpointSlice: false
 
+    # Enable NodePort Service support in AntreaProxy in antrea-agent.
+      AntreaProxyNodePort: true
+
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
     #  Traceflow: true
 
@@ -1439,6 +1442,10 @@ data:
 
     # TLS min version from: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13.
     #tlsMinVersion:
+
+    # A string slice of values which specify the addresses to use for NodePorts. Values may be valid IP blocks
+    # (e.g. 1.2.3.0/24, 1.2.3.4/32). The default empty string slice ([]) means to use all local addresses.
+    #nodePortAddresses: []
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -1504,7 +1511,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-ftckkg2dc8
+  name: antrea-config-4g9mg9ckm8
   namespace: kube-system
 ---
 apiVersion: v1
@@ -1633,7 +1640,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-ftckkg2dc8
+          name: antrea-config-4g9mg9ckm8
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -1932,7 +1939,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-ftckkg2dc8
+          name: antrea-config-4g9mg9ckm8
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml
@@ -1312,6 +1312,9 @@ data:
     # this flag will not take effect.
     #  EndpointSlice: false
 
+    # Enable NodePort Service support in AntreaProxy in antrea-agent.
+      AntreaProxyNodePort: true
+
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
     #  Traceflow: true
 
@@ -1439,6 +1442,10 @@ data:
 
     # TLS min version from: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13.
     #tlsMinVersion:
+
+    # A string slice of values which specify the addresses to use for NodePorts. Values may be valid IP blocks
+    # (e.g. 1.2.3.0/24, 1.2.3.4/32). The default empty string slice ([]) means to use all local addresses.
+    #nodePortAddresses: []
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -1504,7 +1511,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-md64tc85t9
+  name: antrea-config-dgbk4m9gfb
   namespace: kube-system
 ---
 apiVersion: v1
@@ -1624,7 +1631,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-md64tc85t9
+          name: antrea-config-dgbk4m9gfb
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -1888,7 +1895,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-md64tc85t9
+          name: antrea-config-dgbk4m9gfb
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/base/conf/antrea-agent.conf b/build/yamls/base/conf/antrea-agent.conf
@@ -10,6 +10,9 @@ featureGates:
 # this flag will not take effect.
 #  EndpointSlice: false
 
+# Enable NodePort Service support in AntreaProxy in antrea-agent.
+  AntreaProxyNodePort: true
+
 # Enable traceflow which provides packet tracing feature to diagnose network issue.
 #  Traceflow: true
 
@@ -137,3 +140,7 @@ featureGates:
 
 # TLS min version from: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13.
 #tlsMinVersion:
+
+# A string slice of values which specify the addresses to use for NodePorts. Values may be valid IP blocks
+# (e.g. 1.2.3.0/24, 1.2.3.4/32). The default empty string slice ([]) means to use all local addresses.
+#nodePortAddresses: []
diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go
@@ -94,6 +94,8 @@ func run(o *Options) error {
 	ovsBridgeClient := ovsconfig.NewOVSBridge(o.config.OVSBridge, ovsDatapathType, ovsdbConnection)
 	ovsBridgeMgmtAddr := ofconfig.GetMgmtAddress(o.config.OVSRunDir, o.config.OVSBridge)
 	ofClient := openflow.NewClient(o.config.OVSBridge, ovsBridgeMgmtAddr, ovsDatapathType,
+		o.nodePortVirtualIP,
+		o.nodePortVirtualIPv6,
 		features.DefaultFeatureGate.Enabled(features.AntreaProxy),
 		features.DefaultFeatureGate.Enabled(features.AntreaPolicy))
 
@@ -110,7 +112,7 @@ func run(o *Options) error {
 		TrafficEncapMode:  encapMode,
 		EnableIPSecTunnel: o.config.EnableIPSecTunnel}
 
-	routeClient, err := route.NewClient(serviceCIDRNet, networkConfig, o.config.NoSNAT)
+	routeClient, err := route.NewClient(o.nodePortVirtualIP, o.nodePortVirtualIPv6, serviceCIDRNet, networkConfig, o.config.NoSNAT, features.DefaultFeatureGate.Enabled(features.AntreaProxyNodePort))
 	if err != nil {
 		return fmt.Errorf("error creating route client: %v", err)
 	}
@@ -192,15 +194,27 @@ func run(o *Options) error {
 	if features.DefaultFeatureGate.Enabled(features.AntreaProxy) {
 		v4Enabled := config.IsIPv4Enabled(nodeConfig, networkConfig.TrafficEncapMode)
 		v6Enabled := config.IsIPv6Enabled(nodeConfig, networkConfig.TrafficEncapMode)
+		var nodePortAddresses []*net.IPNet
+		nodePortSupport := features.DefaultFeatureGate.Enabled(features.AntreaProxyNodePort)
+		if nodePortSupport {
+			for _, nodePortAddress := range o.config.NodePortAddresses {
+				_, ipNet, _ := net.ParseCIDR(nodePortAddress)
+				nodePortAddresses = append(nodePortAddresses, ipNet)
+			}
+		}
+		var err error
 		switch {
 		case v4Enabled && v6Enabled:
-			proxier = proxy.NewDualStackProxier(nodeConfig.Name, informerFactory, ofClient)
+			proxier, err = proxy.NewDualStackProxier(o.nodePortVirtualIP, o.nodePortVirtualIPv6, nodePortAddresses, nodeConfig.Name, nodeConfig.PodIPv4CIDR, nodeConfig.PodIPv6CIDR, informerFactory, ofClient, routeClient, nodePortSupport)
 		case v4Enabled:
-			proxier = proxy.NewProxier(nodeConfig.Name, informerFactory, ofClient, false)
+			proxier, err = proxy.NewProxier(o.nodePortVirtualIP, nodePortAddresses, nodeConfig.Name, nodeConfig.PodIPv4CIDR, informerFactory, ofClient, routeClient, v6Enabled, nodePortSupport)
 		case v6Enabled:
-			proxier = proxy.NewProxier(nodeConfig.Name, informerFactory, ofClient, true)
+			proxier, err = proxy.NewProxier(o.nodePortVirtualIPv6, nodePortAddresses, nodeConfig.Name, nodeConfig.PodIPv4CIDR, informerFactory, ofClient, routeClient, v6Enabled, nodePortSupport)
 		default:
-			return fmt.Errorf("at least one of IPv4 or IPv6 should be enabled")
+			err = fmt.Errorf("at least one of IPv4 or IPv6 should be enabled")
+		}
+		if err != nil {
+			return fmt.Errorf("error when creating Antrea Proxy: %w", err)
 		}
 	}
 

diff --git a/cmd/antrea-agent/config.go b/cmd/antrea-agent/config.go
@@ -134,4 +134,7 @@ type AgentConfig struct {
 	TLSCipherSuites string `yaml:"tlsCipherSuites,omitempty"`
 	// TLS min version.
 	TLSMinVersion string `yaml:"tlsMinVersion,omitempty"`
+	// A string slice of values which specify the addresses to use for NodePorts. Values may be valid IP blocks
+	// (e.g. 1.2.3.0/24, 1.2.3.4/32). The default empty string slice ([]) means to use all local addresses.
+	NodePortAddresses []string `yaml:"nodePortAddresses,omitempty"`
 }
diff --git a/cmd/antrea-agent/options.go b/cmd/antrea-agent/options.go
@@ -43,6 +43,8 @@ const (
 	defaultFlowPollInterval       = 5 * time.Second
 	defaultFlowExportFrequency    = 12
 	defaultNPLPortRange           = "40000-41000"
+	defaultNodePortVirtualIP      = "169.254.169.110"
+	defaultNodePortVirtualIPv6    = "fec0::ffee:ddcc:bbaa"
 )
 
 type Options struct {
@@ -56,10 +58,14 @@ type Options struct {
 	flowCollectorProto string
 	// Flow exporter poll interval
 	pollInterval time.Duration
+	// The virtual IP for NodePort Service support.
+	nodePortVirtualIP, nodePortVirtualIPv6 net.IP
 }
 
 func newOptions() *Options {
 	return &Options{
+		nodePortVirtualIP:   net.ParseIP(defaultNodePortVirtualIP),
+		nodePortVirtualIPv6: net.ParseIP(defaultNodePortVirtualIPv6),
 		config: &AgentConfig{
 			EnablePrometheusMetrics:   true,
 			EnableTLSToFlowAggregator: true,
@@ -141,6 +147,9 @@ func (o *Options) validate(args []string) error {
 		// (but SNAT can be done by the primary CNI).
 		o.config.NoSNAT = true
 	}
+	if err := o.validateAntreaProxyConfig(); err != nil {
+		return fmt.Errorf("proxy config is invalid: %w", err)
+	}
 	if err := o.validateFlowExporterConfig(); err != nil {
 		return fmt.Errorf("failed to validate flow exporter config: %v", err)
 	}
@@ -208,6 +217,17 @@ func (o *Options) setDefaults() {
 	}
 }
 
+func (o *Options) validateAntreaProxyConfig() error {
+	if features.DefaultFeatureGate.Enabled(features.AntreaProxyNodePort) {
+		for _, nodePortAddress := range o.config.NodePortAddresses {
+			if _, _, err := net.ParseCIDR(nodePortAddress); err != nil {
+				return fmt.Errorf("NodePortAddress is not valid, can not parse `%s`: %w", nodePortAddress, err)
+			}
+		}
+	}
+	return nil
+}
+
 func (o *Options) validateFlowExporterConfig() error {
 	if features.DefaultFeatureGate.Enabled(features.FlowExporter) {
 		host, port, proto, err := flowexport.ParseFlowCollectorAddr(o.config.FlowCollectorAddr, defaultFlowCollectorPort, defaultFlowCollectorTransport)

diff --git a/hack/generate-manifest.sh b/hack/generate-manifest.sh
@@ -241,6 +241,7 @@ fi
 
 if $ALLFEATURES; then
     sed -i.bak -E "s/^[[:space:]]*#[[:space:]]*AntreaPolicy[[:space:]]*:[[:space:]]*[a-z]+[[:space:]]*$/  AntreaPolicy: true/" antrea-agent.conf
+    sed -i.bak -E "s/^[[:space:]]*#[[:space:]]*AntreaProxyNodePort[[:space:]]*:[[:space:]]*[a-z]+[[:space:]]*$/  AntreaProxyNodePort: true/" antrea-agent.conf
     sed -i.bak -E "s/^[[:space:]]*#[[:space:]]*FlowExporter[[:space:]]*:[[:space:]]*[a-z]+[[:space:]]*$/  FlowExporter: true/" antrea-agent.conf
     sed -i.bak -E "s/^[[:space:]]*#[[:space:]]*NetworkPolicyStats[[:space:]]*:[[:space:]]*[a-z]+[[:space:]]*$/  NetworkPolicyStats: true/" antrea-agent.conf
     sed -i.bak -E "s/^[[:space:]]*#[[:space:]]*EndpointSlice[[:space:]]*:[[:space:]]*[a-z]+[[:space:]]*$/  EndpointSlice: true/" antrea-agent.conf