NodePort, LoadBalancer and ClusterIP from k8s Node support for Antrea…

…Proxy on Linux This PR implements: - The connection request of NodePort whose client is from remote or localhost. - The connection request of LoadBalancer whose client is from remote or localhost. - The connection request of ClusterIP whose client is from localhost. For NodePort support, on each interface whose IP addresses can be NodePort IP addresses, Linux TC is used to redirect the request packets to Antrea gateway. For response packets, on interface Antrea gateway, Linux TC is used to redirect the packets back to the interface where the requests packets are from. For LoadBalancer support, when client is from remote hosts, on default route output interface, Linux TC is used to redirect the request packets to Antrea gateway. For response packets, on interface Antrea gateway,Linux TC is used to redirect the packets back to the default route output interface. When client is from localhost, the request packets are routed to Antrea gateway. For ClusterIP support, the request packets are routed to Antrea gateway. To support the Service traffic of above cases, there are main changes of OVS pipeline. - Add table serviceConntrackCommitTable 106 to perform SNAT for Service traffic. - Modify table hairpinSNATTable ID from 106 to 108. - Modify table serviceHairpinTable ID from 29 to 23. - Add table serviceConntrackTable 24 to transform SNATed connnections. - Add table serviceClassifierTable 35 to classify the Service traffic. - Add table serviceDstMacRewriteTable 75 to rewrite destination MAC address of response Service packets. Signed-off-by: Hongliang Liu <[email protected]>
antrea-io · Jul 23, 2021 · 24b55d6 · 24b55d6
1 parent 7121422
commit 24b55d6
Show file tree

Hide file tree

Showing 44 changed files with 3,097 additions and 310 deletions.
diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml
@@ -3673,6 +3673,10 @@ data:
     # this flag will not take effect.
     #  EndpointSlice: false
 
+    # Enable full Service support in AntreaProxy in antrea-agent. All type of Services can be
+    # accessed from outside the cluster.
+    #  AntreaProxyFull: false
+
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
     #  Traceflow: true
 
@@ -3805,6 +3809,10 @@ data:
 
     # TLS min version from: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13.
     #tlsMinVersion:
+
+    # A string slice of values which specifies the host node IPv4/IPv6 addresses for NodePort. Values can be valid IP blocks.
+    # (e.g. 1.2.3.0/24, 1.2.3.4/32). An empty string slice is meant to select all host node IPv4/IPv6 addresses.
+    #nodePortAddresses: []
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -3887,7 +3895,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-gb8bgg4d2m
+  name: antrea-config-dfdc9mmtkb
   namespace: kube-system
 ---
 apiVersion: v1
@@ -3958,7 +3966,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-gb8bgg4d2m
+          value: antrea-config-dfdc9mmtkb
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -4009,7 +4017,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-gb8bgg4d2m
+          name: antrea-config-dfdc9mmtkb
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -4305,7 +4313,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-gb8bgg4d2m
+          name: antrea-config-dfdc9mmtkb
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml
@@ -3673,6 +3673,10 @@ data:
     # this flag will not take effect.
     #  EndpointSlice: false
 
+    # Enable full Service support in AntreaProxy in antrea-agent. All type of Services can be
+    # accessed from outside the cluster.
+    #  AntreaProxyFull: false
+
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
     #  Traceflow: true
 
@@ -3805,6 +3809,10 @@ data:
 
     # TLS min version from: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13.
     #tlsMinVersion:
+
+    # A string slice of values which specifies the host node IPv4/IPv6 addresses for NodePort. Values can be valid IP blocks.
+    # (e.g. 1.2.3.0/24, 1.2.3.4/32). An empty string slice is meant to select all host node IPv4/IPv6 addresses.
+    #nodePortAddresses: []
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -3887,7 +3895,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-gb8bgg4d2m
+  name: antrea-config-dfdc9mmtkb
   namespace: kube-system
 ---
 apiVersion: v1
@@ -3958,7 +3966,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-gb8bgg4d2m
+          value: antrea-config-dfdc9mmtkb
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -4009,7 +4017,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-gb8bgg4d2m
+          name: antrea-config-dfdc9mmtkb
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -4307,7 +4315,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-gb8bgg4d2m
+          name: antrea-config-dfdc9mmtkb
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml
@@ -3673,6 +3673,10 @@ data:
     # this flag will not take effect.
     #  EndpointSlice: false
 
+    # Enable full Service support in AntreaProxy in antrea-agent. All type of Services can be
+    # accessed from outside the cluster.
+    #  AntreaProxyFull: false
+
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
     #  Traceflow: true
 
@@ -3805,6 +3809,10 @@ data:
 
     # TLS min version from: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13.
     #tlsMinVersion:
+
+    # A string slice of values which specifies the host node IPv4/IPv6 addresses for NodePort. Values can be valid IP blocks.
+    # (e.g. 1.2.3.0/24, 1.2.3.4/32). An empty string slice is meant to select all host node IPv4/IPv6 addresses.
+    #nodePortAddresses: []
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -3887,7 +3895,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-4gm249hc95
+  name: antrea-config-72tdcbg8tb
   namespace: kube-system
 ---
 apiVersion: v1
@@ -3958,7 +3966,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-4gm249hc95
+          value: antrea-config-72tdcbg8tb
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -4009,7 +4017,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-4gm249hc95
+          name: antrea-config-72tdcbg8tb
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -4308,7 +4316,7 @@ spec:
           path: /home/kubernetes/bin
         name: host-cni-bin
       - configMap:
-          name: antrea-config-4gm249hc95
+          name: antrea-config-72tdcbg8tb
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml
@@ -3673,6 +3673,10 @@ data:
     # this flag will not take effect.
     #  EndpointSlice: false
 
+    # Enable full Service support in AntreaProxy in antrea-agent. All type of Services can be
+    # accessed from outside the cluster.
+    #  AntreaProxyFull: false
+
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
     #  Traceflow: true
 
@@ -3810,6 +3814,10 @@ data:
 
     # TLS min version from: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13.
     #tlsMinVersion:
+
+    # A string slice of values which specifies the host node IPv4/IPv6 addresses for NodePort. Values can be valid IP blocks.
+    # (e.g. 1.2.3.0/24, 1.2.3.4/32). An empty string slice is meant to select all host node IPv4/IPv6 addresses.
+    #nodePortAddresses: []
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -3892,7 +3900,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-4949869kb7
+  name: antrea-config-45k72h8864
   namespace: kube-system
 ---
 apiVersion: v1
@@ -3972,7 +3980,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-4949869kb7
+          value: antrea-config-45k72h8864
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -4023,7 +4031,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-4949869kb7
+          name: antrea-config-45k72h8864
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -4354,7 +4362,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-4949869kb7
+          name: antrea-config-45k72h8864
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml
@@ -3673,6 +3673,10 @@ data:
     # this flag will not take effect.
     #  EndpointSlice: false
 
+    # Enable full Service support in AntreaProxy in antrea-agent. All type of Services can be
+    # accessed from outside the cluster.
+    #  AntreaProxyFull: false
+
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
     #  Traceflow: true
 
@@ -3810,6 +3814,10 @@ data:
 
     # TLS min version from: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13.
     #tlsMinVersion:
+
+    # A string slice of values which specifies the host node IPv4/IPv6 addresses for NodePort. Values can be valid IP blocks.
+    # (e.g. 1.2.3.0/24, 1.2.3.4/32). An empty string slice is meant to select all host node IPv4/IPv6 addresses.
+    #nodePortAddresses: []
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -3892,7 +3900,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-dcfb6k2hkm
+  name: antrea-config-m82h754m26
   namespace: kube-system
 ---
 apiVersion: v1
@@ -3963,7 +3971,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-dcfb6k2hkm
+          value: antrea-config-m82h754m26
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -4014,7 +4022,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-dcfb6k2hkm
+          name: antrea-config-m82h754m26
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -4310,7 +4318,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-dcfb6k2hkm
+          name: antrea-config-m82h754m26
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/base/conf/antrea-agent.conf b/build/yamls/base/conf/antrea-agent.conf
@@ -10,6 +10,10 @@ featureGates:
 # this flag will not take effect.
 #  EndpointSlice: false
 
+# Enable full Service support in AntreaProxy in antrea-agent. All type of Services can be
+# accessed from outside the cluster.
+#  AntreaProxyFull: false
+
 # Enable traceflow which provides packet tracing feature to diagnose network issue.
 #  Traceflow: true
 
@@ -147,3 +151,7 @@ featureGates:
 
 # TLS min version from: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13.
 #tlsMinVersion:
+
+# A string slice of values which specifies the host node IPv4/IPv6 addresses for NodePort. Values can be valid IP blocks.
+# (e.g. 1.2.3.0/24, 1.2.3.4/32). An empty string slice is meant to select all host node IPv4/IPv6 addresses.
+#nodePortAddresses: []
diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go
@@ -43,6 +43,7 @@ import (
 	"antrea.io/antrea/pkg/agent/route"
 	"antrea.io/antrea/pkg/agent/stats"
 	"antrea.io/antrea/pkg/agent/types"
+	"antrea.io/antrea/pkg/agent/util"
 	crdinformers "antrea.io/antrea/pkg/client/informers/externalversions"
 	"antrea.io/antrea/pkg/features"
 	"antrea.io/antrea/pkg/log"
@@ -104,7 +105,8 @@ func run(o *Options) error {
 		features.DefaultFeatureGate.Enabled(features.AntreaProxy),
 		features.DefaultFeatureGate.Enabled(features.AntreaPolicy),
 		features.DefaultFeatureGate.Enabled(features.Egress),
-		features.DefaultFeatureGate.Enabled(features.FlowExporter))
+		features.DefaultFeatureGate.Enabled(features.FlowExporter),
+		features.DefaultFeatureGate.Enabled(features.AntreaProxyFull))
 
 	_, serviceCIDRNet, _ := net.ParseCIDR(o.config.ServiceCIDR)
 	var serviceCIDRNetv6 *net.IPNet
@@ -168,15 +170,23 @@ func run(o *Options) error {
 
 	var proxier proxy.Proxier
 	if features.DefaultFeatureGate.Enabled(features.AntreaProxy) {
+		var nodePortIPMap, nodePortIPv6Map map[int][]net.IP
+		if features.DefaultFeatureGate.Enabled(features.AntreaProxyFull) {
+			nodePortIPMap, nodePortIPv6Map, err = util.GetAvailableNodePortIPs(o.config.NodePortAddresses, o.config.HostGateway)
+			if err != nil {
+				return fmt.Errorf("get available NodePort IP addresses with error: %v", err)
+			}
+		}
+
 		v4Enabled := config.IsIPv4Enabled(nodeConfig, networkConfig.TrafficEncapMode)
 		v6Enabled := config.IsIPv6Enabled(nodeConfig, networkConfig.TrafficEncapMode)
 		switch {
 		case v4Enabled && v6Enabled:
-			proxier = proxy.NewDualStackProxier(nodeConfig.Name, informerFactory, ofClient)
+			proxier = proxy.NewDualStackProxier(nodeConfig.Name, informerFactory, ofClient, routeClient, nodePortIPMap, nodePortIPv6Map)
 		case v4Enabled:
-			proxier = proxy.NewProxier(nodeConfig.Name, informerFactory, ofClient, false)
+			proxier = proxy.NewProxier(nodeConfig.Name, informerFactory, ofClient, false, routeClient, nodePortIPMap)
 		case v6Enabled:
-			proxier = proxy.NewProxier(nodeConfig.Name, informerFactory, ofClient, true)
+			proxier = proxy.NewProxier(nodeConfig.Name, informerFactory, ofClient, true, routeClient, nodePortIPv6Map)
 		default:
 			return fmt.Errorf("at least one of IPv4 or IPv6 should be enabled")
 		}

diff --git a/cmd/antrea-agent/config.go b/cmd/antrea-agent/config.go
@@ -148,4 +148,7 @@ type AgentConfig struct {
 	TLSCipherSuites string `yaml:"tlsCipherSuites,omitempty"`
 	// TLS min version.
 	TLSMinVersion string `yaml:"tlsMinVersion,omitempty"`
+	// A string slice of values which specifies the host IPv4/IPv6 addresses for NodePorts. Values may be valid IP blocks.
+	// (e.g. 1.2.3.0/24, 1.2.3.4/32). An empty string slice is meant to select all host node IPv4/IPv6 addresses.
+	NodePortAddresses []string `yaml:"nodePortAddresses,omitempty"`
 }
diff --git a/cmd/antrea-agent/options.go b/cmd/antrea-agent/options.go
@@ -146,6 +146,9 @@ func (o *Options) validate(args []string) error {
 		// (but SNAT can be done by the primary CNI).
 		o.config.NoSNAT = true
 	}
+	if err := o.validateAntreaProxyConfig(); err != nil {
+		return fmt.Errorf("proxy config is invalid: %w", err)
+	}
 	if err := o.validateFlowExporterConfig(); err != nil {
 		return fmt.Errorf("failed to validate flow exporter config: %v", err)
 	}
@@ -219,6 +222,17 @@ func (o *Options) setDefaults() {
 	}
 }
 
+func (o *Options) validateAntreaProxyConfig() error {
+	if features.DefaultFeatureGate.Enabled(features.AntreaProxyFull) {
+		for _, nodePortIP := range o.config.NodePortAddresses {
+			if _, _, err := net.ParseCIDR(nodePortIP); err != nil {
+				return fmt.Errorf("invalid NodePort IP address `%s`: %w", nodePortIP, err)
+			}
+		}
+	}
+	return nil
+}
+
 func (o *Options) validateFlowExporterConfig() error {
 	if features.DefaultFeatureGate.Enabled(features.FlowExporter) {
 		host, port, proto, err := flowexport.ParseFlowCollectorAddr(o.config.FlowCollectorAddr, defaultFlowCollectorPort, defaultFlowCollectorTransport)