Graduate Egress feature from Alpha to Beta (#3509)

* Graduate Egress feature from Alpha to Beta Egress will be graduated to Beta and enabled by default starting with v1.6. Signed-off-by: Quan Tian <[email protected]> * Traffic to in-cluster destination should skip EgressMark table Only traffic to external traffic is supposed to go to EgressMark table as the table either redirects traffic to switching stage or drops it, skipping postRouting stage, in which phase SNAT is performed if required. Without this patch, traffic to remote Pods via gateway would go to EgressMark table when Egress feature is enabled. If the traffic needs to be SNATed, e.g. when an external client accesses a NodePort service which has an endpoint on a Node different from the accessed Node, there would be a connnection issue. Signed-off-by: Quan Tian <[email protected]>
antrea-io · Mar 26, 2022 · 8cbd5ba · 8cbd5ba
1 parent 739b8a1
commit 8cbd5ba
Show file tree

Hide file tree

Showing 14 changed files with 52 additions and 59 deletions.
diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml
@@ -2768,7 +2768,7 @@ data:
     #  NetworkPolicyStats: true
 
     # Enable controlling SNAT IPs of Pod egress traffic.
-    #  Egress: false
+    #  Egress: true
 
     # Enable AntreaIPAM, which can allocate IP addresses from IPPools. AntreaIPAM is required by the
     # bridging mode and allocates IPs to Pods in bridging mode.
@@ -3011,7 +3011,7 @@ data:
     #  NetworkPolicyStats: true
 
     # Enable controlling SNAT IPs of Pod egress traffic.
-    #  Egress: false
+    #  Egress: true
 
     # Run Kubernetes NodeIPAMController with Antrea.
     #  NodeIPAM: false
@@ -3074,7 +3074,7 @@ kind: ConfigMap
 metadata:
   labels:
     app: antrea
-  name: antrea-config-665mgk228m
+  name: antrea-config-82h2mk24gg
   namespace: kube-system
 ---
 apiVersion: v1
@@ -3145,7 +3145,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-665mgk228m
+          value: antrea-config-82h2mk24gg
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -3196,7 +3196,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-665mgk228m
+          name: antrea-config-82h2mk24gg
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -3435,7 +3435,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-665mgk228m
+          name: antrea-config-82h2mk24gg
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml
@@ -2768,7 +2768,7 @@ data:
     #  NetworkPolicyStats: true
 
     # Enable controlling SNAT IPs of Pod egress traffic.
-    #  Egress: false
+    #  Egress: true
 
     # Enable AntreaIPAM, which can allocate IP addresses from IPPools. AntreaIPAM is required by the
     # bridging mode and allocates IPs to Pods in bridging mode.
@@ -3011,7 +3011,7 @@ data:
     #  NetworkPolicyStats: true
 
     # Enable controlling SNAT IPs of Pod egress traffic.
-    #  Egress: false
+    #  Egress: true
 
     # Run Kubernetes NodeIPAMController with Antrea.
     #  NodeIPAM: false
@@ -3074,7 +3074,7 @@ kind: ConfigMap
 metadata:
   labels:
     app: antrea
-  name: antrea-config-665mgk228m
+  name: antrea-config-82h2mk24gg
   namespace: kube-system
 ---
 apiVersion: v1
@@ -3145,7 +3145,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-665mgk228m
+          value: antrea-config-82h2mk24gg
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -3196,7 +3196,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-665mgk228m
+          name: antrea-config-82h2mk24gg
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -3437,7 +3437,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-665mgk228m
+          name: antrea-config-82h2mk24gg
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml
@@ -2768,7 +2768,7 @@ data:
     #  NetworkPolicyStats: true
 
     # Enable controlling SNAT IPs of Pod egress traffic.
-    #  Egress: false
+    #  Egress: true
 
     # Enable AntreaIPAM, which can allocate IP addresses from IPPools. AntreaIPAM is required by the
     # bridging mode and allocates IPs to Pods in bridging mode.
@@ -3011,7 +3011,7 @@ data:
     #  NetworkPolicyStats: true
 
     # Enable controlling SNAT IPs of Pod egress traffic.
-    #  Egress: false
+    #  Egress: true
 
     # Run Kubernetes NodeIPAMController with Antrea.
     #  NodeIPAM: false
@@ -3074,7 +3074,7 @@ kind: ConfigMap
 metadata:
   labels:
     app: antrea
-  name: antrea-config-2cfft84t59
+  name: antrea-config-c9ck44454h
   namespace: kube-system
 ---
 apiVersion: v1
@@ -3145,7 +3145,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-2cfft84t59
+          value: antrea-config-c9ck44454h
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -3196,7 +3196,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-2cfft84t59
+          name: antrea-config-c9ck44454h
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -3438,7 +3438,7 @@ spec:
           path: /home/kubernetes/bin
         name: host-cni-bin
       - configMap:
-          name: antrea-config-2cfft84t59
+          name: antrea-config-c9ck44454h
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml
@@ -2768,7 +2768,7 @@ data:
     #  NetworkPolicyStats: true
 
     # Enable controlling SNAT IPs of Pod egress traffic.
-    #  Egress: false
+    #  Egress: true
 
     # Enable AntreaIPAM, which can allocate IP addresses from IPPools. AntreaIPAM is required by the
     # bridging mode and allocates IPs to Pods in bridging mode.
@@ -3016,7 +3016,7 @@ data:
     #  NetworkPolicyStats: true
 
     # Enable controlling SNAT IPs of Pod egress traffic.
-    #  Egress: false
+    #  Egress: true
 
     # Run Kubernetes NodeIPAMController with Antrea.
     #  NodeIPAM: false
@@ -3079,7 +3079,7 @@ kind: ConfigMap
 metadata:
   labels:
     app: antrea
-  name: antrea-config-2m674972kf
+  name: antrea-config-tmhkc66d6c
   namespace: kube-system
 ---
 apiVersion: v1
@@ -3159,7 +3159,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-2m674972kf
+          value: antrea-config-tmhkc66d6c
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -3210,7 +3210,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-2m674972kf
+          name: antrea-config-tmhkc66d6c
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -3484,7 +3484,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-2m674972kf
+          name: antrea-config-tmhkc66d6c
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml
@@ -2768,7 +2768,7 @@ data:
     #  NetworkPolicyStats: true
 
     # Enable controlling SNAT IPs of Pod egress traffic.
-    #  Egress: false
+    #  Egress: true
 
     # Enable AntreaIPAM, which can allocate IP addresses from IPPools. AntreaIPAM is required by the
     # bridging mode and allocates IPs to Pods in bridging mode.
@@ -3016,7 +3016,7 @@ data:
     #  NetworkPolicyStats: true
 
     # Enable controlling SNAT IPs of Pod egress traffic.
-    #  Egress: false
+    #  Egress: true
 
     # Run Kubernetes NodeIPAMController with Antrea.
     #  NodeIPAM: false
@@ -3079,7 +3079,7 @@ kind: ConfigMap
 metadata:
   labels:
     app: antrea
-  name: antrea-config-h6d7mmd9hg
+  name: antrea-config-hkhbh5gf99
   namespace: kube-system
 ---
 apiVersion: v1
@@ -3150,7 +3150,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-h6d7mmd9hg
+          value: antrea-config-hkhbh5gf99
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -3201,7 +3201,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-h6d7mmd9hg
+          name: antrea-config-hkhbh5gf99
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -3440,7 +3440,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-h6d7mmd9hg
+          name: antrea-config-hkhbh5gf99
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/base/conf/antrea-agent.conf b/build/yamls/base/conf/antrea-agent.conf
@@ -29,7 +29,7 @@ featureGates:
 #  NetworkPolicyStats: true
 
 # Enable controlling SNAT IPs of Pod egress traffic.
-#  Egress: false
+#  Egress: true
 
 # Enable AntreaIPAM, which can allocate IP addresses from IPPools. AntreaIPAM is required by the
 # bridging mode and allocates IPs to Pods in bridging mode.

diff --git a/build/yamls/base/conf/antrea-controller.conf b/build/yamls/base/conf/antrea-controller.conf
@@ -12,7 +12,7 @@ featureGates:
 #  NetworkPolicyStats: true
 
 # Enable controlling SNAT IPs of Pod egress traffic.
-#  Egress: false
+#  Egress: true
 
 # Run Kubernetes NodeIPAMController with Antrea.
 #  NodeIPAM: false

diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go
@@ -64,6 +64,7 @@ import (
 	"antrea.io/antrea/pkg/util/channel"
 	"antrea.io/antrea/pkg/util/cipher"
 	"antrea.io/antrea/pkg/util/k8s"
+	"antrea.io/antrea/pkg/util/runtime"
 	"antrea.io/antrea/pkg/version"
 )
 
@@ -82,6 +83,10 @@ var excludeNodePortDevices = []string{"antrea-egress0", "antrea-ingress0", "kube
 // run starts Antrea agent with the given options and waits for termination signal.
 func run(o *Options) error {
 	klog.Infof("Starting Antrea agent (version %s)", version.GetFullVersion())
+
+	// Windows platform doesn't support Egress feature yet.
+	egressEnabled := features.DefaultFeatureGate.Enabled(features.Egress) && !runtime.IsWindowsPlatform()
+
 	// Create K8s Clientset, CRD Clientset and SharedInformerFactory for the given config.
 	k8sClient, _, crdClient, _, err := k8s.CreateClients(o.config.ClientConnection, o.config.KubeAPIServerOverride)
 	if err != nil {
@@ -124,7 +129,7 @@ func run(o *Options) error {
 	ofClient := openflow.NewClient(o.config.OVSBridge, ovsBridgeMgmtAddr, ovsDatapathType,
 		features.DefaultFeatureGate.Enabled(features.AntreaProxy),
 		features.DefaultFeatureGate.Enabled(features.AntreaPolicy),
-		features.DefaultFeatureGate.Enabled(features.Egress),
+		egressEnabled,
 		features.DefaultFeatureGate.Enabled(features.FlowExporter),
 		o.config.AntreaProxy.ProxyAll,
 		connectUplinkToBridge,
@@ -306,7 +311,7 @@ func run(o *Options) error {
 	var externalIPController *serviceexternalip.ServiceExternalIPController
 	var memberlistCluster *memberlist.Cluster
 
-	if features.DefaultFeatureGate.Enabled(features.Egress) || features.DefaultFeatureGate.Enabled(features.ServiceExternalIP) {
+	if egressEnabled || features.DefaultFeatureGate.Enabled(features.ServiceExternalIP) {
 		externalIPPoolController = externalippool.NewExternalIPPoolController(
 			crdClient, externalIPPoolInformer,
 		)
@@ -325,7 +330,7 @@ func run(o *Options) error {
 			return fmt.Errorf("error creating new memberlist cluster: %v", err)
 		}
 	}
-	if features.DefaultFeatureGate.Enabled(features.Egress) {
+	if egressEnabled {
 		egressController, err = egress.NewEgressController(
 			ofClient, antreaClientProvider, crdClient, ifaceStore, routeClient, nodeConfig.Name, nodeConfig.NodeTransportInterfaceName,
 			memberlistCluster, egressInformer, podUpdateChannel,
@@ -528,12 +533,12 @@ func run(o *Options) error {
 	informerFactory.Start(stopCh)
 	crdInformerFactory.Start(stopCh)
 
-	if features.DefaultFeatureGate.Enabled(features.Egress) || features.DefaultFeatureGate.Enabled(features.ServiceExternalIP) {
+	if egressEnabled || features.DefaultFeatureGate.Enabled(features.ServiceExternalIP) {
 		go externalIPPoolController.Run(stopCh)
 		go memberlistCluster.Run(stopCh)
 	}
 
-	if features.DefaultFeatureGate.Enabled(features.Egress) {
+	if egressEnabled {
 		go egressController.Run(stopCh)
 	}
 

diff --git a/docs/egress.md b/docs/egress.md
@@ -40,10 +40,10 @@ This guide demonstrates how to configure `Egress` to achieve the above result.
 
 ## Prerequisites
 
-Egress is introduced in v1.0 as an alpha feature. As with other alpha features,
-a feature gate `Egress` must be enabled on the antrea-controller and
-antrea-agent for the feature to work. The following options in the
-`antrea-config` ConfigMap need to be set:
+Egress was introduced in v1.0 as an alpha feature, and was graduated to beta in
+v1.6, at which time it was enabled by default. Prior to v1.6, a feature gate,
+`Egress` must be enabled on the antrea-controller and antrea-agent in the
+`antrea-config` ConfigMap like the following options for the feature to work:
 
 ```yaml
 kind: ConfigMap

diff --git a/docs/feature-gates.md b/docs/feature-gates.md
@@ -42,7 +42,7 @@ example, to enable `AntreaProxy` on Linux, edit the Agent configuration in the
 | `FlowExporter`          | Agent              | `false` | Alpha | v0.9          | N/A          | N/A        | Yes                |       |
 | `NetworkPolicyStats`    | Agent + Controller | `true`  | Beta  | v0.10         | v1.2         | N/A        | No                 |       |
 | `NodePortLocal`         | Agent              | `true`  | Beta  | v0.13         | v1.4         | N/A        | Yes                | Important user-facing change in v1.2.0 |
-| `Egress`                | Agent + Controller | `false` | Alpha | v1.0          | N/A          | N/A        | Yes                |       |
+| `Egress`                | Agent + Controller | `true`  | Beta  | v1.0          | v1.6         | N/A        | Yes                |       |
 | `NodeIPAM`              | Controller         | `false` | Alpha | v1.4          | N/A          | N/A        | Yes                |       |
 | `AntreaIPAM`            | Agent + Controller | `false` | Alpha | v1.4          | N/A          | N/A        | Yes                |       |
 | `Multicast`             | Agent              | `false` | Alpha | v1.5          | N/A          | N/A        | Yes                |       |

diff --git a/pkg/agent/openflow/pipeline.go b/pkg/agent/openflow/pipeline.go
@@ -1371,7 +1371,7 @@ func (f *featurePodConnectivity) l3FwdFlowToRemoteViaGW(localGatewayMAC net.Hard
 	}
 	return fb.Action().SetDstMAC(localGatewayMAC).
 		Action().LoadRegMark(ToGatewayRegMark).
-		Action().NextTable().
+		Action().GotoTable(L3DecTTLTable.GetID()). // Traffic to in-cluster destination should skip EgressMark table.
 		Done()
 }
 

diff --git a/pkg/apiserver/handlers/featuregates/handler_test.go b/pkg/apiserver/handlers/featuregates/handler_test.go
@@ -47,7 +47,7 @@ func Test_getGatesResponse(t *testing.T) {
 			want: []Response{
 				{Component: "agent", Name: "AntreaPolicy", Status: "Disabled", Version: "BETA"},
 				{Component: "agent", Name: "AntreaProxy", Status: "Enabled", Version: "BETA"},
-				{Component: "agent", Name: "Egress", Status: "Disabled", Version: "ALPHA"},
+				{Component: "agent", Name: "Egress", Status: "Enabled", Version: "BETA"},
 				{Component: "agent", Name: "EndpointSlice", Status: "Disabled", Version: "ALPHA"},
 				{Component: "agent", Name: "AntreaIPAM", Status: "Disabled", Version: "ALPHA"},
 				{Component: "agent", Name: "Traceflow", Status: "Enabled", Version: "BETA"},
@@ -191,7 +191,7 @@ func Test_getControllerGatesResponse(t *testing.T) {
 			name: "good path",
 			want: []Response{
 				{Component: "controller", Name: "AntreaPolicy", Status: "Enabled", Version: "BETA"},
-				{Component: "controller", Name: "Egress", Status: "Disabled", Version: "ALPHA"},
+				{Component: "controller", Name: "Egress", Status: "Enabled", Version: "BETA"},
 				{Component: "controller", Name: "Traceflow", Status: "Enabled", Version: "BETA"},
 				{Component: "controller", Name: "NetworkPolicyStats", Status: "Enabled", Version: "BETA"},
 				{Component: "controller", Name: "NodeIPAM", Status: "Disabled", Version: "ALPHA"},