Support --random-fully for iptables SNAT / MASQUERADE rules (#6602)

We expose 2 new configuration parameters for the Agent:
snatFullyRandomPorts and egress.snatFullyRandomPorts. When the first one
is set to true, the MASQUERADE iptables rules used to implement "local"
Node SNAT will use randomized source port mappings. When the second one
is set to true, the SNAT iptables rules used to implement Egress SNAT
will use randomized source port mappings. This is achieved with the
`--random-fully` iptables flag.

These new configuration parameters are only supported on Linux
Nodes. They require iptables >= 1.6.2, which is not a problem with the
standard antrea-agent container images. They also require Linux kernel
>= 3.13 and >= 3.14 respectively, but these are very old releases so we
can ignore this requirment, which is consistent with the K8s
implementation.

snatFullyRandomPorts is set to false by default (this may change in the
future). egress.snatFullyRandomPorts is set to null (empty) by default,
which means that unless the parameter is explicitly set, we will use the
top-level snatFullyRandomPorts value.

Fixes #6544

Signed-off-by: Antonin Bas <[email protected]>

build/charts/antrea/README.md

@@ -89,6 +89,7 @@ Kubernetes: `>= 1.19.0-0`
 | dnsServerOverride | string | `""` | Address of DNS server, to override the kube-dns Service. It's used to resolve hostnames in a FQDN policy. |
 | egress.exceptCIDRs | list | `[]` | CIDR ranges to which outbound Pod traffic will not be SNAT'd by Egresses. |
 | egress.maxEgressIPsPerNode | int | `255` | The maximum number of Egress IPs that can be assigned to a Node. It is useful when the Node network restricts the number of secondary IPs a Node can have, e.g. EKS. It must not be greater than 255. |
+| egress.snatFullyRandomPorts | bool | `nil` | Fully randomize source port mapping in Egress SNAT rules. This has no impact on the default SNAT rules enforced by each Node for local Pod traffic. By default, we use the same value as for the top-level snatFullyRandomPorts configuration, but this field can be used as an override. |
 | enableBridgingMode | bool | `false` | Enable bridging mode of Pod network on Nodes, in which the Node's transport interface is connected to the OVS bridge. |
 | featureGates | object | `{}` | To explicitly enable or disable a FeatureGate and bypass the Antrea defaults, add an entry to the dictionary with the FeatureGate's name as the key and a boolean as the value. |
 | flowExporter.activeFlowExportTimeout | string | `"5s"` | timeout after which a flow record is sent to the collector for active flows. |
@@ -129,6 +130,7 @@ Kubernetes: `>= 1.19.0-0`
 | secondaryNetwork.ovsBridges | list | `[]` | Configuration of OVS bridges for secondary network. At the moment, at most one OVS bridge can be specified. If the specified bridge does not exist on the Node, antrea-agent will create it based on the configuration. The following configuration specifies an OVS bridge with name "br1" and a physical interface "eth1": [{bridgeName: "br1", physicalInterfaces: ["eth1"]}] |
 | serviceCIDR | string | `""` | IPv4 CIDR range used for Services. Required when AntreaProxy is disabled. |
 | serviceCIDRv6 | string | `""` | IPv6 CIDR range used for Services. Required when AntreaProxy is disabled. |
+| snatFullyRandomPorts | bool | `false` | Fully randomize source port mapping in SNAT rules used for egress traffic from Pods to the external network. |
 | testing.coverage | bool | `false` | Enable code coverage measurement (used when testing Antrea only). |
 | testing.simulator.enable | bool | `false` |  |
 | tlsCipherSuites | string | `""` | Comma-separated list of cipher suites that will be used by the Antrea APIservers. If empty, the default Go Cipher Suites will be used. See https://golang.org/pkg/crypto/tls/#pkg-constants. |

build/charts/antrea/conf/antrea-agent.conf

@@ -130,6 +130,10 @@ trafficEncapMode: {{ .Values.trafficEncapMode | quote }}
 # performs SNAT and this option will be ignored; for other modes it must be set to false.
 noSNAT: {{ .Values.noSNAT }}
 
+# Fully randomize source port mapping in SNAT rules used for egress traffic from Pods to the
+# external network.
+snatFullyRandomPorts: {{ .Values.snatFullyRandomPorts }}
+
 # Tunnel protocols used for encapsulating traffic across Nodes. If WireGuard is enabled in trafficEncryptionMode,
 # this option will not take effect. Supported values:
 # - geneve (default)
@@ -206,6 +210,14 @@ egress:
   # The maximum number of Egress IPs that can be assigned to a Node. It's useful when the Node network restricts
   # the number of secondary IPs a Node can have, e.g. EKS. It must not be greater than 255.
   maxEgressIPsPerNode: {{ .maxEgressIPsPerNode }}
+  # Fully randomize source port mapping in Egress SNAT rules. This has no impact on the default SNAT
+  # rules enforced by each Node for local Pod traffic. By default, we use the same value as for the
+  # top-level snatFullyRandomPorts configuration, but this field can be used as an override.
+  {{- if eq .snatFullyRandomPorts nil }}
+  snatFullyRandomPorts:
+  {{- else }}
+  snatFullyRandomPorts: {{ .snatFullyRandomPorts }}
+  {{- end }}
 {{- end }}
 
 # ClusterIP CIDR range for Services. It's required when AntreaProxy is not enabled, and should be

build/charts/antrea/values.yaml

@@ -45,6 +45,9 @@ disableTXChecksumOffload: false
 # -- Whether or not to SNAT (using the Node IP) the egress traffic from a Pod to
 # the external network.
 noSNAT: false
+# -- Fully randomize source port mapping in SNAT rules used for egress traffic
+# from Pods to the external network.
+snatFullyRandomPorts: false
 # -- Name of the interface antrea-agent will create and use for host <-> Pod
 # communication.
 hostGateway: "antrea-gw0"
@@ -113,6 +116,12 @@ egress:
   # useful when the Node network restricts the number of secondary IPs a Node
   # can have, e.g. EKS. It must not be greater than 255.
   maxEgressIPsPerNode: 255
+  # -- (bool) Fully randomize source port mapping in Egress SNAT rules. This has
+  # no impact on the default SNAT rules enforced by each Node for local Pod
+  # traffic. By default, we use the same value as for the top-level
+  # snatFullyRandomPorts configuration, but this field can be used as an
+  # override.
+  snatFullyRandomPorts:
 
 nodePortLocal:
   # -- Enable the NodePortLocal feature.

build/yamls/antrea-aks.yml

@@ -3846,6 +3846,10 @@ data:
     # performs SNAT and this option will be ignored; for other modes it must be set to false.
     noSNAT: false
 
+    # Fully randomize source port mapping in SNAT rules used for egress traffic from Pods to the
+    # external network.
+    snatFullyRandomPorts: false
+
     # Tunnel protocols used for encapsulating traffic across Nodes. If WireGuard is enabled in trafficEncryptionMode,
     # this option will not take effect. Supported values:
     # - geneve (default)
@@ -3916,6 +3920,10 @@ data:
       # The maximum number of Egress IPs that can be assigned to a Node. It's useful when the Node network restricts
       # the number of secondary IPs a Node can have, e.g. EKS. It must not be greater than 255.
       maxEgressIPsPerNode: 255
+      # Fully randomize source port mapping in Egress SNAT rules. This has no impact on the default SNAT
+      # rules enforced by each Node for local Pod traffic. By default, we use the same value as for the
+      # top-level snatFullyRandomPorts configuration, but this field can be used as an override.
+      snatFullyRandomPorts:
 
     # ClusterIP CIDR range for Services. It's required when AntreaProxy is not enabled, and should be
     # set to the same value as the one specified by --service-cluster-ip-range for kube-apiserver. When
@@ -5130,7 +5138,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 452b01033d20173605ec14cd64eea22a52287f84998c2d4fc7f9c3ce19c907b9
+        checksum/config: 4325a243ab510df539883b6384a30cf8b04ff862796444a6c5c10999159479c5
       labels:
         app: antrea
         component: antrea-agent
@@ -5368,7 +5376,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 452b01033d20173605ec14cd64eea22a52287f84998c2d4fc7f9c3ce19c907b9
+        checksum/config: 4325a243ab510df539883b6384a30cf8b04ff862796444a6c5c10999159479c5
       labels:
         app: antrea
         component: antrea-controller

build/yamls/antrea-eks.yml

@@ -3846,6 +3846,10 @@ data:
     # performs SNAT and this option will be ignored; for other modes it must be set to false.
     noSNAT: false
 
+    # Fully randomize source port mapping in SNAT rules used for egress traffic from Pods to the
+    # external network.
+    snatFullyRandomPorts: false
+
     # Tunnel protocols used for encapsulating traffic across Nodes. If WireGuard is enabled in trafficEncryptionMode,
     # this option will not take effect. Supported values:
     # - geneve (default)
@@ -3916,6 +3920,10 @@ data:
       # The maximum number of Egress IPs that can be assigned to a Node. It's useful when the Node network restricts
       # the number of secondary IPs a Node can have, e.g. EKS. It must not be greater than 255.
       maxEgressIPsPerNode: 255
+      # Fully randomize source port mapping in Egress SNAT rules. This has no impact on the default SNAT
+      # rules enforced by each Node for local Pod traffic. By default, we use the same value as for the
+      # top-level snatFullyRandomPorts configuration, but this field can be used as an override.
+      snatFullyRandomPorts:
 
     # ClusterIP CIDR range for Services. It's required when AntreaProxy is not enabled, and should be
     # set to the same value as the one specified by --service-cluster-ip-range for kube-apiserver. When
@@ -5130,7 +5138,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 452b01033d20173605ec14cd64eea22a52287f84998c2d4fc7f9c3ce19c907b9
+        checksum/config: 4325a243ab510df539883b6384a30cf8b04ff862796444a6c5c10999159479c5
       labels:
         app: antrea
         component: antrea-agent
@@ -5369,7 +5377,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 452b01033d20173605ec14cd64eea22a52287f84998c2d4fc7f9c3ce19c907b9
+        checksum/config: 4325a243ab510df539883b6384a30cf8b04ff862796444a6c5c10999159479c5
       labels:
         app: antrea
         component: antrea-controller

build/yamls/antrea-gke.yml

@@ -3846,6 +3846,10 @@ data:
     # performs SNAT and this option will be ignored; for other modes it must be set to false.
     noSNAT: false
 
+    # Fully randomize source port mapping in SNAT rules used for egress traffic from Pods to the
+    # external network.
+    snatFullyRandomPorts: false
+
     # Tunnel protocols used for encapsulating traffic across Nodes. If WireGuard is enabled in trafficEncryptionMode,
     # this option will not take effect. Supported values:
     # - geneve (default)
@@ -3916,6 +3920,10 @@ data:
       # The maximum number of Egress IPs that can be assigned to a Node. It's useful when the Node network restricts
       # the number of secondary IPs a Node can have, e.g. EKS. It must not be greater than 255.
       maxEgressIPsPerNode: 255
+      # Fully randomize source port mapping in Egress SNAT rules. This has no impact on the default SNAT
+      # rules enforced by each Node for local Pod traffic. By default, we use the same value as for the
+      # top-level snatFullyRandomPorts configuration, but this field can be used as an override.
+      snatFullyRandomPorts:
 
     # ClusterIP CIDR range for Services. It's required when AntreaProxy is not enabled, and should be
     # set to the same value as the one specified by --service-cluster-ip-range for kube-apiserver. When
@@ -5130,7 +5138,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: a72df8ca7bc976062ae8b3091bccd5aa2c5ee69f4ac3c7f3dc7d58e6765c4ebf
+        checksum/config: f5cf00de39a27790a7e158a3eca79123de415b3b09d389ac984b74027bbfaade
       labels:
         app: antrea
         component: antrea-agent
@@ -5366,7 +5374,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: a72df8ca7bc976062ae8b3091bccd5aa2c5ee69f4ac3c7f3dc7d58e6765c4ebf
+        checksum/config: f5cf00de39a27790a7e158a3eca79123de415b3b09d389ac984b74027bbfaade
       labels:
         app: antrea
         component: antrea-controller

build/yamls/antrea-ipsec.yml

@@ -3859,6 +3859,10 @@ data:
     # performs SNAT and this option will be ignored; for other modes it must be set to false.
     noSNAT: false
 
+    # Fully randomize source port mapping in SNAT rules used for egress traffic from Pods to the
+    # external network.
+    snatFullyRandomPorts: false
+
     # Tunnel protocols used for encapsulating traffic across Nodes. If WireGuard is enabled in trafficEncryptionMode,
     # this option will not take effect. Supported values:
     # - geneve (default)
@@ -3929,6 +3933,10 @@ data:
       # The maximum number of Egress IPs that can be assigned to a Node. It's useful when the Node network restricts
       # the number of secondary IPs a Node can have, e.g. EKS. It must not be greater than 255.
       maxEgressIPsPerNode: 255
+      # Fully randomize source port mapping in Egress SNAT rules. This has no impact on the default SNAT
+      # rules enforced by each Node for local Pod traffic. By default, we use the same value as for the
+      # top-level snatFullyRandomPorts configuration, but this field can be used as an override.
+      snatFullyRandomPorts:
 
     # ClusterIP CIDR range for Services. It's required when AntreaProxy is not enabled, and should be
     # set to the same value as the one specified by --service-cluster-ip-range for kube-apiserver. When
@@ -5143,7 +5151,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 524a77a636d0093e1c1a41cc39402cee561cdd4479d9866f573082f905050ce5
+        checksum/config: 9e94f199d125877d889ba73e053c95b342e89323d0423cde074ae074df379494
         checksum/ipsec-secret: d0eb9c52d0cd4311b6d252a951126bf9bea27ec05590bed8a394f0f792dcb2a4
       labels:
         app: antrea
@@ -5425,7 +5433,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 524a77a636d0093e1c1a41cc39402cee561cdd4479d9866f573082f905050ce5
+        checksum/config: 9e94f199d125877d889ba73e053c95b342e89323d0423cde074ae074df379494
       labels:
         app: antrea
         component: antrea-controller

build/yamls/antrea.yml

@@ -3846,6 +3846,10 @@ data:
     # performs SNAT and this option will be ignored; for other modes it must be set to false.
     noSNAT: false
 
+    # Fully randomize source port mapping in SNAT rules used for egress traffic from Pods to the
+    # external network.
+    snatFullyRandomPorts: false
+
     # Tunnel protocols used for encapsulating traffic across Nodes. If WireGuard is enabled in trafficEncryptionMode,
     # this option will not take effect. Supported values:
     # - geneve (default)
@@ -3916,6 +3920,10 @@ data:
       # The maximum number of Egress IPs that can be assigned to a Node. It's useful when the Node network restricts
       # the number of secondary IPs a Node can have, e.g. EKS. It must not be greater than 255.
       maxEgressIPsPerNode: 255
+      # Fully randomize source port mapping in Egress SNAT rules. This has no impact on the default SNAT
+      # rules enforced by each Node for local Pod traffic. By default, we use the same value as for the
+      # top-level snatFullyRandomPorts configuration, but this field can be used as an override.
+      snatFullyRandomPorts:
 
     # ClusterIP CIDR range for Services. It's required when AntreaProxy is not enabled, and should be
     # set to the same value as the one specified by --service-cluster-ip-range for kube-apiserver. When
@@ -5130,7 +5138,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 2276e135e2f7d4d7d1ab070a94a28cf4207f2f9febe427012f41733b7e963c8e
+        checksum/config: 8256bc0d365d60f16d0bdef14cf674be49d525ee1cd921e531f8bf7e521e1421
       labels:
         app: antrea
         component: antrea-agent
@@ -5366,7 +5374,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 2276e135e2f7d4d7d1ab070a94a28cf4207f2f9febe427012f41733b7e963c8e
+        checksum/config: 8256bc0d365d60f16d0bdef14cf674be49d525ee1cd921e531f8bf7e521e1421
       labels:
         app: antrea
         component: antrea-controller

cmd/antrea-agent/agent.go

@@ -239,6 +239,8 @@ func run(o *Options) error {
 		connectUplinkToBridge,
 		nodeNetworkPolicyEnabled,
 		multicastEnabled,
+		o.config.SNATFullyRandomPorts,
+		*o.config.Egress.SNATFullyRandomPorts,
 		serviceCIDRProvider)
 	if err != nil {
 		return fmt.Errorf("error creating route client: %v", err)

cmd/antrea-agent/options.go

@@ -493,6 +493,9 @@ func (o *Options) setK8sNodeDefaultOptions() {
 		if o.config.Egress.MaxEgressIPsPerNode == 0 {
 			o.config.Egress.MaxEgressIPsPerNode = defaultMaxEgressIPsPerNode
 		}
+		if o.config.Egress.SNATFullyRandomPorts == nil {
+			o.config.Egress.SNATFullyRandomPorts = ptr.To(o.config.SNATFullyRandomPorts)
+		}
 	}
 }
 

cmd/antrea-agent/options_windows.go

@@ -60,6 +60,9 @@ func (o *Options) checkUnsupportedFeatures() error {
 	if o.config.EnableBridgingMode {
 		unsupported = append(unsupported, "EnableBridgingMode")
 	}
+	if o.config.SNATFullyRandomPorts {
+		unsupported = append(unsupported, "SNATFullyRandomPorts")
+	}
 	if unsupported != nil {
 		return fmt.Errorf("unsupported features on Windows: {%s}", strings.Join(unsupported, ", "))
 	}