[Windows] NoEncap support

Co-authored-by: Zhecheng Li <[email protected]>

build/yamls/antrea-windows.yml

@@ -71,6 +71,16 @@ data:
     # the flow collector.
     # Flow export frequency should be greater than or equal to 1.
     #flowExportFrequency: 12
+
+    # Determines how traffic is encapsulated. It has the following options:
+    # encap(default):    Inter-node Pod traffic is always encapsulated and Pod to external network
+    #                    traffic is SNAT'd.
+    # noEncap:           Inter-node Pod traffic is not encapsulated; Pod to external network traffic is
+    #                    SNAT'd if noSNAT is not set to true. Underlying network must be capable of
+    #                    supporting Pod traffic across IP subnets.
+    # hybrid:            noEncap if source and destination Nodes are on the same subnet, otherwise encap.
+    #
+    #trafficEncapMode: encap
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -89,7 +99,7 @@ kind: ConfigMap
 metadata:
   labels:
     app: antrea
-  name: antrea-windows-config-kc6bfhk4mg
+  name: antrea-windows-config-4kcddbh9tt
   namespace: kube-system
 ---
 apiVersion: apps/v1
@@ -177,7 +187,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-windows-config-kc6bfhk4mg
+          name: antrea-windows-config-4kcddbh9tt
         name: antrea-windows-config
       - configMap:
           defaultMode: 420

build/yamls/windows/base/conf/antrea-agent.conf

@@ -53,3 +53,13 @@ featureGates:
 # the flow collector.
 # Flow export frequency should be greater than or equal to 1.
 #flowExportFrequency: 12
+
+# Determines how traffic is encapsulated. It has the following options:
+# encap(default):    Inter-node Pod traffic is always encapsulated and Pod to external network
+#                    traffic is SNAT'd.
+# noEncap:           Inter-node Pod traffic is not encapsulated; Pod to external network traffic is
+#                    SNAT'd if noSNAT is not set to true. Underlying network must be capable of
+#                    supporting Pod traffic across IP subnets.
+# hybrid:            noEncap if source and destination Nodes are on the same subnet, otherwise encap.
+#
+#trafficEncapMode: encap

cmd/antrea-agent/options_windows.go

@@ -42,7 +42,7 @@ func (o *Options) checkUnsupportedFeatures() error {
 		unsupported = append(unsupported, "OVSDatapathType: "+o.config.OVSDatapathType)
 	}
 	_, encapMode := config.GetTrafficEncapModeFromStr(o.config.TrafficEncapMode)
-	if encapMode != config.TrafficEncapModeEncap {
+	if encapMode == config.TrafficEncapModeNetworkPolicyOnly {
 		unsupported = append(unsupported, "TrafficEncapMode: "+encapMode.String())
 	}
 	if o.config.TunnelType == ovsconfig.GRETunnel {

cmd/antrea-agent/options_windows_test.go

@@ -57,7 +57,7 @@ func TestCheckUnsupportedFeatures(t *testing.T) {
 		{
 			"noEncap mode",
 			AgentConfig{TrafficEncapMode: config.TrafficEncapModeNoEncap.String()},
-			false,
+			true,
 		},
 		{
 			"GRE tunnel",

pkg/agent/controller/noderoute/node_route_controller.go

@@ -35,6 +35,7 @@ import (
 	"github.com/vmware-tanzu/antrea/pkg/agent/interfacestore"
 	"github.com/vmware-tanzu/antrea/pkg/agent/openflow"
 	"github.com/vmware-tanzu/antrea/pkg/agent/route"
+	"github.com/vmware-tanzu/antrea/pkg/agent/types"
 	"github.com/vmware-tanzu/antrea/pkg/agent/util"
 	"github.com/vmware-tanzu/antrea/pkg/ovs/ovsconfig"
 	utilip "github.com/vmware-tanzu/antrea/pkg/util/ip"
@@ -135,6 +136,7 @@ type nodeRouteInfo struct {
 	podCIDRs  []*net.IPNet
 	nodeIP    net.IP
 	gatewayIP []net.IP
+	peerNodeMAC net.HardwareAddr
 }
 
 // enqueueNode adds an object to the controller work queue
@@ -406,8 +408,14 @@ func (c *Controller) deleteNodeRoute(nodeName string) error {
 }
 
 func (c *Controller) addNodeRoute(nodeName string, node *corev1.Node) error {
-	if _, installed, _ := c.installedNodes.GetByKey(nodeName); installed {
-		// Route is already added for this Node.
+	peerNodeMAC, err := GetNodeMAC(node)
+	if err != nil {
+		klog.Errorf("Error when retrieving MAC of Node %s: %v", nodeName, err)
+	}
+
+	nrInfo, installed, _ := c.installedNodes.GetByKey(nodeName)
+	if installed && nrInfo.(nodeRouteInfo).peerNodeMAC.String() == peerNodeMAC.String() {
+		// Route is already added for this Node and peerNodeMAC isn't changed.
 		return nil
 	}
 
@@ -476,7 +484,8 @@ func (c *Controller) addNodeRoute(nodeName string, node *corev1.Node) error {
 		nodeName,
 		peerConfig,
 		peerNodeIP,
-		uint32(ipsecTunOFPort))
+		uint32(ipsecTunOFPort),
+		peerNodeMAC)
 	if err != nil {
 		return fmt.Errorf("failed to install flows to Node %s: %v", nodeName, err)
 	}
@@ -493,6 +502,7 @@ func (c *Controller) addNodeRoute(nodeName string, node *corev1.Node) error {
 		podCIDRs:  podCIDRs,
 		nodeIP:    peerNodeIP,
 		gatewayIP: peerGatewayIPs,
+		peerNodeMAC: peerNodeMAC,
 	})
 	return err
 }
@@ -653,3 +663,17 @@ func (c *Controller) IPInPodSubnets(ip net.IP) bool {
 	nodeInCluster, _ := c.installedNodes.ByIndex(nodeRouteInfoPodCIDRIndexName, ipCIDRStr)
 	return len(nodeInCluster) > 0 || ipCIDRStr == curNodeCIDRStr
 }
+
+// GetNodeMAC gets Node's br-int MAC from its annotation. It is for Windows Noencap mode only.
+func GetNodeMAC(node *corev1.Node) (net.HardwareAddr, error) {
+	macStr := node.Annotations[types.NodeMACAddressAnnotationKey]
+	if macStr == "" {
+		return nil, nil
+	} else {
+		mac, err := net.ParseMAC(macStr)
+		if err != nil {
+			return nil, err
+		}
+		return mac, nil
+	}
+}

pkg/agent/controller/noderoute/node_route_controller_test.go

@@ -133,7 +133,7 @@ func TestControllerWithDuplicatePodCIDR(t *testing.T) {
 		c.clientset.CoreV1().Nodes().Create(context.TODO(), node1, metav1.CreateOptions{})
 		// The 2nd argument is Any() because the argument is unpredictable when it uses pointer as the key of map.
 		// The argument type is map[*net.IPNet]net.IP.
-		c.ofClient.EXPECT().InstallNodeFlows("node1", gomock.Any(), nodeIP1, uint32(0)).Times(1)
+		c.ofClient.EXPECT().InstallNodeFlows("node1", gomock.Any(), nodeIP1, uint32(0), nil).Times(1)
 		c.routeClient.EXPECT().AddRoutes(podCIDR, "node1", nodeIP1, podCIDRGateway).Times(1)
 		c.processNextWorkItem()
 
@@ -150,7 +150,7 @@ func TestControllerWithDuplicatePodCIDR(t *testing.T) {
 		// After node1 is deleted, routes and flows should be installed for node2 successfully.
 		// The 2nd argument is Any() because the argument is unpredictable when it uses pointer as the key of map.
 		// The argument type is map[*net.IPNet]net.IP.
-		c.ofClient.EXPECT().InstallNodeFlows("node2", gomock.Any(), nodeIP2, uint32(0)).Times(1)
+		c.ofClient.EXPECT().InstallNodeFlows("node2", gomock.Any(), nodeIP2, uint32(0), nil).Times(1)
 		c.routeClient.EXPECT().AddRoutes(podCIDR, "node2", nodeIP2, podCIDRGateway).Times(1)
 		c.processNextWorkItem()
 	}()
@@ -214,12 +214,12 @@ func TestIPInPodSubnets(t *testing.T) {
 	c.clientset.CoreV1().Nodes().Create(context.TODO(), node1, metav1.CreateOptions{})
 	// The 2nd argument is Any() because the argument is unpredictable when it uses pointer as the key of map.
 	// The argument type is map[*net.IPNet]net.IP.
-	c.ofClient.EXPECT().InstallNodeFlows("node1", gomock.Any(), nodeIP1, uint32(0)).Times(1)
+	c.ofClient.EXPECT().InstallNodeFlows("node1", gomock.Any(), nodeIP1, uint32(0), nil).Times(1)
 	c.routeClient.EXPECT().AddRoutes(podCIDR, "node1", nodeIP1, podCIDRGateway).Times(1)
 	c.processNextWorkItem()
 
 	c.clientset.CoreV1().Nodes().Create(context.TODO(), node2, metav1.CreateOptions{})
-	c.ofClient.EXPECT().InstallNodeFlows("node2", gomock.Any(), nodeIP2, uint32(0)).Times(1)
+	c.ofClient.EXPECT().InstallNodeFlows("node2", gomock.Any(), nodeIP2, uint32(0), nil).Times(1)
 	c.routeClient.EXPECT().AddRoutes(podCIDR2, "node2", nodeIP2, podCIDR2Gateway).Times(1)
 	c.processNextWorkItem()
 

pkg/agent/openflow/client.go

@@ -27,6 +27,7 @@ import (
 	"github.com/vmware-tanzu/antrea/pkg/agent/types"
 	"github.com/vmware-tanzu/antrea/pkg/agent/util"
 	binding "github.com/vmware-tanzu/antrea/pkg/ovs/openflow"
+	"github.com/vmware-tanzu/antrea/pkg/util/runtime"
 	"github.com/vmware-tanzu/antrea/third_party/proxy"
 )
 
@@ -68,7 +69,8 @@ type Client interface {
 		hostname string,
 		peerConfigs map[*net.IPNet]net.IP,
 		tunnelPeerIP net.IP,
-		ipsecTunOFPort uint32) error
+		ipsecTunOFPort uint32,
+	    	peerNodeMAC net.HardwareAddr) error
 
 	// UninstallNodeFlows removes the connection to the remote Node specified with the
 	// hostname. UninstallNodeFlows will do nothing if no connection to the host was established.
@@ -335,7 +337,8 @@ func (c *client) deleteFlows(cache *flowCategoryCache, flowCacheKey string) erro
 func (c *client) InstallNodeFlows(hostname string,
 	peerConfigs map[*net.IPNet]net.IP,
 	tunnelPeerIP net.IP,
-	ipsecTunOFPort uint32) error {
+	ipsecTunOFPort uint32,
+	remoteGatewayMAC net.HardwareAddr) error {
 	c.replayMutex.RLock()
 	defer c.replayMutex.RUnlock()
 
@@ -352,6 +355,8 @@ func (c *client) InstallNodeFlows(hostname string,
 			// tunnelPeerIP is the Node Internal Address. In a dual-stack setup, whether this address is an IPv4 address or an
 			// IPv6 one is decided by the address family of Node Internal Address.
 			flows = append(flows, c.l3FwdFlowToRemote(localGatewayMAC, *peerPodCIDR, tunnelPeerIP, cookie.Node))
+		} else if runtime.IsWindowsPlatform() && !c.encapMode.NeedsRoutingToPeer(tunnelPeerIP, c.nodeConfig.NodeIPAddr) && remoteGatewayMAC != nil {
+			flows = append(flows, c.l3FwdFlowToRemoteViaRouting(remoteGatewayMAC, *peerPodCIDR, cookie.Node)...)
 		} else {
 			flows = append(flows, c.l3FwdFlowToRemoteViaGW(localGatewayMAC, *peerPodCIDR, cookie.Node))
 		}
@@ -580,6 +585,14 @@ func (c *client) InstallGatewayFlows() error {
 	if c.encapMode.SupportsEncap() {
 		flows = append(flows, c.l3FwdFlowToGateway(gatewayIPs, gatewayConfig.MAC, cookie.Default)...)
 	}
+	if runtime.IsWindowsPlatform() && c.encapMode.SupportsNoEncap() {
+		if gatewayConfig.IPv4 != nil {
+			flows = append(flows, c.l3FwdFlowRouteToWindowsGW(gatewayConfig.MAC, gatewayConfig.IPv4, cookie.Default)...)
+		}
+		if gatewayConfig.IPv6 != nil {
+			flows = append(flows, c.l3FwdFlowRouteToWindowsGW(gatewayConfig.MAC, gatewayConfig.IPv6, cookie.Default)...)
+		}
+	}
 
 	if err := c.ofEntryOperations.AddAll(flows); err != nil {
 		return err

pkg/agent/openflow/client_test.go

@@ -57,7 +57,7 @@ func installNodeFlows(ofClient Client, cacheKey string) (int, error) {
 	peerConfig := map[*net.IPNet]net.IP{
 		ipNet: gwIP,
 	}
-	err := ofClient.InstallNodeFlows(hostName, peerConfig, peerNodeIP, 0)
+	err := ofClient.InstallNodeFlows(hostName, peerConfig, peerNodeIP, 0, nil)
 	client := ofClient.(*client)
 	fCacheI, ok := client.nodeFlowCache.Load(hostName)
 	if ok {

pkg/agent/openflow/pipeline.go

@@ -234,6 +234,7 @@ const (
 	macRewriteMark = 0b1
 	// cnpDenyMark indicates the packet is denied(Drop/Reject).
 	cnpDenyMark = 0b1
+	bypassHostMark = 0b11
 
 	// gatewayCTMark is used to to mark connections initiated through the host gateway interface
 	// (i.e. for which the first packet of the connection was received through the gateway).
@@ -2155,3 +2156,41 @@ func NewClient(bridgeName, mgmtAddr string, ovsDatapathType ovsconfig.OVSDatapat
 	c.generatePipeline()
 	return c
 }
+
+// l3FwdFlowRouteToWindowsGW adds a flow table to forward traffic to antrea-gw0 on Windows in NoEncap mode. It is for Windows.
+func (c *client) l3FwdFlowRouteToWindowsGW(localGatewayMAC net.HardwareAddr, localGatewayIP net.IP, category cookie.Category) []binding.Flow {
+	flows := []binding.Flow{
+		c.pipeline[l3ForwardingTable].BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIP).
+			MatchRegRange(int(marksReg), macRewriteMark, macRewriteMarkRange).
+			MatchDstIP(localGatewayIP).
+			Action().SetDstMAC(localGatewayMAC).
+			Action().GotoTable(l3DecTTLTable).
+			Cookie(c.cookieAllocator.Request(category).Raw()).
+			Done(),
+	}
+	return flows
+}
+
+// l3FwdFlowToRemoteViaRouting enhances Windows Noencap mode performance by bypassing host network.
+func (c *client) l3FwdFlowToRemoteViaRouting(
+    gatewayMAC net.HardwareAddr,
+    peerSubnet net.IPNet,
+    category cookie.Category) []binding.Flow {
+	var flows []binding.Flow
+	ipProto := getIPProtocol(peerSubnet.IP)
+	l3FwdTable := c.pipeline[l3ForwardingTable]
+	flows = append(flows, l3FwdTable.BuildFlow(priorityNormal).MatchProtocol(ipProto).
+		MatchDstIPNet(peerSubnet).
+		Action().SetDstMAC(gatewayMAC).
+		Action().GotoTable(l3FwdTable.GetNext()).
+		Cookie(c.cookieAllocator.Request(category).Raw()).
+		Done())
+	flows = append(flows, c.pipeline[l2ForwardingCalcTable].BuildFlow(priorityNormal).
+		MatchDstMAC(gatewayMAC).
+		Action().LoadRegRange(int(PortCacheReg), bypassHostMark, ofPortRegRange).
+		Action().LoadRegRange(int(marksReg), macRewriteMark, ofPortMarkRange).
+		Action().GotoTable(conntrackCommitTable).
+		Cookie(c.cookieAllocator.Request(category).Raw()).
+		Done())
+	return flows
+}

pkg/agent/openflow/pipeline_windows.go

@@ -281,5 +281,17 @@ func (c *client) hostBridgeUplinkFlows(localSubnet net.IPNet, category cookie.Ca
 			Cookie(c.cookieAllocator.Request(category).Raw()).
 			Done(),
 	}
+	if c.encapMode.SupportsNoEncap() {
+		// If NoEncap is enabled, the reply packets from remote Pod can be forwarded to local Pod directly.
+		// by explicitly resubmitting them to endpointDNATTable and marking "macRewriteMark" at same time.
+		flows = append(flows, c.pipeline[conntrackStateTable].BuildFlow(priorityHigh).MatchProtocol(binding.ProtocolIP).
+			MatchRegRange(int(marksReg), markTrafficFromUplink, binding.Range{0, 15}).
+			MatchDstIPNet(localSubnet).
+			Action().LoadRegRange(int(marksReg), macRewriteMark, macRewriteMarkRange).
+			Action().GotoTable(endpointDNATTable).
+			Cookie(c.cookieAllocator.Request(category).Raw()).
+			Done())
+
+	}
 	return flows
 }