[IPv6] Support no-encap mode in dual-stack setup

Signed-off-by: Zhecheng Li <[email protected]>

cmd/antrea-agent/agent.go

@@ -226,7 +226,7 @@ func run(o *Options) error {
 	var egressController *egress.EgressController
 	if features.DefaultFeatureGate.Enabled(features.Egress) {
 		egressController, err = egress.NewEgressController(
-			ofClient, antreaClientProvider, crdClient, ifaceStore, routeClient, nodeConfig.Name, nodeConfig.NodeIPAddr.IP,
+			ofClient, antreaClientProvider, crdClient, ifaceStore, routeClient, nodeConfig.Name, nodeConfig.NodeIPv4Addr.IP,
 			o.config.ClusterMembershipPort, egressInformer, nodeInformer, externalIPPoolInformer,
 		)
 		if err != nil {

pkg/agent/agent.go

@@ -530,15 +530,17 @@ func (i *Initializer) configureGatewayInterface(gatewayIface *interfacestore.Int
 
 	i.nodeConfig.GatewayConfig = &config.GatewayConfig{Name: i.hostGateway, MAC: gwMAC}
 	gatewayIface.MAC = gwMAC
+	gatewayIface.IPs = []net.IP{}
 	if i.networkConfig.TrafficEncapMode.IsNetworkPolicyOnly() {
-		// Assign IP to gw as required by SpoofGuard.
-		// NodeIPAddr can be either IPv4 or IPv6.
-		if i.nodeConfig.NodeIPAddr.IP.To4() != nil {
-			i.nodeConfig.GatewayConfig.IPv4 = i.nodeConfig.NodeIPAddr.IP
-		} else {
-			i.nodeConfig.GatewayConfig.IPv6 = i.nodeConfig.NodeIPAddr.IP
+		// Assign IPs to gw as required by SpoofGuard.
+		if i.nodeConfig.NodeIPv4Addr != nil {
+			i.nodeConfig.GatewayConfig.IPv4 = i.nodeConfig.NodeIPv4Addr.IP
+			gatewayIface.IPs = append(gatewayIface.IPs, i.nodeConfig.NodeIPv4Addr.IP)
+		}
+		if i.nodeConfig.NodeIPv6Addr != nil {
+			i.nodeConfig.GatewayConfig.IPv6 = i.nodeConfig.NodeIPv6Addr.IP
+			gatewayIface.IPs = append(gatewayIface.IPs, i.nodeConfig.NodeIPv6Addr.IP)
 		}
-		gatewayIface.IPs = []net.IP{i.nodeConfig.NodeIPAddr.IP}
 		// No need to assign local CIDR to gw0 because local CIDR is not managed by Antrea
 		return nil
 	}
@@ -642,13 +644,13 @@ func (i *Initializer) initNodeLocalConfig() error {
 		return err
 	}
 
-	ipAddr, err := k8s.GetNodeAddr(node)
+	ipAddrs, err := k8s.GetNodeAddrs(node)
 	if err != nil {
-		return fmt.Errorf("failed to obtain local IP address from K8s: %w", err)
+		return fmt.Errorf("failed to obtain local IP addresses from K8s: %w", err)
 	}
-	localAddr, localIntf, err := getIPNetDeviceFromIP(ipAddr)
+	localV4Addr, localV6Addr, localIntf, err := getIPNetDeviceFromIP(ipAddrs)
 	if err != nil {
-		return fmt.Errorf("failed to get local IPNet device with IP %v: %v", ipAddr, err)
+		return fmt.Errorf("failed to get local IPNet device with IP %v: %v", ipAddrs, err)
 	}
 
 	// Update the Node's MAC address in the annotations of the Node. The MAC address will be used for direct routing by
@@ -675,7 +677,8 @@ func (i *Initializer) initNodeLocalConfig() error {
 		Name:            nodeName,
 		OVSBridge:       i.ovsBridge,
 		DefaultTunName:  defaultTunInterfaceName,
-		NodeIPAddr:      localAddr,
+		NodeIPv4Addr:    localV4Addr,
+		NodeIPv6Addr:    localV6Addr,
 		UplinkNetConfig: new(config.AdapterNetConfig)}
 
 	mtu, err := i.getNodeMTU(localIntf)
@@ -849,7 +852,7 @@ func (i *Initializer) getNodeMTU(localIntf *net.Interface) (int, error) {
 		} else if i.networkConfig.TunnelType == ovsconfig.GRETunnel {
 			mtu -= config.GREOverhead
 		}
-		if i.nodeConfig.NodeIPAddr.IP.To4() == nil {
+		if i.nodeConfig.NodeIPv6Addr != nil {
 			mtu -= config.IPv6ExtraOverhead
 		}
 	}

pkg/agent/agent_test.go

@@ -235,7 +235,7 @@ func TestInitNodeLocalConfig(t *testing.T) {
 				OVSBridge:       ovsBridge,
 				DefaultTunName:  defaultTunInterfaceName,
 				PodIPv4CIDR:     podCIDR,
-				NodeIPAddr:      nodeIPNet,
+				NodeIPv4Addr:    nodeIPNet,
 				NodeMTU:         tt.expectedMTU,
 				UplinkNetConfig: new(config.AdapterNetConfig),
 			}
@@ -249,8 +249,8 @@ func TestInitNodeLocalConfig(t *testing.T) {
 
 func mockGetIPNetDeviceFromIP(ipNet *net.IPNet, ipDevice *net.Interface) func() {
 	prevGetIPNetDeviceFromIP := getIPNetDeviceFromIP
-	getIPNetDeviceFromIP = func(localIP net.IP) (*net.IPNet, *net.Interface, error) {
-		return ipNet, ipDevice, nil
+	getIPNetDeviceFromIP = func(localIP []net.IP) (*net.IPNet, *net.IPNet, *net.Interface, error) {
+		return ipNet, nil, ipDevice, nil
 	}
 	return func() { getIPNetDeviceFromIP = prevGetIPNetDeviceFromIP }
 }

pkg/agent/config/node_config.go

@@ -83,8 +83,10 @@ type NodeConfig struct {
 	// The CIDR block from where to allocate IPv6 address to Pod.
 	// It's nil for the net workPolicyOnly trafficEncapMode which doesn't do IPAM.
 	PodIPv6CIDR *net.IPNet
-	// The Node's IP used in Kubernetes. It has the network mask information.
-	NodeIPAddr *net.IPNet
+	// The Node's IPv4 IP used in Kubernetes. It has the network mask information.
+	NodeIPv4Addr *net.IPNet
+	// The Node's IPv6 IP used in Kubernetes. It has the network mask information.
+	NodeIPv6Addr *net.IPNet
 	// Set either via defaultMTU config in antrea.yaml or auto discovered.
 	// Auto discovery will use MTU value of the Node's primary interface.
 	// For Encap and Hybrid mode, Node MTU will be adjusted to account for encap header.
@@ -96,8 +98,8 @@ type NodeConfig struct {
 }
 
 func (n *NodeConfig) String() string {
-	return fmt.Sprintf("NodeName: %s, OVSBridge: %s, PodIPv4CIDR: %s, PodIPv6CIDR: %s, NodeIP: %s, Gateway: %s",
-		n.Name, n.OVSBridge, n.PodIPv4CIDR, n.PodIPv6CIDR, n.NodeIPAddr, n.GatewayConfig)
+	return fmt.Sprintf("NodeName: %s, OVSBridge: %s, PodIPv4CIDR: %s, PodIPv6CIDR: %s, NodeIPv4: %s, NodeIPv6: %s, Gateway: %s",
+		n.Name, n.OVSBridge, n.PodIPv4CIDR, n.PodIPv6CIDR, n.NodeIPv4Addr, n.NodeIPv6Addr, n.GatewayConfig)
 }
 
 // User provided network configuration parameters.
@@ -109,15 +111,13 @@ type NetworkConfig struct {
 }
 
 // IsIPv4Enabled returns true if the cluster network supports IPv4.
-// TODO: support dual-stack in networkPolicyOnly mode.
 func IsIPv4Enabled(nodeConfig *NodeConfig, trafficEncapMode TrafficEncapModeType) bool {
 	return nodeConfig.PodIPv4CIDR != nil ||
-		(trafficEncapMode.IsNetworkPolicyOnly() && nodeConfig.NodeIPAddr.IP.To4() != nil)
+		(trafficEncapMode.IsNetworkPolicyOnly() && nodeConfig.NodeIPv4Addr != nil)
 }
 
 // IsIPv6Enabled returns true if the cluster network supports IPv6.
-// TODO: support dual-stack in networkPolicyOnly mode.
 func IsIPv6Enabled(nodeConfig *NodeConfig, trafficEncapMode TrafficEncapModeType) bool {
 	return nodeConfig.PodIPv6CIDR != nil ||
-		(trafficEncapMode.IsNetworkPolicyOnly() && nodeConfig.NodeIPAddr.IP.To4() == nil)
+		(trafficEncapMode.IsNetworkPolicyOnly() && nodeConfig.NodeIPv6Addr != nil)
 }

pkg/agent/controller/egress/ipassigner/ip_assigner_linux.go

@@ -49,7 +49,7 @@ type ipAssigner struct {
 
 // NewIPAssigner returns an *ipAssigner.
 func NewIPAssigner(nodeIPAddr net.IP, dummyDeviceName string) (*ipAssigner, error) {
-	_, egressInterface, err := util.GetIPNetDeviceFromIP(nodeIPAddr)
+	_, _, egressInterface, err := util.GetIPNetDeviceFromIP([]net.IP{nodeIPAddr})
 	if err != nil {
 		return nil, fmt.Errorf("get IPNetDevice from ip %v error: %+v", nodeIPAddr, err)
 	}

pkg/agent/controller/noderoute/node_route_controller.go

@@ -135,7 +135,7 @@ func nodeRouteInfoPodCIDRIndexFunc(obj interface{}) ([]string, error) {
 type nodeRouteInfo struct {
 	nodeName  string
 	podCIDRs  []*net.IPNet
-	nodeIP    net.IP
+	nodeIPs   []net.IP
 	gatewayIP []net.IP
 	nodeMAC   net.HardwareAddr
 }
@@ -215,17 +215,26 @@ func (c *Controller) removeStaleTunnelPorts() error {
 				continue
 			}
 
-			peerNodeIP, err := k8s.GetNodeAddr(node)
+			peerNodeIPs, err := k8s.GetNodeAddrs(node)
 			if err != nil {
 				klog.Errorf("Failed to retrieve IP address of Node %s: %v", node.Name, err)
 				continue
 			}
+			var peerNodeIPv4, peerNodeIPv6 net.IP
+			for _, ip := range peerNodeIPs {
+				if ip.To4() == nil {
+					peerNodeIPv6 = ip
+				} else {
+					peerNodeIPv4 = ip
+				}
+			}
 
 			ifaceID := util.GenerateNodeTunnelInterfaceKey(node.Name)
 			validConfiguration := interfaceConfig.PSK == c.networkConfig.IPSecPSK &&
-				interfaceConfig.RemoteIP.Equal(peerNodeIP) &&
 				interfaceConfig.TunnelInterfaceConfig.Type == c.networkConfig.TunnelType
-			if validConfiguration {
+			v4RemoteIP := peerNodeIPv4 != nil && interfaceConfig.RemoteIP.Equal(peerNodeIPv4)
+			v6RemoteIP := peerNodeIPv6 != nil && interfaceConfig.RemoteIP.Equal(peerNodeIPv6)
+			if validConfiguration && (v4RemoteIP || v6RemoteIP) {
 				desiredInterfaces[ifaceID] = true
 			}
 		}
@@ -430,7 +439,7 @@ func (c *Controller) addNodeRoute(nodeName string, node *corev1.Node) error {
 	klog.Infof("Adding routes and flows to Node %s, podCIDRs: %v, addresses: %v",
 		nodeName, podCIDRStrs, node.Status.Addresses)
 
-	var podCIDRs []*net.IPNet
+	var peerPodCIDRs []*net.IPNet
 	peerConfig := make(map[*net.IPNet]net.IP, len(podCIDRStrs))
 	for _, podCIDR := range podCIDRStrs {
 		klog.Infof("Adding routes and flows to Node %s, podCIDR: %s, addresses: %v",
@@ -467,49 +476,66 @@ func (c *Controller) addNodeRoute(nodeName string, node *corev1.Node) error {
 		}
 		peerGatewayIP := ip.NextIP(peerPodCIDRAddr)
 		peerConfig[peerPodCIDR] = peerGatewayIP
-		podCIDRs = append(podCIDRs, peerPodCIDR)
+		peerPodCIDRs = append(peerPodCIDRs, peerPodCIDR)
 	}
 
-	peerNodeIP, err := k8s.GetNodeAddr(node)
+	peerNodeIPs, err := k8s.GetNodeAddrs(node)
 	if err != nil {
-		klog.Errorf("Failed to retrieve IP address of Node %s: %v", nodeName, err)
+		klog.Errorf("Failed to retrieve IP addresses of Node %s: %v", nodeName, err)
 		return nil
 	}
 
-	ipsecTunOFPort := int32(0)
-	if c.networkConfig.EnableIPSecTunnel {
-		// Create a separate tunnel port for the Node, as OVS IPSec monitor needs to
-		// read PSK and remote IP from the Node's tunnel interface to create IPSec
-		// security policies.
-		if ipsecTunOFPort, err = c.createIPSecTunnelPort(nodeName, peerNodeIP); err != nil {
-			return err
+	for _, peerNodeIP := range peerNodeIPs {
+		ipsecTunOFPort := int32(0)
+		if c.networkConfig.EnableIPSecTunnel {
+			// Create a separate tunnel port for the Node, as OVS IPSec monitor needs to
+			// read PSK and remote IP from the Node's tunnel interface to create IPSec
+			// security policies.
+			if ipsecTunOFPort, err = c.createIPSecTunnelPort(nodeName, peerNodeIP); err != nil {
+				return err
+			}
 		}
-	}
 
-	err = c.ofClient.InstallNodeFlows(
-		nodeName,
-		peerConfig,
-		peerNodeIP,
-		uint32(ipsecTunOFPort),
-		peerNodeMAC)
-	if err != nil {
-		return fmt.Errorf("failed to install flows to Node %s: %v", nodeName, err)
+		err = c.ofClient.InstallNodeFlows(
+			nodeName,
+			peerConfig,
+			peerNodeIP,
+			uint32(ipsecTunOFPort),
+			peerNodeMAC)
+		if err != nil {
+			return fmt.Errorf("failed to install flows to Node %s: %v", nodeName, err)
+		}
 	}
 
 	var peerGatewayIPs []net.IP
+	var peerNodeIPv4, peerNodeIPv6 net.IP
+	for _, ip := range peerNodeIPs {
+		if ip.To4() == nil {
+			peerNodeIPv6 = ip
+		} else {
+			peerNodeIPv4 = ip
+		}
+	}
 	for peerPodCIDR, peerGatewayIP := range peerConfig {
+		var peerNodeIP net.IP
+		if peerGatewayIP.To4() == nil {
+			peerNodeIP = peerNodeIPv6
+		} else {
+			peerNodeIP = peerNodeIPv4
+		}
 		if err := c.routeClient.AddRoutes(peerPodCIDR, nodeName, peerNodeIP, peerGatewayIP); err != nil {
 			return err
 		}
 		peerGatewayIPs = append(peerGatewayIPs, peerGatewayIP)
 	}
 	c.installedNodes.Add(&nodeRouteInfo{
 		nodeName:  nodeName,
-		podCIDRs:  podCIDRs,
-		nodeIP:    peerNodeIP,
+		podCIDRs:  peerPodCIDRs,
+		nodeIPs:   peerNodeIPs,
 		gatewayIP: peerGatewayIPs,
 		nodeMAC:   peerNodeMAC,
 	})
+
 	return err
 }
 

pkg/agent/controller/traceflow/packetin.go

@@ -231,7 +231,7 @@ func (c *Controller) parsePacketIn(pktIn *ofctrl.PacketIn) (*crdv1alpha1.Tracefl
 	if tableID == uint8(openflow.L2ForwardingOutTable) {
 		ob := new(crdv1alpha1.Observation)
 		tunnelDstIP := ""
-		isIPv6 := c.nodeConfig.NodeIPAddr.IP.To4() == nil
+		isIPv6 := c.nodeConfig.NodeIPv4Addr == nil
 		if match := getMatchTunnelDstField(matchers, isIPv6); match != nil {
 			tunnelDstIP, err = getTunnelDstValue(match)
 			if err != nil {

pkg/agent/memberlist/cluster.go

@@ -252,7 +252,7 @@ func (c *Cluster) enqueueExternalIPPool(obj interface{}) {
 // newClusterMember gets the Node's IP and returns a cluster member "<IP>:<clusterMemberlistPort>"
 // representing that Node in the memberlist cluster.
 func (c *Cluster) newClusterMember(node *corev1.Node) (string, error) {
-	nodeAddr, err := k8s.GetNodeAddr(node)
+	nodeAddr, err := k8s.GetNodeAddrs(node)
 	if err != nil {
 		return "", fmt.Errorf("obtain IP address from K8s Node failed: %v", err)
 	}

pkg/agent/memberlist/cluster_test.go

@@ -58,7 +58,7 @@ func newFakeCluster(nodeConfig *config.NodeConfig, stopCh <-chan struct{}, i int
 	crdInformerFactory := crdinformers.NewSharedInformerFactory(crdClient, 0)
 	ipPoolInformer := crdInformerFactory.Crd().V1alpha2().ExternalIPPools()
 
-	cluster, err := NewCluster(port, nodeConfig.NodeIPAddr.IP, nodeConfig.Name, nodeInformer, ipPoolInformer)
+	cluster, err := NewCluster(port, nodeConfig.NodeIPv4Addr.IP, nodeConfig.Name, nodeInformer, ipPoolInformer)
 	if err != nil {
 		return nil, err
 	}
@@ -137,8 +137,8 @@ func TestCluster_Run(t *testing.T) {
 	for i, tCase := range testCases {
 		t.Run(tCase.name, func(t *testing.T) {
 			nodeConfig := &config.NodeConfig{
-				Name:       localNodeName,
-				NodeIPAddr: &net.IPNet{IP: net.IPv4(127, 0, 0, 1), Mask: net.IPv4Mask(255, 255, 255, 255)},
+				Name:         localNodeName,
+				NodeIPv4Addr: &net.IPNet{IP: net.IPv4(127, 0, 0, 1), Mask: net.IPv4Mask(255, 255, 255, 255)},
 			}
 			stopCh := make(chan struct{})
 			defer close(stopCh)
@@ -177,8 +177,8 @@ func TestCluster_RunClusterEvents(t *testing.T) {
 
 	nodeName := "localNodeName"
 	nodeConfig := &config.NodeConfig{
-		Name:       nodeName,
-		NodeIPAddr: &net.IPNet{IP: net.IPv4(127, 0, 0, 1)},
+		Name:         nodeName,
+		NodeIPv4Addr: &net.IPNet{IP: net.IPv4(127, 0, 0, 1)},
 	}
 	localNode := &v1.Node{
 		ObjectMeta: metav1.ObjectMeta{Name: nodeName},

pkg/agent/openflow/client.go

@@ -390,7 +390,7 @@ func (c *client) InstallNodeFlows(hostname string,
 			// only work for IPv4 addresses.
 			flows = append(flows, c.arpResponderFlow(peerGatewayIP, cookie.Node))
 		}
-		if c.encapMode.NeedsEncapToPeer(tunnelPeerIP, c.nodeConfig.NodeIPAddr) {
+		if c.encapMode.NeedsEncapToPeer(tunnelPeerIP, c.nodeConfig.NodeIPv4Addr) || c.encapMode.NeedsEncapToPeer(tunnelPeerIP, c.nodeConfig.NodeIPv6Addr) {
 			// tunnelPeerIP is the Node Internal Address. In a dual-stack setup, whether this address is an IPv4 address or an
 			// IPv6 one is decided by the address family of Node Internal Address.
 			flows = append(flows, c.l3FwdFlowToRemote(localGatewayMAC, *peerPodCIDR, tunnelPeerIP, cookie.Node))
@@ -724,21 +724,34 @@ func (c *client) Initialize(roundInfo types.RoundInfo, nodeConfig *config.NodeCo
 }
 
 func (c *client) InstallExternalFlows() error {
-	nodeIP := c.nodeConfig.NodeIPAddr.IP
-	localGatewayMAC := c.nodeConfig.GatewayConfig.MAC
+	install := func(nodeIP net.IP) error {
+		localGatewayMAC := c.nodeConfig.GatewayConfig.MAC
 
-	var flows []binding.Flow
-	if c.nodeConfig.PodIPv4CIDR != nil {
-		flows = c.externalFlows(nodeIP, *c.nodeConfig.PodIPv4CIDR, localGatewayMAC)
-	}
-	if c.nodeConfig.PodIPv6CIDR != nil {
-		flows = append(flows, c.externalFlows(nodeIP, *c.nodeConfig.PodIPv6CIDR, localGatewayMAC)...)
+		var flows []binding.Flow
+		if c.nodeConfig.PodIPv4CIDR != nil {
+			flows = c.externalFlows(nodeIP, *c.nodeConfig.PodIPv4CIDR, localGatewayMAC)
+		}
+		if c.nodeConfig.PodIPv6CIDR != nil {
+			flows = append(flows, c.externalFlows(nodeIP, *c.nodeConfig.PodIPv6CIDR, localGatewayMAC)...)
+		}
+
+		if err := c.ofEntryOperations.AddAll(flows); err != nil {
+			return fmt.Errorf("failed to install flows for external communication: %v", err)
+		}
+		c.hostNetworkingFlows = append(c.hostNetworkingFlows, flows...)
+		return nil
 	}
 
-	if err := c.ofEntryOperations.AddAll(flows); err != nil {
-		return fmt.Errorf("failed to install flows for external communication: %v", err)
+	if c.nodeConfig.NodeIPv4Addr != nil {
+		if err := install(c.nodeConfig.NodeIPv4Addr.IP); err != nil {
+			return err
+		}
+	}
+	if c.nodeConfig.NodeIPv6Addr != nil {
+		if err := install(c.nodeConfig.NodeIPv6Addr.IP); err != nil {
+			return err
+		}
 	}
-	c.hostNetworkingFlows = append(c.hostNetworkingFlows, flows...)
 	return nil
 }
 

pkg/agent/openflow/pipeline_windows.go

@@ -297,7 +297,8 @@ func (c *client) hostBridgeUplinkFlows(localSubnet net.IPNet, category cookie.Ca
 
 func (c *client) l3FwdFlowToRemoteViaRouting(localGatewayMAC net.HardwareAddr, remoteGatewayMAC net.HardwareAddr,
 	category cookie.Category, peerIP net.IP, peerPodCIDR *net.IPNet) []binding.Flow {
-	if c.encapMode.NeedsDirectRoutingToPeer(peerIP, c.nodeConfig.NodeIPAddr) && remoteGatewayMAC != nil {
+	// Currently, IPv6 is not supported on Windows.
+	if c.encapMode.NeedsDirectRoutingToPeer(peerIP, c.nodeConfig.NodeIPv4Addr) && remoteGatewayMAC != nil {
 		// It enhances Windows Noencap mode performance by bypassing host network.
 		flows := []binding.Flow{c.pipeline[l2ForwardingCalcTable].BuildFlow(priorityNormal).
 			MatchDstMAC(remoteGatewayMAC).