[IPv6] Support no-encap mode in dual-stack setup

* Extend single IP address to IPv4 and IPv6 ones for NodeAddr and NodeTransportAddr in nodeConfig * Refactor occurrences of Node IP address to adjust to 2 IP: agent initialization, node route * Still use one IP for IPv6/dual-stack not supported cases: Windows, NPL * Update tests Signed-off-by: Zhecheng Li <[email protected]> Related: antrea-io#2426
lzhecheng · Aug 12, 2021 · 9275be5 · 9275be5
1 parent d785904
commit 9275be5
Show file tree

Hide file tree

Showing 30 changed files with 493 additions and 293 deletions.
diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go
@@ -229,7 +229,7 @@ func run(o *Options) error {
 	var egressController *egress.EgressController
 	if features.DefaultFeatureGate.Enabled(features.Egress) {
 		egressController, err = egress.NewEgressController(
-			ofClient, antreaClientProvider, crdClient, ifaceStore, routeClient, nodeConfig.Name, nodeConfig.NodeIPAddr.IP,
+			ofClient, antreaClientProvider, crdClient, ifaceStore, routeClient, nodeConfig.Name, nodeConfig.NodeIPv4Addr.IP,
 			o.config.ClusterMembershipPort, egressInformer, nodeInformer, externalIPPoolInformer,
 		)
 		if err != nil {

diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go
@@ -21,6 +21,7 @@ import (
 	"net"
 	"os"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 
@@ -533,15 +534,17 @@ func (i *Initializer) configureGatewayInterface(gatewayIface *interfacestore.Int
 
 	i.nodeConfig.GatewayConfig = &config.GatewayConfig{Name: i.hostGateway, MAC: gwMAC}
 	gatewayIface.MAC = gwMAC
+	gatewayIface.IPs = []net.IP{}
 	if i.networkConfig.TrafficEncapMode.IsNetworkPolicyOnly() {
 		// Assign IP to gw as required by SpoofGuard.
-		// NodeIPAddr can be either IPv4 or IPv6.
-		if i.nodeConfig.NodeIPAddr.IP.To4() != nil {
-			i.nodeConfig.GatewayConfig.IPv4 = i.nodeConfig.NodeTransportIPAddr.IP
-		} else {
-			i.nodeConfig.GatewayConfig.IPv6 = i.nodeConfig.NodeTransportIPAddr.IP
+		if i.nodeConfig.NodeIPv4Addr != nil {
+			i.nodeConfig.GatewayConfig.IPv4 = i.nodeConfig.NodeTransportIPv4Addr.IP
+			gatewayIface.IPs = append(gatewayIface.IPs, i.nodeConfig.NodeTransportIPv4Addr.IP)
+		}
+		if i.nodeConfig.NodeIPv6Addr != nil {
+			i.nodeConfig.GatewayConfig.IPv6 = i.nodeConfig.NodeTransportIPv6Addr.IP
+			gatewayIface.IPs = append(gatewayIface.IPs, i.nodeConfig.NodeTransportIPv6Addr.IP)
 		}
-		gatewayIface.IPs = []net.IP{i.nodeConfig.NodeTransportIPAddr.IP}
 		// No need to assign local CIDR to gw0 because local CIDR is not managed by Antrea
 		return nil
 	}
@@ -644,26 +647,34 @@ func (i *Initializer) initNodeLocalConfig() error {
 		return err
 	}
 
-	var nodeIPAddr, transportIPAddr *net.IPNet
+	var nodeIPv4Addr, nodeIPv6Addr, transportIPv4Addr, transportIPv6Addr *net.IPNet
 	var localIntf *net.Interface
 	// Find the interface configured with Node IP and use it for Pod traffic.
-	ipAddr, err := k8s.GetNodeAddr(node)
+	ipAddrs, err := k8s.GetNodeAddrs(node)
 	if err != nil {
-		return fmt.Errorf("failed to obtain local IP address from K8s: %w", err)
+		return fmt.Errorf("failed to obtain local IP addresses from K8s: %w", err)
 	}
-	nodeIPAddr, localIntf, err = getIPNetDeviceFromIP(ipAddr)
+	nodeIPv4Addr, nodeIPv6Addr, localIntf, err = getIPNetDeviceFromIP(ipAddrs)
 	if err != nil {
-		return fmt.Errorf("failed to get local IPNet device with IP %v: %v", ipAddr, err)
+		return fmt.Errorf("failed to get local IPNet device with IP %v: %v", ipAddrs, err)
 	}
-	transportIPAddr = nodeIPAddr
+	transportIPv4Addr = nodeIPv4Addr
+	transportIPv6Addr = nodeIPv6Addr
 	if i.networkConfig.TransportIface != "" {
 		// Find the configured transport interface, and update its IP address in Node's annotation.
-		transportIPAddr, localIntf, err = getTransportIPNetDeviceByName(i.networkConfig.TransportIface, i.ovsBridge)
+		transportIPv4Addr, transportIPv6Addr, localIntf, err = getTransportIPNetDeviceByName(i.networkConfig.TransportIface, i.ovsBridge)
 		if err != nil {
 			return fmt.Errorf("failed to get local IPNet device with transport interface %s: %v", i.networkConfig.TransportIface, err)
 		}
-		klog.InfoS("Updating Node transport address annotation")
-		if err := i.patchNodeAnnotations(nodeName, types.NodeTransportAddressAnnotationKey, transportIPAddr.IP.String()); err != nil {
+		klog.InfoS("Updating Node transport addresses annotation")
+		var ips []string
+		if transportIPv4Addr != nil {
+			ips = append(ips, transportIPv4Addr.IP.String())
+		}
+		if transportIPv6Addr != nil {
+			ips = append(ips, transportIPv6Addr.IP.String())
+		}
+		if err := i.patchNodeAnnotations(nodeName, types.NodeTransportAddressAnnotationKey, strings.Join(ips, ",")); err != nil {
 			return err
 		}
 	} else {
@@ -685,12 +696,14 @@ func (i *Initializer) initNodeLocalConfig() error {
 	}
 
 	i.nodeConfig = &config.NodeConfig{
-		Name:                nodeName,
-		OVSBridge:           i.ovsBridge,
-		DefaultTunName:      defaultTunInterfaceName,
-		NodeIPAddr:          nodeIPAddr,
-		NodeTransportIPAddr: transportIPAddr,
-		UplinkNetConfig:     new(config.AdapterNetConfig)}
+		Name:                  nodeName,
+		OVSBridge:             i.ovsBridge,
+		DefaultTunName:        defaultTunInterfaceName,
+		NodeIPv4Addr:          nodeIPv4Addr,
+		NodeIPv6Addr:          nodeIPv6Addr,
+		NodeTransportIPv4Addr: transportIPv4Addr,
+		NodeTransportIPv6Addr: transportIPv6Addr,
+		UplinkNetConfig:       new(config.AdapterNetConfig)}
 
 	mtu, err := i.getNodeMTU(localIntf)
 	if err != nil {
@@ -863,7 +876,7 @@ func (i *Initializer) getNodeMTU(localIntf *net.Interface) (int, error) {
 		} else if i.networkConfig.TunnelType == ovsconfig.GRETunnel {
 			mtu -= config.GREOverhead
 		}
-		if i.nodeConfig.NodeIPAddr.IP.To4() == nil {
+		if i.nodeConfig.NodeIPv6Addr != nil {
 			mtu -= config.IPv6ExtraOverhead
 		}
 	}

diff --git a/pkg/agent/agent_linux.go b/pkg/agent/agent_linux.go
@@ -43,6 +43,6 @@ func (i *Initializer) getTunnelPortLocalIP() net.IP {
 	return nil
 }
 
-func GetTransportIPNetDeviceByName(ifaceName string, ovsBridgeName string) (*net.IPNet, *net.Interface, error) {
+func GetTransportIPNetDeviceByName(ifaceName string, ovsBridgeName string) (*net.IPNet, *net.IPNet, *net.Interface, error) {
 	return util.GetIPNetDeviceByName(ifaceName)
 }
diff --git a/pkg/agent/agent_test.go b/pkg/agent/agent_test.go
@@ -36,6 +36,7 @@ import (
 	"antrea.io/antrea/pkg/ovs/ovsconfig"
 	ovsconfigtest "antrea.io/antrea/pkg/ovs/ovsconfig/testing"
 	"antrea.io/antrea/pkg/util/env"
+	"antrea.io/antrea/pkg/util/ip"
 )
 
 func newAgentInitializer(ovsBridgeClient ovsconfig.OVSBridgeClient, ifaceStore interfacestore.InterfaceStore) *Initializer {
@@ -278,14 +279,14 @@ func TestInitNodeLocalConfig(t *testing.T) {
 			client := fake.NewSimpleClientset(node)
 			ifaceStore := interfacestore.NewInterfaceStore()
 			expectedNodeConfig := config.NodeConfig{
-				Name:                nodeName,
-				OVSBridge:           ovsBridge,
-				DefaultTunName:      defaultTunInterfaceName,
-				PodIPv4CIDR:         podCIDR,
-				NodeIPAddr:          nodeIPNet,
-				NodeTransportIPAddr: nodeIPNet,
-				NodeMTU:             tt.expectedMTU,
-				UplinkNetConfig:     new(config.AdapterNetConfig),
+				Name:                  nodeName,
+				OVSBridge:             ovsBridge,
+				DefaultTunName:        defaultTunInterfaceName,
+				PodIPv4CIDR:           podCIDR,
+				NodeIPv4Addr:          nodeIPNet,
+				NodeTransportIPv4Addr: nodeIPNet,
+				NodeMTU:               tt.expectedMTU,
+				UplinkNetConfig:       new(config.AdapterNetConfig),
 			}
 
 			initializer := &Initializer{
@@ -300,7 +301,7 @@ func TestInitNodeLocalConfig(t *testing.T) {
 			}
 			if tt.transportInterface != nil {
 				initializer.networkConfig.TransportIface = tt.transportInterface.iface.Name
-				expectedNodeConfig.NodeTransportIPAddr = tt.transportInterface.ipNet
+				expectedNodeConfig.NodeTransportIPv4Addr = tt.transportInterface.ipNet
 				defer mockGetTransportIPNetDeviceByName(tt.transportInterface.ipNet, tt.transportInterface.iface)()
 			}
 			defer mockGetIPNetDeviceFromIP(nodeIPNet, ipDevice)()
@@ -317,8 +318,8 @@ func TestInitNodeLocalConfig(t *testing.T) {
 
 func mockGetIPNetDeviceFromIP(ipNet *net.IPNet, ipDevice *net.Interface) func() {
 	prevGetIPNetDeviceFromIP := getIPNetDeviceFromIP
-	getIPNetDeviceFromIP = func(localIP net.IP) (*net.IPNet, *net.Interface, error) {
-		return ipNet, ipDevice, nil
+	getIPNetDeviceFromIP = func(localIP *ip.DualStackIPs) (*net.IPNet, *net.IPNet, *net.Interface, error) {
+		return ipNet, nil, ipDevice, nil
 	}
 	return func() { getIPNetDeviceFromIP = prevGetIPNetDeviceFromIP }
 }
@@ -330,8 +331,8 @@ func mockNodeNameEnv(name string) func() {
 
 func mockGetTransportIPNetDeviceByName(ipNet *net.IPNet, ipDevice *net.Interface) func() {
 	prevGetIPNetDeviceByName := getTransportIPNetDeviceByName
-	getTransportIPNetDeviceByName = func(ifName, brName string) (*net.IPNet, *net.Interface, error) {
-		return ipNet, ipDevice, nil
+	getTransportIPNetDeviceByName = func(ifName, brName string) (*net.IPNet, *net.IPNet, *net.Interface, error) {
+		return ipNet, nil, ipDevice, nil
 	}
 	return func() { getTransportIPNetDeviceByName = prevGetIPNetDeviceByName }
 }
diff --git a/pkg/agent/agent_windows.go b/pkg/agent/agent_windows.go
@@ -28,6 +28,7 @@ import (
 	"antrea.io/antrea/pkg/agent/interfacestore"
 	"antrea.io/antrea/pkg/agent/util"
 	"antrea.io/antrea/pkg/ovs/ovsctl"
+	"antrea.io/antrea/pkg/util/ip"
 )
 
 // prepareHostNetwork creates HNS Network for containers.
@@ -49,7 +50,7 @@ func (i *Initializer) prepareHostNetwork() error {
 	// Get uplink network configuration. The uplink interface is the one used for transporting Pod traffic across Nodes.
 	// Use the interface specified with "transportInterface" in the configuration if configured, otherwise the interface
 	// configured with NodeIP is used as uplink.
-	_, adapter, err := util.GetIPNetDeviceFromIP(i.nodeConfig.NodeTransportIPAddr.IP)
+	_, _, adapter, err := util.GetIPNetDeviceFromIP(&ip.DualStackIPs{IPv4: i.nodeConfig.NodeTransportIPv4Addr.IP})
 	if err != nil {
 		return err
 	}
@@ -68,7 +69,7 @@ func (i *Initializer) prepareHostNetwork() error {
 	}
 	i.nodeConfig.UplinkNetConfig.Name = adapter.Name
 	i.nodeConfig.UplinkNetConfig.MAC = adapter.HardwareAddr
-	i.nodeConfig.UplinkNetConfig.IP = i.nodeConfig.NodeTransportIPAddr
+	i.nodeConfig.UplinkNetConfig.IP = i.nodeConfig.NodeTransportIPv4Addr
 	i.nodeConfig.UplinkNetConfig.Index = adapter.Index
 	defaultGW, err := util.GetDefaultGatewayByInterfaceIndex(adapter.Index)
 	if err != nil {
@@ -96,7 +97,7 @@ func (i *Initializer) prepareHostNetwork() error {
 	if subnetCIDR == nil {
 		return fmt.Errorf("failed to find valid IPv4 PodCIDR")
 	}
-	return util.PrepareHNSNetwork(subnetCIDR, i.nodeConfig.NodeTransportIPAddr, adapter)
+	return util.PrepareHNSNetwork(subnetCIDR, i.nodeConfig.NodeTransportIPv4Addr, adapter)
 }
 
 // prepareOVSBridge adds local port and uplink to ovs bridge.
@@ -213,7 +214,7 @@ func (i *Initializer) initHostNetworkFlows() error {
 
 // getTunnelLocalIP returns local_ip of tunnel port
 func (i *Initializer) getTunnelPortLocalIP() net.IP {
-	return i.nodeConfig.NodeTransportIPAddr.IP
+	return i.nodeConfig.NodeTransportIPv4Addr.IP
 }
 
 // saveHostRoutes saves routes which are configured on uplink interface before
@@ -271,17 +272,17 @@ func (i *Initializer) restoreHostRoutes() error {
 	return nil
 }
 
-func GetTransportIPNetDeviceByName(ifaceName string, ovsBridgeName string) (*net.IPNet, *net.Interface, error) {
+func GetTransportIPNetDeviceByName(ifaceName string, ovsBridgeName string) (*net.IPNet, *net.IPNet, *net.Interface, error) {
 	// Find transport Interface in the order: ifaceName -> "vEthernet (ifaceName)" -> br-int. Return immediately if
 	// an interface using the specified name exists. Using "vEthernet (ifaceName)" or br-int is for restart agent case.
 	for _, name := range []string{ifaceName, fmt.Sprintf("vEthernet (%s)", ifaceName), ovsBridgeName} {
-		ipNet, link, err := util.GetIPNetDeviceByName(name)
+		ipNet, _, link, err := util.GetIPNetDeviceByName(name)
 		if err == nil {
-			return ipNet, link, nil
+			return ipNet, nil, link, nil
 		}
 		if !strings.Contains(err.Error(), "no such network interface") {
-			return nil, nil, err
+			return nil, nil, nil, err
 		}
 	}
-	return nil, nil, fmt.Errorf("unable to find local IP and device")
+	return nil, nil, nil, fmt.Errorf("unable to find local IP and device")
 }
diff --git a/pkg/agent/config/node_config.go b/pkg/agent/config/node_config.go
@@ -83,10 +83,14 @@ type NodeConfig struct {
 	// The CIDR block from where to allocate IPv6 address to Pod.
 	// It's nil for the net workPolicyOnly trafficEncapMode which doesn't do IPAM.
 	PodIPv6CIDR *net.IPNet
-	// The Node's IP used in Kubernetes. It has the network mask information.
-	NodeIPAddr *net.IPNet
-	// The IP on the Node's transport interface. It is used for tunneling or routing the Pod traffic across Nodes.
-	NodeTransportIPAddr *net.IPNet
+	// The Node's IPv4 address used in Kubernetes. It has the network mask information.
+	NodeIPv4Addr *net.IPNet
+	// The Node's IPv6 address used in Kubernetes. It has the network mask information.
+	NodeIPv6Addr *net.IPNet
+	// The IPv4 address on the Node's transport interface. It is used for tunneling or routing the Pod traffic across Nodes.
+	NodeTransportIPv4Addr *net.IPNet
+	// The IPv6 address on the Node's transport interface. It is used for tunneling or routing the Pod traffic across Nodes.
+	NodeTransportIPv6Addr *net.IPNet
 	// Set either via defaultMTU config in antrea.yaml or auto discovered.
 	// Auto discovery will use MTU value of the Node's primary interface.
 	// For Encap and Hybrid mode, Node MTU will be adjusted to account for encap header.
@@ -98,8 +102,8 @@ type NodeConfig struct {
 }
 
 func (n *NodeConfig) String() string {
-	return fmt.Sprintf("NodeName: %s, OVSBridge: %s, PodIPv4CIDR: %s, PodIPv6CIDR: %s, NodeIP: %s, TransportIP: %s, Gateway: %s",
-		n.Name, n.OVSBridge, n.PodIPv4CIDR, n.PodIPv6CIDR, n.NodeIPAddr, n.NodeTransportIPAddr, n.GatewayConfig)
+	return fmt.Sprintf("NodeName: %s, OVSBridge: %s, PodIPv4CIDR: %s, PodIPv6CIDR: %s, NodeIPv4: %s, NodeIPv6: %s, TransportIPv4: %s, TransportIPv6: %s, Gateway: %s",
+		n.Name, n.OVSBridge, n.PodIPv4CIDR, n.PodIPv6CIDR, n.NodeIPv4Addr, n.NodeIPv6Addr, n.NodeTransportIPv4Addr, n.NodeTransportIPv6Addr, n.GatewayConfig)
 }
 
 // User provided network configuration parameters.
@@ -112,15 +116,13 @@ type NetworkConfig struct {
 }
 
 // IsIPv4Enabled returns true if the cluster network supports IPv4.
-// TODO: support dual-stack in networkPolicyOnly mode.
 func IsIPv4Enabled(nodeConfig *NodeConfig, trafficEncapMode TrafficEncapModeType) bool {
 	return nodeConfig.PodIPv4CIDR != nil ||
-		(trafficEncapMode.IsNetworkPolicyOnly() && nodeConfig.NodeIPAddr.IP.To4() != nil)
+		(trafficEncapMode.IsNetworkPolicyOnly() && nodeConfig.NodeIPv4Addr != nil)
 }
 
 // IsIPv6Enabled returns true if the cluster network supports IPv6.
-// TODO: support dual-stack in networkPolicyOnly mode.
 func IsIPv6Enabled(nodeConfig *NodeConfig, trafficEncapMode TrafficEncapModeType) bool {
 	return nodeConfig.PodIPv6CIDR != nil ||
-		(trafficEncapMode.IsNetworkPolicyOnly() && nodeConfig.NodeIPAddr.IP.To4() == nil)
+		(trafficEncapMode.IsNetworkPolicyOnly() && nodeConfig.NodeIPv6Addr != nil)
 }
diff --git a/pkg/agent/controller/egress/ipassigner/ip_assigner_linux.go b/pkg/agent/controller/egress/ipassigner/ip_assigner_linux.go
@@ -26,6 +26,7 @@ import (
 
 	"antrea.io/antrea/pkg/agent/util"
 	"antrea.io/antrea/pkg/agent/util/arping"
+	"antrea.io/antrea/pkg/util/ip"
 )
 
 var ipv6NotSupportErr = errors.New("IPv6 not supported")
@@ -49,7 +50,13 @@ type ipAssigner struct {
 
 // NewIPAssigner returns an *ipAssigner.
 func NewIPAssigner(nodeIPAddr net.IP, dummyDeviceName string) (*ipAssigner, error) {
-	_, egressInterface, err := util.GetIPNetDeviceFromIP(nodeIPAddr)
+	nodeIPs := new(ip.DualStackIPs)
+	if nodeIPAddr.To4() == nil {
+		nodeIPs.IPv6 = nodeIPAddr
+	} else {
+		nodeIPs.IPv4 = nodeIPAddr
+	}
+	_, _, egressInterface, err := util.GetIPNetDeviceFromIP(nodeIPs)
 	if err != nil {
 		return nil, fmt.Errorf("get IPNetDevice from ip %v error: %+v", nodeIPAddr, err)
 	}