From 920dcb6d3fd5a57960667825f8c0098289232329 Mon Sep 17 00:00:00 2001 From: Zhecheng Li Date: Fri, 16 Jul 2021 19:11:41 +0800 Subject: [PATCH] [IPv6] Support no-encap mode in dual-stack setup * Extend single IP address to IPv4 and IPv6 ones for NodeAddr and NodeTransportAddr in nodeConfig * Refactor occurrences of Node IP address to adjust to 2 IP: agent initialization, node route * Still use one IP for IPv6/dual-stack not supported cases: Windows, NPL * Update tests Signed-off-by: Zhecheng Li Related: #2426 --- cmd/antrea-agent/agent.go | 2 +- pkg/agent/agent.go | 57 ++++--- pkg/agent/agent_linux.go | 2 +- pkg/agent/agent_test.go | 27 ++-- pkg/agent/agent_windows.go | 19 +-- pkg/agent/config/node_config.go | 22 +-- .../egress/ipassigner/ip_assigner_linux.go | 9 +- .../noderoute/node_route_controller.go | 119 +++++++++------ .../noderoute/node_route_controller_test.go | 17 +-- pkg/agent/controller/traceflow/packetin.go | 2 +- pkg/agent/memberlist/cluster.go | 12 +- pkg/agent/memberlist/cluster_test.go | 10 +- pkg/agent/openflow/client.go | 29 ++-- pkg/agent/openflow/client_test.go | 5 +- pkg/agent/openflow/pipeline_windows.go | 2 +- pkg/agent/openflow/testing/mock_openflow.go | 3 +- pkg/agent/querier/querier_test.go | 16 +- pkg/agent/route/route_linux.go | 38 ++++- pkg/agent/route/route_windows.go | 4 +- pkg/agent/types/annotations.go | 4 +- pkg/agent/util/net.go | 59 +++++-- pkg/agent/util/net_test.go | 9 +- pkg/antctl/raw/helper.go | 30 +++- pkg/antctl/raw/supportbundle/command.go | 23 ++- pkg/util/ip/ip.go | 9 ++ pkg/util/k8s/node.go | 40 +++-- pkg/util/k8s/node_test.go | 10 +- test/integration/agent/openflow_test.go | 144 +++++++++++------- test/integration/agent/route_test.go | 23 +-- test/integration/ovs/openflow_test_utils.go | 20 +-- 30 files changed, 490 insertions(+), 276 deletions(-) diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go index 97499e41b63..eb62ae142e1 100644 --- a/cmd/antrea-agent/agent.go +++ b/cmd/antrea-agent/agent.go @@ -229,7 +229,7 @@ func run(o *Options) error { var egressController *egress.EgressController if features.DefaultFeatureGate.Enabled(features.Egress) { egressController, err = egress.NewEgressController( - ofClient, antreaClientProvider, crdClient, ifaceStore, routeClient, nodeConfig.Name, nodeConfig.NodeIPAddr.IP, + ofClient, antreaClientProvider, crdClient, ifaceStore, routeClient, nodeConfig.Name, nodeConfig.NodeIPv4Addr.IP, o.config.ClusterMembershipPort, egressInformer, nodeInformer, externalIPPoolInformer, ) if err != nil { diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index e6925ee4614..b55f3a1c8c4 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -21,6 +21,7 @@ import ( "net" "os" "strconv" + "strings" "sync" "time" @@ -533,15 +534,17 @@ func (i *Initializer) configureGatewayInterface(gatewayIface *interfacestore.Int i.nodeConfig.GatewayConfig = &config.GatewayConfig{Name: i.hostGateway, MAC: gwMAC} gatewayIface.MAC = gwMAC + gatewayIface.IPs = []net.IP{} if i.networkConfig.TrafficEncapMode.IsNetworkPolicyOnly() { // Assign IP to gw as required by SpoofGuard. - // NodeIPAddr can be either IPv4 or IPv6. - if i.nodeConfig.NodeIPAddr.IP.To4() != nil { - i.nodeConfig.GatewayConfig.IPv4 = i.nodeConfig.NodeTransportIPAddr.IP - } else { - i.nodeConfig.GatewayConfig.IPv6 = i.nodeConfig.NodeTransportIPAddr.IP + if i.nodeConfig.NodeIPv4Addr != nil { + i.nodeConfig.GatewayConfig.IPv4 = i.nodeConfig.NodeTransportIPv4Addr.IP + gatewayIface.IPs = append(gatewayIface.IPs, i.nodeConfig.NodeTransportIPv4Addr.IP) + } + if i.nodeConfig.NodeIPv6Addr != nil { + i.nodeConfig.GatewayConfig.IPv6 = i.nodeConfig.NodeTransportIPv6Addr.IP + gatewayIface.IPs = append(gatewayIface.IPs, i.nodeConfig.NodeTransportIPv6Addr.IP) } - gatewayIface.IPs = []net.IP{i.nodeConfig.NodeTransportIPAddr.IP} // No need to assign local CIDR to gw0 because local CIDR is not managed by Antrea return nil } @@ -644,26 +647,34 @@ func (i *Initializer) initNodeLocalConfig() error { return err } - var nodeIPAddr, transportIPAddr *net.IPNet + var nodeIPv4Addr, nodeIPv6Addr, transportIPv4Addr, transportIPv6Addr *net.IPNet var localIntf *net.Interface // Find the interface configured with Node IP and use it for Pod traffic. - ipAddr, err := k8s.GetNodeAddr(node) + ipAddrs, err := k8s.GetNodeAddrs(node) if err != nil { - return fmt.Errorf("failed to obtain local IP address from K8s: %w", err) + return fmt.Errorf("failed to obtain local IP addresses from K8s: %w", err) } - nodeIPAddr, localIntf, err = getIPNetDeviceFromIP(ipAddr) + nodeIPv4Addr, nodeIPv6Addr, localIntf, err = getIPNetDeviceFromIP(ipAddrs) if err != nil { - return fmt.Errorf("failed to get local IPNet device with IP %v: %v", ipAddr, err) + return fmt.Errorf("failed to get local IPNet device with IP %v: %v", ipAddrs, err) } - transportIPAddr = nodeIPAddr + transportIPv4Addr = nodeIPv4Addr + transportIPv6Addr = nodeIPv6Addr if i.networkConfig.TransportIface != "" { // Find the configured transport interface, and update its IP address in Node's annotation. - transportIPAddr, localIntf, err = getTransportIPNetDeviceByName(i.networkConfig.TransportIface, i.ovsBridge) + transportIPv4Addr, transportIPv6Addr, localIntf, err = getTransportIPNetDeviceByName(i.networkConfig.TransportIface, i.ovsBridge) if err != nil { return fmt.Errorf("failed to get local IPNet device with transport interface %s: %v", i.networkConfig.TransportIface, err) } - klog.InfoS("Updating Node transport address annotation") - if err := i.patchNodeAnnotations(nodeName, types.NodeTransportAddressAnnotationKey, transportIPAddr.IP.String()); err != nil { + klog.InfoS("Updating Node transport addresses annotation") + var ips []string + if transportIPv4Addr != nil { + ips = append(ips, transportIPv4Addr.IP.String()) + } + if transportIPv6Addr != nil { + ips = append(ips, transportIPv6Addr.IP.String()) + } + if err := i.patchNodeAnnotations(nodeName, types.NodeTransportAddressAnnotationKey, strings.Join(ips, ",")); err != nil { return err } } else { @@ -685,12 +696,14 @@ func (i *Initializer) initNodeLocalConfig() error { } i.nodeConfig = &config.NodeConfig{ - Name: nodeName, - OVSBridge: i.ovsBridge, - DefaultTunName: defaultTunInterfaceName, - NodeIPAddr: nodeIPAddr, - NodeTransportIPAddr: transportIPAddr, - UplinkNetConfig: new(config.AdapterNetConfig)} + Name: nodeName, + OVSBridge: i.ovsBridge, + DefaultTunName: defaultTunInterfaceName, + NodeIPv4Addr: nodeIPv4Addr, + NodeIPv6Addr: nodeIPv6Addr, + NodeTransportIPv4Addr: transportIPv4Addr, + NodeTransportIPv6Addr: transportIPv6Addr, + UplinkNetConfig: new(config.AdapterNetConfig)} mtu, err := i.getNodeMTU(localIntf) if err != nil { @@ -863,7 +876,7 @@ func (i *Initializer) getNodeMTU(localIntf *net.Interface) (int, error) { } else if i.networkConfig.TunnelType == ovsconfig.GRETunnel { mtu -= config.GREOverhead } - if i.nodeConfig.NodeIPAddr.IP.To4() == nil { + if i.nodeConfig.NodeIPv6Addr != nil { mtu -= config.IPv6ExtraOverhead } } diff --git a/pkg/agent/agent_linux.go b/pkg/agent/agent_linux.go index 20e399cb4c8..1efcad7c23e 100644 --- a/pkg/agent/agent_linux.go +++ b/pkg/agent/agent_linux.go @@ -43,6 +43,6 @@ func (i *Initializer) getTunnelPortLocalIP() net.IP { return nil } -func GetTransportIPNetDeviceByName(ifaceName string, ovsBridgeName string) (*net.IPNet, *net.Interface, error) { +func GetTransportIPNetDeviceByName(ifaceName string, ovsBridgeName string) (*net.IPNet, *net.IPNet, *net.Interface, error) { return util.GetIPNetDeviceByName(ifaceName) } diff --git a/pkg/agent/agent_test.go b/pkg/agent/agent_test.go index b5741a52aca..3b975e736f3 100644 --- a/pkg/agent/agent_test.go +++ b/pkg/agent/agent_test.go @@ -36,6 +36,7 @@ import ( "antrea.io/antrea/pkg/ovs/ovsconfig" ovsconfigtest "antrea.io/antrea/pkg/ovs/ovsconfig/testing" "antrea.io/antrea/pkg/util/env" + "antrea.io/antrea/pkg/util/ip" ) func newAgentInitializer(ovsBridgeClient ovsconfig.OVSBridgeClient, ifaceStore interfacestore.InterfaceStore) *Initializer { @@ -278,14 +279,14 @@ func TestInitNodeLocalConfig(t *testing.T) { client := fake.NewSimpleClientset(node) ifaceStore := interfacestore.NewInterfaceStore() expectedNodeConfig := config.NodeConfig{ - Name: nodeName, - OVSBridge: ovsBridge, - DefaultTunName: defaultTunInterfaceName, - PodIPv4CIDR: podCIDR, - NodeIPAddr: nodeIPNet, - NodeTransportIPAddr: nodeIPNet, - NodeMTU: tt.expectedMTU, - UplinkNetConfig: new(config.AdapterNetConfig), + Name: nodeName, + OVSBridge: ovsBridge, + DefaultTunName: defaultTunInterfaceName, + PodIPv4CIDR: podCIDR, + NodeIPv4Addr: nodeIPNet, + NodeTransportIPv4Addr: nodeIPNet, + NodeMTU: tt.expectedMTU, + UplinkNetConfig: new(config.AdapterNetConfig), } initializer := &Initializer{ @@ -300,7 +301,7 @@ func TestInitNodeLocalConfig(t *testing.T) { } if tt.transportInterface != nil { initializer.networkConfig.TransportIface = tt.transportInterface.iface.Name - expectedNodeConfig.NodeTransportIPAddr = tt.transportInterface.ipNet + expectedNodeConfig.NodeTransportIPv4Addr = tt.transportInterface.ipNet defer mockGetTransportIPNetDeviceByName(tt.transportInterface.ipNet, tt.transportInterface.iface)() } defer mockGetIPNetDeviceFromIP(nodeIPNet, ipDevice)() @@ -317,8 +318,8 @@ func TestInitNodeLocalConfig(t *testing.T) { func mockGetIPNetDeviceFromIP(ipNet *net.IPNet, ipDevice *net.Interface) func() { prevGetIPNetDeviceFromIP := getIPNetDeviceFromIP - getIPNetDeviceFromIP = func(localIP net.IP) (*net.IPNet, *net.Interface, error) { - return ipNet, ipDevice, nil + getIPNetDeviceFromIP = func(localIP *ip.DualStackIPs) (*net.IPNet, *net.IPNet, *net.Interface, error) { + return ipNet, nil, ipDevice, nil } return func() { getIPNetDeviceFromIP = prevGetIPNetDeviceFromIP } } @@ -330,8 +331,8 @@ func mockNodeNameEnv(name string) func() { func mockGetTransportIPNetDeviceByName(ipNet *net.IPNet, ipDevice *net.Interface) func() { prevGetIPNetDeviceByName := getTransportIPNetDeviceByName - getTransportIPNetDeviceByName = func(ifName, brName string) (*net.IPNet, *net.Interface, error) { - return ipNet, ipDevice, nil + getTransportIPNetDeviceByName = func(ifName, brName string) (*net.IPNet, *net.IPNet, *net.Interface, error) { + return ipNet, nil, ipDevice, nil } return func() { getTransportIPNetDeviceByName = prevGetIPNetDeviceByName } } diff --git a/pkg/agent/agent_windows.go b/pkg/agent/agent_windows.go index 8c4d441423b..0328ded4c2b 100644 --- a/pkg/agent/agent_windows.go +++ b/pkg/agent/agent_windows.go @@ -28,6 +28,7 @@ import ( "antrea.io/antrea/pkg/agent/interfacestore" "antrea.io/antrea/pkg/agent/util" "antrea.io/antrea/pkg/ovs/ovsctl" + "antrea.io/antrea/pkg/util/ip" ) // prepareHostNetwork creates HNS Network for containers. @@ -49,7 +50,7 @@ func (i *Initializer) prepareHostNetwork() error { // Get uplink network configuration. The uplink interface is the one used for transporting Pod traffic across Nodes. // Use the interface specified with "transportInterface" in the configuration if configured, otherwise the interface // configured with NodeIP is used as uplink. - _, adapter, err := util.GetIPNetDeviceFromIP(i.nodeConfig.NodeTransportIPAddr.IP) + _, _, adapter, err := util.GetIPNetDeviceFromIP(&ip.DualStackIPs{IPv4: i.nodeConfig.NodeTransportIPv4Addr.IP}) if err != nil { return err } @@ -68,7 +69,7 @@ func (i *Initializer) prepareHostNetwork() error { } i.nodeConfig.UplinkNetConfig.Name = adapter.Name i.nodeConfig.UplinkNetConfig.MAC = adapter.HardwareAddr - i.nodeConfig.UplinkNetConfig.IP = i.nodeConfig.NodeTransportIPAddr + i.nodeConfig.UplinkNetConfig.IP = i.nodeConfig.NodeTransportIPv4Addr i.nodeConfig.UplinkNetConfig.Index = adapter.Index defaultGW, err := util.GetDefaultGatewayByInterfaceIndex(adapter.Index) if err != nil { @@ -96,7 +97,7 @@ func (i *Initializer) prepareHostNetwork() error { if subnetCIDR == nil { return fmt.Errorf("failed to find valid IPv4 PodCIDR") } - return util.PrepareHNSNetwork(subnetCIDR, i.nodeConfig.NodeTransportIPAddr, adapter) + return util.PrepareHNSNetwork(subnetCIDR, i.nodeConfig.NodeTransportIPv4Addr, adapter) } // prepareOVSBridge adds local port and uplink to ovs bridge. @@ -213,7 +214,7 @@ func (i *Initializer) initHostNetworkFlows() error { // getTunnelLocalIP returns local_ip of tunnel port func (i *Initializer) getTunnelPortLocalIP() net.IP { - return i.nodeConfig.NodeTransportIPAddr.IP + return i.nodeConfig.NodeTransportIPv4Addr.IP } // saveHostRoutes saves routes which are configured on uplink interface before @@ -271,17 +272,17 @@ func (i *Initializer) restoreHostRoutes() error { return nil } -func GetTransportIPNetDeviceByName(ifaceName string, ovsBridgeName string) (*net.IPNet, *net.Interface, error) { +func GetTransportIPNetDeviceByName(ifaceName string, ovsBridgeName string) (*net.IPNet, *net.IPNet, *net.Interface, error) { // Find transport Interface in the order: ifaceName -> "vEthernet (ifaceName)" -> br-int. Return immediately if // an interface using the specified name exists. Using "vEthernet (ifaceName)" or br-int is for restart agent case. for _, name := range []string{ifaceName, fmt.Sprintf("vEthernet (%s)", ifaceName), ovsBridgeName} { - ipNet, link, err := util.GetIPNetDeviceByName(name) + ipNet, _, link, err := util.GetIPNetDeviceByName(name) if err == nil { - return ipNet, link, nil + return ipNet, nil, link, nil } if !strings.Contains(err.Error(), "no such network interface") { - return nil, nil, err + return nil, nil, nil, err } } - return nil, nil, fmt.Errorf("unable to find local IP and device") + return nil, nil, nil, fmt.Errorf("unable to find local IP and device") } diff --git a/pkg/agent/config/node_config.go b/pkg/agent/config/node_config.go index 750b4ba2ee3..2089d359f9a 100644 --- a/pkg/agent/config/node_config.go +++ b/pkg/agent/config/node_config.go @@ -83,10 +83,14 @@ type NodeConfig struct { // The CIDR block from where to allocate IPv6 address to Pod. // It's nil for the net workPolicyOnly trafficEncapMode which doesn't do IPAM. PodIPv6CIDR *net.IPNet - // The Node's IP used in Kubernetes. It has the network mask information. - NodeIPAddr *net.IPNet - // The IP on the Node's transport interface. It is used for tunneling or routing the Pod traffic across Nodes. - NodeTransportIPAddr *net.IPNet + // The Node's IPv4 address used in Kubernetes. It has the network mask information. + NodeIPv4Addr *net.IPNet + // The Node's IPv6 address used in Kubernetes. It has the network mask information. + NodeIPv6Addr *net.IPNet + // The IPv4 address on the Node's transport interface. It is used for tunneling or routing the Pod traffic across Nodes. + NodeTransportIPv4Addr *net.IPNet + // The IPv6 address on the Node's transport interface. It is used for tunneling or routing the Pod traffic across Nodes. + NodeTransportIPv6Addr *net.IPNet // Set either via defaultMTU config in antrea.yaml or auto discovered. // Auto discovery will use MTU value of the Node's primary interface. // For Encap and Hybrid mode, Node MTU will be adjusted to account for encap header. @@ -98,8 +102,8 @@ type NodeConfig struct { } func (n *NodeConfig) String() string { - return fmt.Sprintf("NodeName: %s, OVSBridge: %s, PodIPv4CIDR: %s, PodIPv6CIDR: %s, NodeIP: %s, TransportIP: %s, Gateway: %s", - n.Name, n.OVSBridge, n.PodIPv4CIDR, n.PodIPv6CIDR, n.NodeIPAddr, n.NodeTransportIPAddr, n.GatewayConfig) + return fmt.Sprintf("NodeName: %s, OVSBridge: %s, PodIPv4CIDR: %s, PodIPv6CIDR: %s, NodeIPv4: %s, NodeIPv6: %s, TransportIPv4: %s, TransportIPv6: %s, Gateway: %s", + n.Name, n.OVSBridge, n.PodIPv4CIDR, n.PodIPv6CIDR, n.NodeIPv4Addr, n.NodeIPv6Addr, n.NodeTransportIPv4Addr, n.NodeTransportIPv6Addr, n.GatewayConfig) } // User provided network configuration parameters. @@ -112,15 +116,13 @@ type NetworkConfig struct { } // IsIPv4Enabled returns true if the cluster network supports IPv4. -// TODO: support dual-stack in networkPolicyOnly mode. func IsIPv4Enabled(nodeConfig *NodeConfig, trafficEncapMode TrafficEncapModeType) bool { return nodeConfig.PodIPv4CIDR != nil || - (trafficEncapMode.IsNetworkPolicyOnly() && nodeConfig.NodeIPAddr.IP.To4() != nil) + (trafficEncapMode.IsNetworkPolicyOnly() && nodeConfig.NodeIPv4Addr != nil) } // IsIPv6Enabled returns true if the cluster network supports IPv6. -// TODO: support dual-stack in networkPolicyOnly mode. func IsIPv6Enabled(nodeConfig *NodeConfig, trafficEncapMode TrafficEncapModeType) bool { return nodeConfig.PodIPv6CIDR != nil || - (trafficEncapMode.IsNetworkPolicyOnly() && nodeConfig.NodeIPAddr.IP.To4() == nil) + (trafficEncapMode.IsNetworkPolicyOnly() && nodeConfig.NodeIPv6Addr != nil) } diff --git a/pkg/agent/controller/egress/ipassigner/ip_assigner_linux.go b/pkg/agent/controller/egress/ipassigner/ip_assigner_linux.go index 0870e13f1da..6674f5677d8 100644 --- a/pkg/agent/controller/egress/ipassigner/ip_assigner_linux.go +++ b/pkg/agent/controller/egress/ipassigner/ip_assigner_linux.go @@ -26,6 +26,7 @@ import ( "antrea.io/antrea/pkg/agent/util" "antrea.io/antrea/pkg/agent/util/arping" + "antrea.io/antrea/pkg/util/ip" ) var ipv6NotSupportErr = errors.New("IPv6 not supported") @@ -49,7 +50,13 @@ type ipAssigner struct { // NewIPAssigner returns an *ipAssigner. func NewIPAssigner(nodeIPAddr net.IP, dummyDeviceName string) (*ipAssigner, error) { - _, egressInterface, err := util.GetIPNetDeviceFromIP(nodeIPAddr) + nodeIPs := new(ip.DualStackIPs) + if nodeIPAddr.To4() == nil { + nodeIPs.IPv6 = nodeIPAddr + } else { + nodeIPs.IPv4 = nodeIPAddr + } + _, _, egressInterface, err := util.GetIPNetDeviceFromIP(nodeIPs) if err != nil { return nil, fmt.Errorf("get IPNetDevice from ip %v error: %+v", nodeIPAddr, err) } diff --git a/pkg/agent/controller/noderoute/node_route_controller.go b/pkg/agent/controller/noderoute/node_route_controller.go index 8480395984e..5cd5b09416e 100644 --- a/pkg/agent/controller/noderoute/node_route_controller.go +++ b/pkg/agent/controller/noderoute/node_route_controller.go @@ -17,6 +17,7 @@ package noderoute import ( "fmt" "net" + "strings" "time" "github.com/containernetworking/plugins/pkg/ip" @@ -133,11 +134,11 @@ func nodeRouteInfoPodCIDRIndexFunc(obj interface{}) ([]string, error) { // nodeRouteInfo is the route related information extracted from corev1.Node. type nodeRouteInfo struct { - nodeName string - podCIDRs []*net.IPNet - nodeIP net.IP - gatewayIP []net.IP - nodeMAC net.HardwareAddr + nodeName string + podCIDRs []*net.IPNet + nodeIPs *utilip.DualStackIPs + gatewayIPs *utilip.DualStackIPs + nodeMAC net.HardwareAddr } // enqueueNode adds an object to the controller work queue @@ -215,7 +216,7 @@ func (c *Controller) removeStaleTunnelPorts() error { continue } - peerNodeIP, err := k8s.GetNodeAddr(node) + peerNodeIPs, err := k8s.GetNodeAddrs(node) if err != nil { klog.Errorf("Failed to retrieve IP address of Node %s: %v", node.Name, err) continue @@ -223,7 +224,7 @@ func (c *Controller) removeStaleTunnelPorts() error { ifaceID := util.GenerateNodeTunnelInterfaceKey(node.Name) ifaceName := util.GenerateNodeTunnelInterfaceName(node.Name) - if c.compareInterfaceConfig(interfaceConfig, peerNodeIP, ifaceName) { + if c.compareInterfaceConfig(interfaceConfig, peerNodeIPs.IPv4, ifaceName) || c.compareInterfaceConfig(interfaceConfig, peerNodeIPs.IPv6, ifaceName) { desiredInterfaces[ifaceID] = true } } @@ -420,14 +421,13 @@ func (c *Controller) addNodeRoute(nodeName string, node *corev1.Node) error { if err != nil { return fmt.Errorf("error when retrieving MAC of Node %s: %v", nodeName, err) } - peerNodeIP, err := c.getNodeTransportAddress(node) + peerNodeIPs, err := c.getNodeTransportAddrs(node) if err != nil { return err } nrInfo, installed, _ := c.installedNodes.GetByKey(nodeName) - - if installed && nrInfo.(*nodeRouteInfo).nodeMAC.String() == peerNodeMAC.String() && nrInfo.(*nodeRouteInfo).nodeIP.Equal(peerNodeIP) { + if installed && nrInfo.(*nodeRouteInfo).nodeMAC.String() == peerNodeMAC.String() && peerNodeIPs.Equal(*nrInfo.(*nodeRouteInfo).nodeIPs) { // Route is already added for this Node and both Node MAC and transport IP are not changed. return nil } @@ -437,15 +437,12 @@ func (c *Controller) addNodeRoute(nodeName string, node *corev1.Node) error { // If no valid PodCIDR is configured in Node.Spec, return immediately. return nil } - klog.Infof("Adding routes and flows to Node %s, podCIDRs: %v, addresses: %v", - nodeName, podCIDRStrs, node.Status.Addresses) + klog.InfoS("Adding routes and flows to Node", "Node", nodeName, "podCIDRs", podCIDRStrs, + "addresses", node.Status.Addresses) - var podCIDRs []*net.IPNet - peerConfig := make(map[*net.IPNet]net.IP, len(podCIDRStrs)) + var peerPodCIDRs []*net.IPNet + peerConfigs := make(map[*net.IPNet]net.IP, len(podCIDRStrs)) for _, podCIDR := range podCIDRStrs { - klog.Infof("Adding routes and flows to Node %s, podCIDR: %s, addresses: %v", - nodeName, podCIDR, node.Status.Addresses) - if podCIDR == "" { klog.Errorf("PodCIDR is empty for Node %s", nodeName) // Does not help to return an error and trigger controller retries. @@ -476,44 +473,64 @@ func (c *Controller) addNodeRoute(nodeName string, node *corev1.Node) error { return nil } peerGatewayIP := ip.NextIP(peerPodCIDRAddr) - peerConfig[peerPodCIDR] = peerGatewayIP - podCIDRs = append(podCIDRs, peerPodCIDR) + peerConfigs[peerPodCIDR] = peerGatewayIP + peerPodCIDRs = append(peerPodCIDRs, peerPodCIDR) + peerNodeIP := peerNodeIPs.IPv4 + if peerGatewayIP.To4() == nil { + peerNodeIP = peerNodeIPs.IPv6 + } + + klog.InfoS("Adding route and flow to Node", "Node", nodeName, "podCIDR", podCIDR, + "peerNodeIP", peerNodeIP) } - ipsecTunOFPort := int32(0) + var ipsecTunOFPort uint32 if c.networkConfig.EnableIPSecTunnel { // Create a separate tunnel port for the Node, as OVS IPSec monitor needs to // read PSK and remote IP from the Node's tunnel interface to create IPSec // security policies. - if ipsecTunOFPort, err = c.createIPSecTunnelPort(nodeName, peerNodeIP); err != nil { + peerNodeIP := peerNodeIPs.IPv4 + if peerNodeIP == nil { + peerNodeIP = peerNodeIPs.IPv6 + } + port, err := c.createIPSecTunnelPort(nodeName, peerNodeIP) + if err != nil { return err } + ipsecTunOFPort = uint32(port) } - err = c.ofClient.InstallNodeFlows( + if err = c.ofClient.InstallNodeFlows( nodeName, - peerConfig, - peerNodeIP, - uint32(ipsecTunOFPort), - peerNodeMAC) - if err != nil { + peerConfigs, + peerNodeIPs, + ipsecTunOFPort, + peerNodeMAC); err != nil { return fmt.Errorf("failed to install flows to Node %s: %v", nodeName, err) } - var peerGatewayIPs []net.IP - for peerPodCIDR, peerGatewayIP := range peerConfig { - if err := c.routeClient.AddRoutes(peerPodCIDR, nodeName, peerNodeIP, peerGatewayIP); err != nil { - return err + peerGatewayIPs := new(utilip.DualStackIPs) + for peerPodCIDR, peerGatewayIP := range peerConfigs { + if peerGatewayIP.To4() == nil { + if err := c.routeClient.AddRoutes(peerPodCIDR, nodeName, peerNodeIPs.IPv6, peerGatewayIP); err != nil { + return err + } + peerGatewayIPs.IPv6 = peerGatewayIP + } else { + if err := c.routeClient.AddRoutes(peerPodCIDR, nodeName, peerNodeIPs.IPv4, peerGatewayIP); err != nil { + return err + } + peerGatewayIPs.IPv4 = peerGatewayIP } - peerGatewayIPs = append(peerGatewayIPs, peerGatewayIP) } c.installedNodes.Add(&nodeRouteInfo{ - nodeName: nodeName, - podCIDRs: podCIDRs, - nodeIP: peerNodeIP, - gatewayIP: peerGatewayIPs, - nodeMAC: peerNodeMAC, + nodeName: nodeName, + podCIDRs: peerPodCIDRs, + nodeIPs: peerNodeIPs, + gatewayIPs: peerGatewayIPs, + nodeMAC: peerNodeMAC, }) + return err } @@ -666,23 +683,31 @@ func getNodeMAC(node *corev1.Node) (net.HardwareAddr, error) { return mac, nil } -func (c *Controller) getNodeTransportAddress(node *corev1.Node) (net.IP, error) { +func (c *Controller) getNodeTransportAddrs(node *corev1.Node) (*utilip.DualStackIPs, error) { + var transportAddrs *utilip.DualStackIPs if c.networkConfig.TransportIface != "" { - transportAddrStr := node.Annotations[types.NodeTransportAddressAnnotationKey] - if transportAddrStr != "" { - peerNodeAddr := net.ParseIP(transportAddrStr) - if peerNodeAddr == nil { - return nil, fmt.Errorf("invalid annotation for transport-address on Node %s: %s", node.Name, transportAddrStr) + transportAddrsStr := node.Annotations[types.NodeTransportAddressAnnotationKey] + if transportAddrsStr != "" { + for _, addr := range strings.Split(transportAddrsStr, ",") { + peerNodeAddr := net.ParseIP(addr) + if peerNodeAddr == nil { + return nil, fmt.Errorf("invalid annotation for transport-address on Node %s: %s", node.Name, transportAddrsStr) + } + if peerNodeAddr.To4() == nil { + transportAddrs.IPv6 = peerNodeAddr + } else { + transportAddrs.IPv4 = peerNodeAddr + } } - return peerNodeAddr, nil + return transportAddrs, nil } klog.InfoS("Transport address is not found, using NodeIP instead") } // Use NodeIP if the transport IP address is not set or not found. - peerNodeIP, err := k8s.GetNodeAddr(node) + peerNodeIPs, err := k8s.GetNodeAddrs(node) if err != nil { - klog.ErrorS(err, "Failed to retrieve Node IP address", "node", node.Name) + klog.ErrorS(err, "Failed to retrieve Node IP addresses", "node", node.Name) return nil, err } - return peerNodeIP, nil + return peerNodeIPs, nil } diff --git a/pkg/agent/controller/noderoute/node_route_controller_test.go b/pkg/agent/controller/noderoute/node_route_controller_test.go index 893ea50a96c..658116e244d 100644 --- a/pkg/agent/controller/noderoute/node_route_controller_test.go +++ b/pkg/agent/controller/noderoute/node_route_controller_test.go @@ -35,6 +35,7 @@ import ( "antrea.io/antrea/pkg/agent/util" "antrea.io/antrea/pkg/ovs/ovsconfig" ovsconfigtest "antrea.io/antrea/pkg/ovs/ovsconfig/testing" + utilip "antrea.io/antrea/pkg/util/ip" ) var ( @@ -44,7 +45,9 @@ var ( podCIDRGateway = ip.NextIP(podCIDR.IP) podCIDR2Gateway = ip.NextIP(podCIDR2.IP) nodeIP1 = net.ParseIP("10.10.10.10") + dsIPs1 = utilip.DualStackIPs{IPv4: nodeIP1} nodeIP2 = net.ParseIP("10.10.10.11") + dsIPs2 = utilip.DualStackIPs{IPv4: nodeIP2} ) type fakeController struct { @@ -134,9 +137,7 @@ func TestControllerWithDuplicatePodCIDR(t *testing.T) { defer close(finishCh) c.clientset.CoreV1().Nodes().Create(context.TODO(), node1, metav1.CreateOptions{}) - // The 2nd argument is Any() because the argument is unpredictable when it uses pointer as the key of map. - // The argument type is map[*net.IPNet]net.IP. - c.ofClient.EXPECT().InstallNodeFlows("node1", gomock.Any(), nodeIP1, uint32(0), nil).Times(1) + c.ofClient.EXPECT().InstallNodeFlows("node1", gomock.Any(), &dsIPs1, uint32(0), nil).Times(1) c.routeClient.EXPECT().AddRoutes(podCIDR, "node1", nodeIP1, podCIDRGateway).Times(1) c.processNextWorkItem() @@ -151,9 +152,7 @@ func TestControllerWithDuplicatePodCIDR(t *testing.T) { c.processNextWorkItem() // After node1 is deleted, routes and flows should be installed for node2 successfully. - // The 2nd argument is Any() because the argument is unpredictable when it uses pointer as the key of map. - // The argument type is map[*net.IPNet]net.IP. - c.ofClient.EXPECT().InstallNodeFlows("node2", gomock.Any(), nodeIP2, uint32(0), nil).Times(1) + c.ofClient.EXPECT().InstallNodeFlows("node2", gomock.Any(), &dsIPs2, uint32(0), nil).Times(1) c.routeClient.EXPECT().AddRoutes(podCIDR, "node2", nodeIP2, podCIDRGateway).Times(1) c.processNextWorkItem() }() @@ -215,14 +214,12 @@ func TestIPInPodSubnets(t *testing.T) { } c.clientset.CoreV1().Nodes().Create(context.TODO(), node1, metav1.CreateOptions{}) - // The 2nd argument is Any() because the argument is unpredictable when it uses pointer as the key of map. - // The argument type is map[*net.IPNet]net.IP. - c.ofClient.EXPECT().InstallNodeFlows("node1", gomock.Any(), nodeIP1, uint32(0), nil).Times(1) + c.ofClient.EXPECT().InstallNodeFlows("node1", gomock.Any(), &dsIPs1, uint32(0), nil).Times(1) c.routeClient.EXPECT().AddRoutes(podCIDR, "node1", nodeIP1, podCIDRGateway).Times(1) c.processNextWorkItem() c.clientset.CoreV1().Nodes().Create(context.TODO(), node2, metav1.CreateOptions{}) - c.ofClient.EXPECT().InstallNodeFlows("node2", gomock.Any(), nodeIP2, uint32(0), nil).Times(1) + c.ofClient.EXPECT().InstallNodeFlows("node2", gomock.Any(), &dsIPs2, uint32(0), nil).Times(1) c.routeClient.EXPECT().AddRoutes(podCIDR2, "node2", nodeIP2, podCIDR2Gateway).Times(1) c.processNextWorkItem() diff --git a/pkg/agent/controller/traceflow/packetin.go b/pkg/agent/controller/traceflow/packetin.go index 63c7ae6e4ec..00134eb3997 100644 --- a/pkg/agent/controller/traceflow/packetin.go +++ b/pkg/agent/controller/traceflow/packetin.go @@ -231,7 +231,7 @@ func (c *Controller) parsePacketIn(pktIn *ofctrl.PacketIn) (*crdv1alpha1.Tracefl if tableID == uint8(openflow.L2ForwardingOutTable) { ob := new(crdv1alpha1.Observation) tunnelDstIP := "" - isIPv6 := c.nodeConfig.NodeIPAddr.IP.To4() == nil + isIPv6 := c.nodeConfig.NodeIPv6Addr != nil if match := getMatchTunnelDstField(matchers, isIPv6); match != nil { tunnelDstIP, err = getTunnelDstValue(match) if err != nil { diff --git a/pkg/agent/memberlist/cluster.go b/pkg/agent/memberlist/cluster.go index 27b38ba4a4f..77cccfcc26d 100644 --- a/pkg/agent/memberlist/cluster.go +++ b/pkg/agent/memberlist/cluster.go @@ -252,11 +252,17 @@ func (c *Cluster) enqueueExternalIPPool(obj interface{}) { // newClusterMember gets the Node's IP and returns a cluster member ":" // representing that Node in the memberlist cluster. func (c *Cluster) newClusterMember(node *corev1.Node) (string, error) { - nodeAddr, err := k8s.GetNodeAddr(node) + nodeAddrs, err := k8s.GetNodeAddrs(node) if err != nil { - return "", fmt.Errorf("obtain IP address from K8s Node failed: %v", err) + return "", fmt.Errorf("obtain IP addresses from K8s Node failed: %v", err) } - member := fmt.Sprintf("%s:%d", nodeAddr, c.bindPort) + nodeAddr := nodeAddrs.IPv4 + fmtStr := "%s:%d" + if nodeAddr == nil { + nodeAddr = nodeAddrs.IPv6 + fmtStr = "[%s]:%d" + } + member := fmt.Sprintf(fmtStr, nodeAddr, c.bindPort) return member, nil } diff --git a/pkg/agent/memberlist/cluster_test.go b/pkg/agent/memberlist/cluster_test.go index f08d5688e04..f5f45b8670f 100644 --- a/pkg/agent/memberlist/cluster_test.go +++ b/pkg/agent/memberlist/cluster_test.go @@ -58,7 +58,7 @@ func newFakeCluster(nodeConfig *config.NodeConfig, stopCh <-chan struct{}, i int crdInformerFactory := crdinformers.NewSharedInformerFactory(crdClient, 0) ipPoolInformer := crdInformerFactory.Crd().V1alpha2().ExternalIPPools() - cluster, err := NewCluster(port, nodeConfig.NodeIPAddr.IP, nodeConfig.Name, nodeInformer, ipPoolInformer) + cluster, err := NewCluster(port, nodeConfig.NodeIPv4Addr.IP, nodeConfig.Name, nodeInformer, ipPoolInformer) if err != nil { return nil, err } @@ -137,8 +137,8 @@ func TestCluster_Run(t *testing.T) { for i, tCase := range testCases { t.Run(tCase.name, func(t *testing.T) { nodeConfig := &config.NodeConfig{ - Name: localNodeName, - NodeIPAddr: &net.IPNet{IP: net.IPv4(127, 0, 0, 1), Mask: net.IPv4Mask(255, 255, 255, 255)}, + Name: localNodeName, + NodeIPv4Addr: &net.IPNet{IP: net.IPv4(127, 0, 0, 1), Mask: net.IPv4Mask(255, 255, 255, 255)}, } stopCh := make(chan struct{}) defer close(stopCh) @@ -177,8 +177,8 @@ func TestCluster_RunClusterEvents(t *testing.T) { nodeName := "localNodeName" nodeConfig := &config.NodeConfig{ - Name: nodeName, - NodeIPAddr: &net.IPNet{IP: net.IPv4(127, 0, 0, 1)}, + Name: nodeName, + NodeIPv4Addr: &net.IPNet{IP: net.IPv4(127, 0, 0, 1)}, } localNode := &v1.Node{ ObjectMeta: metav1.ObjectMeta{Name: nodeName}, diff --git a/pkg/agent/openflow/client.go b/pkg/agent/openflow/client.go index c00bec3be23..106626cfd6b 100644 --- a/pkg/agent/openflow/client.go +++ b/pkg/agent/openflow/client.go @@ -27,6 +27,7 @@ import ( "antrea.io/antrea/pkg/agent/types" "antrea.io/antrea/pkg/agent/util" binding "antrea.io/antrea/pkg/ovs/openflow" + utilip "antrea.io/antrea/pkg/util/ip" "antrea.io/antrea/third_party/proxy" ) @@ -67,7 +68,7 @@ type Client interface { InstallNodeFlows( hostname string, peerConfigs map[*net.IPNet]net.IP, - tunnelPeerIP net.IP, + tunnelPeerIP *utilip.DualStackIPs, ipsecTunOFPort uint32, peerNodeMAC net.HardwareAddr) error @@ -393,7 +394,7 @@ func (c *client) deleteFlows(cache *flowCategoryCache, flowCacheKey string) erro // InstallNodeFlows installs flows for peer Nodes. Parameter remoteGatewayMAC is only for Windows. func (c *client) InstallNodeFlows(hostname string, peerConfigs map[*net.IPNet]net.IP, - tunnelPeerIP net.IP, + tunnelPeerIPs *utilip.DualStackIPs, ipsecTunOFPort uint32, remoteGatewayMAC net.HardwareAddr) error { c.replayMutex.RLock() @@ -403,20 +404,24 @@ func (c *client) InstallNodeFlows(hostname string, localGatewayMAC := c.nodeConfig.GatewayConfig.MAC for peerPodCIDR, peerGatewayIP := range peerConfigs { - if peerGatewayIP.To4() != nil { + isIPv6 := peerGatewayIP.To4() == nil + tunnelPeerIP := tunnelPeerIPs.IPv4 + if isIPv6 { + tunnelPeerIP = tunnelPeerIPs.IPv6 + } else { // Since broadcast is not supported in IPv6, ARP should happen only with IPv4 address, and ARP responder flows // only work for IPv4 addresses. flows = append(flows, c.arpResponderFlow(peerGatewayIP, cookie.Node)) } - if c.encapMode.NeedsEncapToPeer(tunnelPeerIP, c.nodeConfig.NodeTransportIPAddr) { - // tunnelPeerIP is the Node Internal Address. In a dual-stack setup, whether this address is an IPv4 address or an - // IPv6 one is decided by the address family of Node Internal Address. + if (!isIPv6 && c.encapMode.NeedsEncapToPeer(tunnelPeerIPs.IPv4, c.nodeConfig.NodeTransportIPv4Addr)) || + (isIPv6 && c.encapMode.NeedsEncapToPeer(tunnelPeerIPs.IPv6, c.nodeConfig.NodeTransportIPv6Addr)) { + // tunnelPeerIP is the Node Internal Address. In a dual-stack setup, one Node has 2 Node Internal + // Addresses (IPv4 and IPv6) . flows = append(flows, c.l3FwdFlowToRemote(localGatewayMAC, *peerPodCIDR, tunnelPeerIP, cookie.Node)) } else { flows = append(flows, c.l3FwdFlowToRemoteViaRouting(localGatewayMAC, remoteGatewayMAC, cookie.Node, tunnelPeerIP, peerPodCIDR)...) } } - if ipsecTunOFPort != 0 { // When IPSec tunnel is enabled, packets received from the remote Node are // input from the Node's IPSec tunnel port, not the default tunnel port. So, @@ -743,17 +748,15 @@ func (c *client) Initialize(roundInfo types.RoundInfo, nodeConfig *config.NodeCo } func (c *client) InstallExternalFlows() error { - nodeIP := c.nodeConfig.NodeIPAddr.IP localGatewayMAC := c.nodeConfig.GatewayConfig.MAC var flows []binding.Flow - if c.nodeConfig.PodIPv4CIDR != nil { - flows = c.externalFlows(nodeIP, *c.nodeConfig.PodIPv4CIDR, localGatewayMAC) + if c.nodeConfig.NodeIPv4Addr != nil && c.nodeConfig.PodIPv4CIDR != nil { + flows = c.externalFlows(c.nodeConfig.NodeIPv4Addr.IP, *c.nodeConfig.PodIPv4CIDR, localGatewayMAC) } - if c.nodeConfig.PodIPv6CIDR != nil { - flows = append(flows, c.externalFlows(nodeIP, *c.nodeConfig.PodIPv6CIDR, localGatewayMAC)...) + if c.nodeConfig.NodeIPv6Addr != nil && c.nodeConfig.PodIPv6CIDR != nil { + flows = append(flows, c.externalFlows(c.nodeConfig.NodeIPv6Addr.IP, *c.nodeConfig.PodIPv6CIDR, localGatewayMAC)...) } - if err := c.ofEntryOperations.AddAll(flows); err != nil { return fmt.Errorf("failed to install flows for external communication: %v", err) } diff --git a/pkg/agent/openflow/client_test.go b/pkg/agent/openflow/client_test.go index d2e4f3edd57..387ab70a059 100644 --- a/pkg/agent/openflow/client_test.go +++ b/pkg/agent/openflow/client_test.go @@ -34,6 +34,7 @@ import ( binding "antrea.io/antrea/pkg/ovs/openflow" ovsoftest "antrea.io/antrea/pkg/ovs/openflow/testing" "antrea.io/antrea/pkg/ovs/ovsconfig" + utilip "antrea.io/antrea/pkg/util/ip" ) const bridgeName = "dummy-br" @@ -54,10 +55,10 @@ var ( func installNodeFlows(ofClient Client, cacheKey string) (int, error) { hostName := cacheKey peerNodeIP := net.ParseIP("192.168.1.1") - peerConfig := map[*net.IPNet]net.IP{ + peerConfigs := map[*net.IPNet]net.IP{ ipNet: gwIP, } - err := ofClient.InstallNodeFlows(hostName, peerConfig, peerNodeIP, 0, nil) + err := ofClient.InstallNodeFlows(hostName, peerConfigs, &utilip.DualStackIPs{IPv4: peerNodeIP}, 0, nil) client := ofClient.(*client) fCacheI, ok := client.nodeFlowCache.Load(hostName) if ok { diff --git a/pkg/agent/openflow/pipeline_windows.go b/pkg/agent/openflow/pipeline_windows.go index 1f623165d92..a5a35dada4d 100644 --- a/pkg/agent/openflow/pipeline_windows.go +++ b/pkg/agent/openflow/pipeline_windows.go @@ -108,7 +108,7 @@ func (c *client) hostBridgeUplinkFlows(localSubnet net.IPNet, category cookie.Ca func (c *client) l3FwdFlowToRemoteViaRouting(localGatewayMAC net.HardwareAddr, remoteGatewayMAC net.HardwareAddr, category cookie.Category, peerIP net.IP, peerPodCIDR *net.IPNet) []binding.Flow { - if c.encapMode.NeedsDirectRoutingToPeer(peerIP, c.nodeConfig.NodeTransportIPAddr) && remoteGatewayMAC != nil { + if c.encapMode.NeedsDirectRoutingToPeer(peerIP, c.nodeConfig.NodeTransportIPv4Addr) && remoteGatewayMAC != nil { ipProto := getIPProtocol(peerIP) l3FwdTable := c.pipeline[l3ForwardingTable] // It enhances Windows Noencap mode performance by bypassing host network. diff --git a/pkg/agent/openflow/testing/mock_openflow.go b/pkg/agent/openflow/testing/mock_openflow.go index 917fca88551..fee6881e9b1 100644 --- a/pkg/agent/openflow/testing/mock_openflow.go +++ b/pkg/agent/openflow/testing/mock_openflow.go @@ -23,6 +23,7 @@ import ( config "antrea.io/antrea/pkg/agent/config" types "antrea.io/antrea/pkg/agent/types" openflow "antrea.io/antrea/pkg/ovs/openflow" + ip "antrea.io/antrea/pkg/util/ip" proxy "antrea.io/antrea/third_party/proxy" gomock "github.com/golang/mock/gomock" net "net" @@ -349,7 +350,7 @@ func (mr *MockClientMockRecorder) InstallLoadBalancerServiceFromOutsideFlows(arg } // InstallNodeFlows mocks base method -func (m *MockClient) InstallNodeFlows(arg0 string, arg1 map[*net.IPNet]net.IP, arg2 net.IP, arg3 uint32, arg4 net.HardwareAddr) error { +func (m *MockClient) InstallNodeFlows(arg0 string, arg1 map[*net.IPNet]net.IP, arg2 *ip.DualStackIPs, arg3 uint32, arg4 net.HardwareAddr) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "InstallNodeFlows", arg0, arg1, arg2, arg3, arg4) ret0, _ := ret[0].(error) diff --git a/pkg/agent/querier/querier_test.go b/pkg/agent/querier/querier_test.go index 2661398057b..6034d6ef581 100644 --- a/pkg/agent/querier/querier_test.go +++ b/pkg/agent/querier/querier_test.go @@ -74,9 +74,9 @@ func TestAgentQuerierGetAgentInfo(t *testing.T) { { name: "networkPolicyOnly-mode non-partial", nodeConfig: &config.NodeConfig{ - Name: "foo", - OVSBridge: "br-int", - NodeIPAddr: getIPNet("10.10.0.10"), + Name: "foo", + OVSBridge: "br-int", + NodeIPv4Addr: getIPNet("10.10.0.10"), }, apiPort: 10350, partial: false, @@ -120,11 +120,11 @@ func TestAgentQuerierGetAgentInfo(t *testing.T) { { name: "encap-mode non-partial", nodeConfig: &config.NodeConfig{ - Name: "foo", - OVSBridge: "br-int", - NodeIPAddr: getIPNet("10.10.0.10"), - PodIPv4CIDR: getIPNet("20.20.20.0/24"), - PodIPv6CIDR: getIPNet("2001:ab03:cd04:55ef::/64"), + Name: "foo", + OVSBridge: "br-int", + NodeIPv4Addr: getIPNet("10.10.0.10"), + PodIPv4CIDR: getIPNet("20.20.20.0/24"), + PodIPv6CIDR: getIPNet("2001:ab03:cd04:55ef::/64"), }, apiPort: 10350, partial: false, diff --git a/pkg/agent/route/route_linux.go b/pkg/agent/route/route_linux.go index 68aae961dc3..fe6e82d617f 100644 --- a/pkg/agent/route/route_linux.go +++ b/pkg/agent/route/route_linux.go @@ -35,6 +35,7 @@ import ( "antrea.io/antrea/pkg/agent/util" "antrea.io/antrea/pkg/agent/util/ipset" "antrea.io/antrea/pkg/agent/util/iptables" + "antrea.io/antrea/pkg/agent/util/sysctl" "antrea.io/antrea/pkg/ovs/ovsconfig" "antrea.io/antrea/pkg/util/env" ) @@ -130,6 +131,18 @@ func (c *Client) Initialize(nodeConfig *config.NodeConfig, done func()) error { return fmt.Errorf("failed to initialize ip routes: %v", err) } + // Ensure IPv6 forwarding is enabled if it is a dual-stack or IPv6-only cluster. + if c.nodeConfig.NodeIPv6Addr != nil { + sysctlFilename := "ipv6/conf/all/forwarding" + v, err := sysctl.GetSysctlNet(sysctlFilename) + if err != nil { + return fmt.Errorf("failed to read value of sysctl file: %s", sysctlFilename) + } + if v != 1 { + return fmt.Errorf("IPv6 forwarding is not enabled") + } + } + return nil } @@ -439,9 +452,17 @@ func (c *Client) restoreIptablesData(podCIDR *net.IPNet, podIPSet string, snatMa func (c *Client) initIPRoutes() error { if c.networkConfig.TrafficEncapMode.IsNetworkPolicyOnly() { gwLink := util.GetNetLink(c.nodeConfig.GatewayConfig.Name) - _, gwIP, _ := net.ParseCIDR(fmt.Sprintf("%s/32", c.nodeConfig.NodeTransportIPAddr.IP.String())) - if err := netlink.AddrReplace(gwLink, &netlink.Addr{IPNet: gwIP}); err != nil { - return fmt.Errorf("failed to add address %s to gw %s: %v", gwIP, gwLink.Attrs().Name, err) + if c.nodeConfig.NodeTransportIPv4Addr != nil { + _, gwIP, _ := net.ParseCIDR(fmt.Sprintf("%s/32", c.nodeConfig.NodeTransportIPv4Addr.IP.String())) + if err := netlink.AddrReplace(gwLink, &netlink.Addr{IPNet: gwIP}); err != nil { + return fmt.Errorf("failed to add address %s to gw %s: %v", gwIP, gwLink.Attrs().Name, err) + } + } + if c.nodeConfig.NodeTransportIPv6Addr != nil { + _, gwIP, _ := net.ParseCIDR(fmt.Sprintf("%s/128", c.nodeConfig.NodeTransportIPv6Addr.IP.String())) + if err := netlink.AddrReplace(gwLink, &netlink.Addr{IPNet: gwIP}); err != nil { + return fmt.Errorf("failed to add address %s to gw %s: %v", gwIP, gwLink.Attrs().Name, err) + } } } return nil @@ -566,6 +587,13 @@ func (c *Client) listIPv6NeighborsOnGateway() (map[string]*netlink.Neigh, error) // AddRoutes adds routes to a new podCIDR. It overrides the routes if they already exist. func (c *Client) AddRoutes(podCIDR *net.IPNet, nodeName string, nodeIP, nodeGwIP net.IP) error { + var nodeTransportIPAddr *net.IPNet + if podCIDR.IP.To4() == nil { + nodeTransportIPAddr = c.nodeConfig.NodeTransportIPv6Addr + } else { + nodeTransportIPAddr = c.nodeConfig.NodeTransportIPv4Addr + } + podCIDRStr := podCIDR.String() ipsetName := getIPSetName(podCIDR.IP) // Add this podCIDR to antreaPodIPSet so that packets to them won't be masqueraded when they leave the host. @@ -577,7 +605,7 @@ func (c *Client) AddRoutes(podCIDR *net.IPNet, nodeName string, nodeIP, nodeGwIP Dst: podCIDR, } var routes []*netlink.Route - if c.networkConfig.TrafficEncapMode.NeedsEncapToPeer(nodeIP, c.nodeConfig.NodeTransportIPAddr) { + if c.networkConfig.TrafficEncapMode.NeedsEncapToPeer(nodeIP, nodeTransportIPAddr) { if podCIDR.IP.To4() == nil { // "on-link" is not identified in IPv6 route entries, so split the configuration into 2 entries. routes = []*netlink.Route{ @@ -591,7 +619,7 @@ func (c *Client) AddRoutes(podCIDR *net.IPNet, nodeName string, nodeIP, nodeGwIP } route.LinkIndex = c.nodeConfig.GatewayConfig.LinkIndex route.Gw = nodeGwIP - } else if c.networkConfig.TrafficEncapMode.NeedsDirectRoutingToPeer(nodeIP, c.nodeConfig.NodeTransportIPAddr) { + } else if c.networkConfig.TrafficEncapMode.NeedsDirectRoutingToPeer(nodeIP, nodeTransportIPAddr) { // NoEncap traffic to Node on the same subnet. // Set the peerNodeIP as next hop. route.Gw = nodeIP diff --git a/pkg/agent/route/route_windows.go b/pkg/agent/route/route_windows.go index 605e51eadf3..c81a1d9f605 100644 --- a/pkg/agent/route/route_windows.go +++ b/pkg/agent/route/route_windows.go @@ -117,10 +117,10 @@ func (c *Client) AddRoutes(podCIDR *net.IPNet, nodeName string, peerNodeIP, peer DestinationSubnet: podCIDR, RouteMetric: util.DefaultMetric, } - if c.networkConfig.TrafficEncapMode.NeedsEncapToPeer(peerNodeIP, c.nodeConfig.NodeTransportIPAddr) { + if c.networkConfig.TrafficEncapMode.NeedsEncapToPeer(peerNodeIP, c.nodeConfig.NodeTransportIPv4Addr) { route.LinkIndex = c.nodeConfig.GatewayConfig.LinkIndex route.GatewayAddress = peerGwIP - } else if c.networkConfig.TrafficEncapMode.NeedsDirectRoutingToPeer(peerNodeIP, c.nodeConfig.NodeTransportIPAddr) { + } else if c.networkConfig.TrafficEncapMode.NeedsDirectRoutingToPeer(peerNodeIP, c.nodeConfig.NodeTransportIPv4Addr) { // NoEncap traffic to Node on the same subnet. // Set the peerNodeIP as next hop. route.LinkIndex = c.bridgeInfIndex diff --git a/pkg/agent/types/annotations.go b/pkg/agent/types/annotations.go index 4353eb11d9b..586a1b51b26 100644 --- a/pkg/agent/types/annotations.go +++ b/pkg/agent/types/annotations.go @@ -18,6 +18,6 @@ const ( // NodeMACAddressAnnotationKey represents the key of the Node's MAC address in the Annotations of the Node. NodeMACAddressAnnotationKey string = "node.antrea.io/mac-address" - // NodeTransportAddressAnnotationKey represents the key of the interface's IP address on which the Node transfers Pod traffic in the Annotations of the Node. - NodeTransportAddressAnnotationKey string = "node.antrea.io/transport-address" + // NodeTransportAddressAnnotationKey represents the key of the interface's IP addresses on which the Node transfers Pod traffic in the Annotations of the Node. + NodeTransportAddressAnnotationKey string = "node.antrea.io/transport-addresses" ) diff --git a/pkg/agent/util/net.go b/pkg/agent/util/net.go index 6b1ef244345..abaee70f683 100644 --- a/pkg/agent/util/net.go +++ b/pkg/agent/util/net.go @@ -22,6 +22,8 @@ import ( "io" "net" "strings" + + "antrea.io/antrea/pkg/util/ip" ) const ( @@ -106,47 +108,74 @@ func dialUnix(address string) (net.Conn, error) { return net.Dial("unix", address) } -// GetIPNetDeviceFromIP returns a local IP/mask and associated device from IP. -func GetIPNetDeviceFromIP(localIP net.IP) (*net.IPNet, *net.Interface, error) { +// GetIPNetDeviceFromIP returns local IPs/masks and associated device from IP. +func GetIPNetDeviceFromIP(localIPs *ip.DualStackIPs) (v4IPNet *net.IPNet, v6IPNet *net.IPNet, iface *net.Interface, err error) { linkList, err := net.Interfaces() if err != nil { - return nil, nil, err + return nil, nil, nil, err } - for _, link := range linkList { - addrList, err := link.Addrs() + // localIPs includes at most one IPv4 address and one IPv6 address. For each device in linkList, all its addresses + // are compared with IPs in localIPs. If found, the iface is set to the device and v4IPNet, v6IPNet are set to + // the matching addresses. + saveIface := func(current *net.Interface) error { + if iface != nil && iface.Index != current.Index { + return fmt.Errorf("IPs of localIPs should be on the same device") + } + iface = current + return nil + } + for i := range linkList { + addrList, err := linkList[i].Addrs() if err != nil { continue } for _, addr := range addrList { if ipNet, ok := addr.(*net.IPNet); ok { - if ipNet.IP.Equal(localIP) { - return ipNet, &link, nil + if ipNet.IP.Equal(localIPs.IPv4) { + if err := saveIface(&linkList[i]); err != nil { + return nil, nil, nil, err + } + v4IPNet = ipNet + } else if ipNet.IP.Equal(localIPs.IPv6) { + if err := saveIface(&linkList[i]); err != nil { + return nil, nil, nil, err + } + v6IPNet = ipNet } } } } - return nil, nil, fmt.Errorf("unable to find local IP and device") + if iface == nil { + return nil, nil, nil, fmt.Errorf("unable to find local IPs and device") + } + return v4IPNet, v6IPNet, iface, nil } -func GetIPNetDeviceByName(ifaceName string) (*net.IPNet, *net.Interface, error) { - link, err := net.InterfaceByName(ifaceName) +func GetIPNetDeviceByName(ifaceName string) (v4IPNet *net.IPNet, v6IPNet *net.IPNet, link *net.Interface, err error) { + link, err = net.InterfaceByName(ifaceName) if err != nil { - return nil, nil, err + return nil, nil, nil, err } addrList, err := link.Addrs() if err != nil { - return nil, nil, err + return nil, nil, nil, err } for _, addr := range addrList { if ipNet, ok := addr.(*net.IPNet); ok { if ipNet.IP.IsGlobalUnicast() { - return ipNet, link, nil + if ipNet.IP.To4() != nil { + v6IPNet = ipNet + } else { + v4IPNet = ipNet + } } } - continue } - return nil, nil, fmt.Errorf("unable to find local IP and device") + if v4IPNet != nil || v6IPNet != nil { + return v4IPNet, v6IPNet, link, nil + } + return nil, nil, nil, fmt.Errorf("unable to find local IP and device") } func GetIPv4Addr(ips []net.IP) net.IP { diff --git a/pkg/agent/util/net_test.go b/pkg/agent/util/net_test.go index 4042564bfeb..be4e3b5e012 100644 --- a/pkg/agent/util/net_test.go +++ b/pkg/agent/util/net_test.go @@ -19,6 +19,8 @@ import ( "net" "strings" "testing" + + "antrea.io/antrea/pkg/util/ip" ) func TestGenerateContainerInterfaceName(t *testing.T) { @@ -53,11 +55,12 @@ func TestGetDefaultLocalNodeAddr(t *testing.T) { t.Error(err) } defer conn.Close() - ip := conn.LocalAddr().(*net.UDPAddr).IP + localAddr := conn.LocalAddr().(*net.UDPAddr).IP - _, dev, err := GetIPNetDeviceFromIP(ip) + nodeIPs := &ip.DualStackIPs{IPv4: localAddr} + _, _, dev, err := GetIPNetDeviceFromIP(nodeIPs) if err != nil { t.Error(err) } - t.Logf("IP obtained %s, %v", ip, dev) + t.Logf("IP obtained %s, %v", localAddr, dev) } diff --git a/pkg/antctl/raw/helper.go b/pkg/antctl/raw/helper.go index 0a6b78edadf..67f8758f136 100644 --- a/pkg/antctl/raw/helper.go +++ b/pkg/antctl/raw/helper.go @@ -32,6 +32,7 @@ import ( controllerapiserver "antrea.io/antrea/pkg/apiserver" antrea "antrea.io/antrea/pkg/client/clientset/versioned" "antrea.io/antrea/pkg/client/clientset/versioned/scheme" + "antrea.io/antrea/pkg/util/ip" "antrea.io/antrea/pkg/util/k8s" ) @@ -97,12 +98,21 @@ func CreateAgentClientCfg(k8sClientset kubernetes.Interface, antreaClientset ant if agentInfo == nil { return nil, fmt.Errorf("no Antrea Agent found for Node name %s", nodeName) } - nodeIP, err := k8s.GetNodeAddr(node) + nodeIPs, err := k8s.GetNodeAddrs(node) if err != nil { return nil, fmt.Errorf("error when parsing IP of Node %s", nodeName) } cfg := rest.CopyConfig(cfgTmpl) - cfg.Host = fmt.Sprintf("https://%s", net.JoinHostPort(nodeIP.String(), fmt.Sprint(agentInfo.APIPort))) + + var nodeIP string + if nodeIPs.IPv4 != nil { + nodeIP = nodeIPs.IPv4.String() + } else if nodeIPs.IPv6 != nil { + nodeIP = nodeIPs.IPv6.String() + } else { + return nil, fmt.Errorf("there is no NodeIP on agent Node") + } + cfg.Host = fmt.Sprintf("https://%s", net.JoinHostPort(nodeIP, fmt.Sprint(agentInfo.APIPort))) return cfg, nil } @@ -116,13 +126,23 @@ func CreateControllerClientCfg(k8sClientset kubernetes.Interface, antreaClientse if err != nil { return nil, fmt.Errorf("error when searching the Node of the controller: %w", err) } - var controllerNodeIP net.IP - controllerNodeIP, err = k8s.GetNodeAddr(controllerNode) + var controllerNodeIPs *ip.DualStackIPs + controllerNodeIPs, err = k8s.GetNodeAddrs(controllerNode) if err != nil { return nil, fmt.Errorf("error when parsing controller IP: %w", err) } cfg := rest.CopyConfig(cfgTmpl) - cfg.Host = fmt.Sprintf("https://%s", net.JoinHostPort(controllerNodeIP.String(), fmt.Sprint(controllerInfo.APIPort))) + + var nodeIP string + if controllerNodeIPs.IPv4 != nil { + nodeIP = controllerNodeIPs.IPv4.String() + } else if controllerNodeIPs.IPv6 != nil { + nodeIP = controllerNodeIPs.IPv6.String() + } else { + return nil, fmt.Errorf("there is no NodeIP on controller Node") + } + + cfg.Host = fmt.Sprintf("https://%s", net.JoinHostPort(nodeIP, fmt.Sprint(controllerInfo.APIPort))) return cfg, nil } diff --git a/pkg/antctl/raw/supportbundle/command.go b/pkg/antctl/raw/supportbundle/command.go index 8bf2b007d8a..28381d61f99 100644 --- a/pkg/antctl/raw/supportbundle/command.go +++ b/pkg/antctl/raw/supportbundle/command.go @@ -43,6 +43,7 @@ import ( "antrea.io/antrea/pkg/antctl/runtime" systemv1beta1 "antrea.io/antrea/pkg/apis/system/v1beta1" antrea "antrea.io/antrea/pkg/client/clientset/versioned" + "antrea.io/antrea/pkg/util/ip" "antrea.io/antrea/pkg/util/k8s" ) @@ -300,13 +301,19 @@ func createAgentClients(k8sClientset kubernetes.Interface, antreaClientset antre if !ok { continue } - ip, err := k8s.GetNodeAddr(&node) + ips, err := k8s.GetNodeAddrs(&node) if err != nil { klog.Warningf("Error when parsing IP of Node %s", node.Name) continue } cfg := rest.CopyConfig(cfgTmpl) - cfg.Host = net.JoinHostPort(ip.String(), port) + var nodeIP string + if ips.IPv4 != nil { + nodeIP = ips.IPv4.String() + } else { + nodeIP = ips.IPv6.String() + } + cfg.Host = net.JoinHostPort(nodeIP, port) client, err := rest.RESTClientFor(cfg) if err != nil { klog.Warningf("Error when creating agent client for node: %s", node.Name) @@ -327,14 +334,20 @@ func createControllerClient(k8sClientset kubernetes.Interface, antreaClientset a if err != nil { return nil, fmt.Errorf("error when searching the Node of the controller: %w", err) } - var controllerNodeIP net.IP - controllerNodeIP, err = k8s.GetNodeAddr(controllerNode) + var controllerNodeIPs *ip.DualStackIPs + controllerNodeIPs, err = k8s.GetNodeAddrs(controllerNode) if err != nil { return nil, fmt.Errorf("error when parsing controllre IP: %w", err) } cfg := rest.CopyConfig(cfgTmpl) - cfg.Host = net.JoinHostPort(controllerNodeIP.String(), fmt.Sprint(controllerInfo.APIPort)) + var nodeIP string + if controllerNodeIPs.IPv4 != nil { + nodeIP = controllerNodeIPs.IPv4.String() + } else { + nodeIP = controllerNodeIPs.IPv6.String() + } + cfg.Host = net.JoinHostPort(nodeIP, fmt.Sprint(controllerInfo.APIPort)) controllerClient, err := rest.RESTClientFor(cfg) if err != nil { klog.Warningf("Error when creating controller client for node: %s", controllerInfo.NodeRef.Name) diff --git a/pkg/util/ip/ip.go b/pkg/util/ip/ip.go index ff0926eda23..444975aef4f 100644 --- a/pkg/util/ip/ip.go +++ b/pkg/util/ip/ip.go @@ -28,6 +28,15 @@ const ( V6BitLen = 8 * net.IPv6len ) +type DualStackIPs struct { + IPv4 net.IP + IPv6 net.IP +} + +func (ips DualStackIPs) Equal(x DualStackIPs) bool { + return ips.IPv4.Equal(x.IPv4) && ips.IPv6.Equal(x.IPv6) +} + // This function takes in one allow CIDR and multiple except CIDRs and gives diff CIDRs // in allowCIDR eliminating except CIDRs. It currently supports only IPv4. except CIDR input // can be changed. diff --git a/pkg/util/k8s/node.go b/pkg/util/k8s/node.go index 0f4b48796e2..56176d6f58a 100644 --- a/pkg/util/k8s/node.go +++ b/pkg/util/k8s/node.go @@ -19,29 +19,41 @@ import ( "net" v1 "k8s.io/api/core/v1" + + "antrea.io/antrea/pkg/util/ip" ) -// GetNodeAddr gets the available IP address of a Node. GetNodeAddr will first try to get the NodeInternalIP, then try +// GetNodeAddrs gets the available IP addresses of a Node. GetNodeAddrs will first try to get the NodeInternalIP, then try // to get the NodeExternalIP. -// Note: Although K8s supports dual-stack, there is only a single Internal address per Node because of issue ( -// kubernetes/kubernetes#91940 ). The Node might have multiple addresses after the issue is fixed, and one per address -// family. And we should change the return type at that time. -func GetNodeAddr(node *v1.Node) (net.IP, error) { - addresses := make(map[v1.NodeAddressType]string) +// If no error is returned, the returned DualStackIPs includes at least one IPv4 or IPv6 address. +func GetNodeAddrs(node *v1.Node) (*ip.DualStackIPs, error) { + addresses := make(map[v1.NodeAddressType][]string) for _, addr := range node.Status.Addresses { - addresses[addr.Type] = addr.Address + addresses[addr.Type] = append(addresses[addr.Type], addr.Address) } - var ipAddrStr string + var ipAddrStrs []string if internalIP, ok := addresses[v1.NodeInternalIP]; ok { - ipAddrStr = internalIP + ipAddrStrs = internalIP } else if externalIP, ok := addresses[v1.NodeExternalIP]; ok { - ipAddrStr = externalIP + ipAddrStrs = externalIP } else { return nil, fmt.Errorf("Node %s has neither external ip nor internal ip", node.Name) } - ipAddr := net.ParseIP(ipAddrStr) - if ipAddr == nil { - return nil, fmt.Errorf("<%v> is not a valid ip address", ipAddrStr) + if len(ipAddrStrs) == 0 { + return nil, fmt.Errorf("no IP is found for Node '%s'", node.Name) + } + + nodeAddrs := new(ip.DualStackIPs) + for i := range ipAddrStrs { + addr := net.ParseIP(ipAddrStrs[i]) + if addr == nil { + return nil, fmt.Errorf("'%s' is not a valid IP address", ipAddrStrs[i]) + } + if addr.To4() == nil { + nodeAddrs.IPv6 = addr + } else { + nodeAddrs.IPv4 = addr + } } - return ipAddr, nil + return nodeAddrs, nil } diff --git a/pkg/util/k8s/node_test.go b/pkg/util/k8s/node_test.go index 07362337a30..53d17a2ae74 100644 --- a/pkg/util/k8s/node_test.go +++ b/pkg/util/k8s/node_test.go @@ -22,13 +22,15 @@ import ( "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "antrea.io/antrea/pkg/util/ip" ) func TestGetNodeAddr(t *testing.T) { tests := []struct { name string node *corev1.Node - expectedAddr net.IP + expectedAddr *ip.DualStackIPs expectedErr error }{ { @@ -54,7 +56,7 @@ func TestGetNodeAddr(t *testing.T) { }, }, }, - expectedAddr: net.ParseIP("192.168.10.10"), + expectedAddr: &ip.DualStackIPs{IPv4: net.ParseIP("192.168.10.10")}, expectedErr: nil, }, { @@ -76,7 +78,7 @@ func TestGetNodeAddr(t *testing.T) { }, }, }, - expectedAddr: net.ParseIP("1.1.1.1"), + expectedAddr: &ip.DualStackIPs{IPv4: net.ParseIP("1.1.1.1")}, expectedErr: nil, }, { @@ -100,7 +102,7 @@ func TestGetNodeAddr(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - addr, err := GetNodeAddr(tt.node) + addr, err := GetNodeAddrs(tt.node) assert.Equal(t, tt.expectedErr, err) assert.Equal(t, tt.expectedAddr, addr) }) diff --git a/test/integration/agent/openflow_test.go b/test/integration/agent/openflow_test.go index e343658d789..67804385c48 100644 --- a/test/integration/agent/openflow_test.go +++ b/test/integration/agent/openflow_test.go @@ -41,6 +41,7 @@ import ( ofconfig "antrea.io/antrea/pkg/ovs/openflow" "antrea.io/antrea/pkg/ovs/ovsconfig" "antrea.io/antrea/pkg/ovs/ovsctl" + utilip "antrea.io/antrea/pkg/util/ip" antrearuntime "antrea.io/antrea/pkg/util/runtime" ofTestUtils "antrea.io/antrea/test/integration/ovs" k8sproxy "antrea.io/antrea/third_party/proxy" @@ -121,19 +122,33 @@ func TestConnectivityFlows(t *testing.T) { }() config := prepareConfiguration() - for _, f := range []func(t *testing.T, config *testConfig){ - testInitialize, - testInstallGatewayFlows, - testInstallServiceFlows, - testInstallTunnelFlows, - testInstallNodeFlows, - testInstallPodFlows, - testUninstallPodFlows, - testUninstallNodeFlows, - testExternalFlows, - } { - f(t, config) - } + t.Run("testInitialize", func(t *testing.T) { + testInitialize(t, config) + }) + t.Run("testInstallGatewayFlows", func(t *testing.T) { + testInstallGatewayFlows(t, config) + }) + t.Run("testInstallServiceFlows", func(t *testing.T) { + testInstallServiceFlows(t, config) + }) + t.Run("testInstallTunnelFlows", func(t *testing.T) { + testInstallTunnelFlows(t, config) + }) + t.Run("testInstallNodeFlows", func(t *testing.T) { + testInstallNodeFlows(t, config) + }) + t.Run("testInstallPodFlows", func(t *testing.T) { + testInstallPodFlows(t, config) + }) + t.Run("testUninstallPodFlows", func(t *testing.T) { + testUninstallPodFlows(t, config) + }) + t.Run("testUninstallNodeFlows", func(t *testing.T) { + testUninstallNodeFlows(t, config) + }) + t.Run("testExternalFlows", func(t *testing.T) { + testExternalFlows(t, config) + }) } func TestReplayFlowsConnectivityFlows(t *testing.T) { @@ -149,18 +164,27 @@ func TestReplayFlowsConnectivityFlows(t *testing.T) { }() config := prepareConfiguration() - for _, f := range []func(t *testing.T, config *testConfig){ - testInitialize, - testInstallGatewayFlows, - testInstallServiceFlows, - testInstallTunnelFlows, - testInstallNodeFlows, - testInstallPodFlows, - } { - f(t, config) - } - - testReplayFlows(t) + t.Run("testInitialize", func(t *testing.T) { + testInitialize(t, config) + }) + t.Run("testInstallGatewayFlows", func(t *testing.T) { + testInstallGatewayFlows(t, config) + }) + t.Run("testInstallServiceFlows", func(t *testing.T) { + testInstallServiceFlows(t, config) + }) + t.Run("testInstallTunnelFlows", func(t *testing.T) { + testInstallTunnelFlows(t, config) + }) + t.Run("testInstallNodeFlows", func(t *testing.T) { + testInstallNodeFlows(t, config) + }) + t.Run("testInstallPodFlows", func(t *testing.T) { + testInstallPodFlows(t, config) + }) + t.Run("testInstallPodFlows", func(t *testing.T) { + testReplayFlows(t) + }) } func TestReplayFlowsNetworkPolicyFlows(t *testing.T) { @@ -216,20 +240,20 @@ func TestReplayFlowsNetworkPolicyFlows(t *testing.T) { } func testExternalFlows(t *testing.T, config *testConfig) { - nodeIP := config.nodeConfig.NodeIPAddr.IP - var localSubnet *net.IPNet - if config.nodeConfig.PodIPv4CIDR != nil { - localSubnet = config.nodeConfig.PodIPv4CIDR - } else { - localSubnet = config.nodeConfig.PodIPv6CIDR - } - gwMAC := config.nodeConfig.GatewayConfig.MAC - if err := c.InstallExternalFlows(); err != nil { t.Errorf("Failed to install OpenFlow entries to allow Pod to communicate to the external addresses: %v", err) } - for _, tableFlow := range prepareExternalFlows(nodeIP, localSubnet, gwMAC) { - ofTestUtils.CheckFlowExists(t, ovsCtlClient, tableFlow.tableID, true, tableFlow.flows) + + gwMAC := config.nodeConfig.GatewayConfig.MAC + if config.nodeConfig.NodeIPv4Addr != nil && config.nodeConfig.PodIPv4CIDR != nil { + for _, tableFlow := range expectedExternalFlows(config.nodeConfig.NodeIPv4Addr.IP, config.nodeConfig.PodIPv4CIDR, gwMAC) { + ofTestUtils.CheckFlowExists(t, ovsCtlClient, tableFlow.tableID, true, tableFlow.flows) + } + } + if config.nodeConfig.NodeIPv6Addr != nil && config.nodeConfig.PodIPv6CIDR != nil { + for _, tableFlow := range expectedExternalFlows(config.nodeConfig.NodeIPv6Addr.IP, config.nodeConfig.PodIPv6CIDR, gwMAC) { + ofTestUtils.CheckFlowExists(t, ovsCtlClient, tableFlow.tableID, true, tableFlow.flows) + } } } @@ -287,10 +311,16 @@ func testInstallServiceFlows(t *testing.T, config *testConfig) { func testInstallNodeFlows(t *testing.T, config *testConfig) { gatewayConfig := config.nodeConfig.GatewayConfig for _, node := range config.peers { - peerConfig := map[*net.IPNet]net.IP{ + peerConfigs := map[*net.IPNet]net.IP{ &node.subnet: node.gateway, } - err := c.InstallNodeFlows(node.name, peerConfig, node.nodeAddress, 0, nil) + dsIPs := new(utilip.DualStackIPs) + if node.gateway.To4() == nil { + dsIPs.IPv6 = node.nodeAddress + } else { + dsIPs.IPv4 = node.nodeAddress + } + err := c.InstallNodeFlows(node.name, peerConfigs, dsIPs, 0, nil) if err != nil { t.Fatalf("Failed to install Openflow entries for node connectivity: %v", err) } @@ -464,17 +494,27 @@ func TestIPv6ConnectivityFlows(t *testing.T) { assert.Nil(t, err, fmt.Sprintf("Error while deleting OVS bridge: %v", err)) }() config := prepareIPv6Configuration() - for _, f := range []func(t *testing.T, config *testConfig){ - testInitialize, - testInstallNodeFlows, - testInstallPodFlows, - testInstallGatewayFlows, - testUninstallPodFlows, - testUninstallNodeFlows, - testExternalFlows, - } { - f(t, config) - } + t.Run("testInitialize", func(t *testing.T) { + testInitialize(t, config) + }) + t.Run("testInstallNodeFlows", func(t *testing.T) { + testInstallNodeFlows(t, config) + }) + t.Run("testInstallPodFlows", func(t *testing.T) { + testInstallPodFlows(t, config) + }) + t.Run("testInstallGatewayFlows", func(t *testing.T) { + testInstallGatewayFlows(t, config) + }) + t.Run("testUninstallPodFlows", func(t *testing.T) { + testUninstallPodFlows(t, config) + }) + t.Run("testUninstallNodeFlows", func(t *testing.T) { + testUninstallNodeFlows(t, config) + }) + t.Run("testExternalFlows", func(t *testing.T) { + testExternalFlows(t, config) + }) } type svcConfig struct { @@ -854,7 +894,7 @@ func prepareConfiguration() *testConfig { MAC: gwMAC, } nodeConfig := &config1.NodeConfig{ - NodeIPAddr: nodeSubnet, + NodeIPv4Addr: nodeSubnet, GatewayConfig: gatewayConfig, PodIPv4CIDR: podIPv4CIDR, } @@ -899,7 +939,7 @@ func prepareIPv6Configuration() *testConfig { MAC: gwMAC, } nodeConfig := &config1.NodeConfig{ - NodeIPAddr: nodeSubnet, + NodeIPv4Addr: nodeSubnet, GatewayConfig: gatewayConfig, PodIPv6CIDR: podIPv6CIDR, } @@ -1279,7 +1319,7 @@ func prepareIPNetAddresses(addresses []string) []types.Address { return ipAddresses } -func prepareExternalFlows(nodeIP net.IP, localSubnet *net.IPNet, gwMAC net.HardwareAddr) []expectTableFlows { +func expectedExternalFlows(nodeIP net.IP, localSubnet *net.IPNet, gwMAC net.HardwareAddr) []expectTableFlows { var ipProtoStr, nwDstFieldName string if localSubnet.IP.To4() != nil { ipProtoStr = "ip" diff --git a/test/integration/agent/route_test.go b/test/integration/agent/route_test.go index 7bd1ef8a39e..478b0da78ce 100644 --- a/test/integration/agent/route_test.go +++ b/test/integration/agent/route_test.go @@ -37,6 +37,7 @@ import ( "antrea.io/antrea/pkg/agent/util/ipset" "antrea.io/antrea/pkg/agent/util/iptables" "antrea.io/antrea/pkg/ovs/ovsconfig" + utilip "antrea.io/antrea/pkg/util/ip" ) func ExecOutputTrim(cmd string) (string, error) { @@ -48,14 +49,14 @@ func ExecOutputTrim(cmd string) (string, error) { } var ( - _, podCIDR, _ = net.ParseCIDR("10.10.10.0/24") - nodeIP, nodeIntf, _ = util.GetIPNetDeviceFromIP(func() net.IP { + _, podCIDR, _ = net.ParseCIDR("10.10.10.0/24") + nodeIPv4, _, nodeIntf, _ = util.GetIPNetDeviceFromIP(func() *utilip.DualStackIPs { conn, _ := net.Dial("udp", "8.8.8.8:80") defer conn.Close() - return conn.LocalAddr().(*net.UDPAddr).IP + return &utilip.DualStackIPs{IPv4: conn.LocalAddr().(*net.UDPAddr).IP} }()) nodeLink, _ = netlink.LinkByName(nodeIntf.Name) - localPeerIP = ip.NextIP(nodeIP.IP) + localPeerIP = ip.NextIP(nodeIPv4.IP) remotePeerIP = net.ParseIP("50.50.50.1") _, serviceCIDR, _ = net.ParseCIDR("200.200.0.0/16") gwIP = net.ParseIP("10.10.10.1") @@ -63,11 +64,11 @@ var ( gwName = "antrea-gw0" gwConfig = &config.GatewayConfig{IPv4: gwIP, MAC: gwMAC, Name: gwName} nodeConfig = &config.NodeConfig{ - Name: "test", - PodIPv4CIDR: podCIDR, - NodeIPAddr: nodeIP, - NodeTransportIPAddr: nodeIP, - GatewayConfig: gwConfig, + Name: "test", + PodIPv4CIDR: podCIDR, + NodeIPv4Addr: nodeIPv4, + NodeTransportIPv4Addr: nodeIPv4, + GatewayConfig: gwConfig, } ) @@ -549,7 +550,7 @@ func TestRouteTablePolicyOnly(t *testing.T) { gwIPOut, err := ExecOutputTrim(fmt.Sprintf("ip addr show %s", gwName)) assert.NoError(t, err) gwIP := net.IPNet{ - IP: nodeConfig.NodeIPAddr.IP, + IP: nodeConfig.NodeIPv4Addr.IP, Mask: net.CIDRMask(32, 32), } assert.Contains(t, gwIPOut, gwIP.String()) @@ -608,7 +609,7 @@ func TestIPv6RoutesAndNeighbors(t *testing.T) { Name: "test", PodIPv4CIDR: podCIDR, PodIPv6CIDR: ipv6Subnet, - NodeIPAddr: nodeIP, + NodeIPv4Addr: nodeIPv4, GatewayConfig: dualGWConfig, } err = routeClient.Initialize(dualNodeConfig, func() {}) diff --git a/test/integration/ovs/openflow_test_utils.go b/test/integration/ovs/openflow_test_utils.go index 006ec36a785..d29c54ce1ae 100644 --- a/test/integration/ovs/openflow_test_utils.go +++ b/test/integration/ovs/openflow_test_utils.go @@ -53,19 +53,19 @@ type ExpectFlow struct { func CheckFlowExists(t *testing.T, ovsCtlClient ovsctl.OVSCtlClient, tableID uint8, exist bool, flows []*ExpectFlow) []string { flowList, _ := OfctlDumpTableFlows(ovsCtlClient, tableID) - if exist { - for _, flow := range flows { - if !OfctlFlowMatch(flowList, tableID, flow) { - t.Errorf("Failed to install flow:\n%v\nExisting flows:\n%v", flow, flowList) - } + + for _, flow := range flows { + found := OfctlFlowMatch(flowList, tableID, flow) + if exist && !found { + t.Errorf("Failed to install flow:\n%v", flow) } - } else { - for _, flow := range flows { - if OfctlFlowMatch(flowList, tableID, flow) { - t.Errorf("Failed to uninstall flow:\n%v\nExisting flows:\n%v", flow, flowList) - } + if !exist && found { + t.Errorf("Failed to uninstall flow:\n%v", flow) } } + if t.Failed() { + t.Errorf("Existing flows:\n%v", flowList) + } return flowList }