From 34602a4b4be16353f2c3e99796030b2ec79b495b Mon Sep 17 00:00:00 2001 From: Zhecheng Li Date: Fri, 16 Jul 2021 19:11:41 +0800 Subject: [PATCH] [IPv6] Support no-encap mode in dual-stack setup * Extend single IP address to IPv4 and IPv6 ones for NodeAddr and NodeTransportAddr in nodeConfig * Refactor occurrences of Node IP address to adjust to 2 IP: agent initialization, node route * Still use one IP for IPv6/dual-stack not supported cases: Windows, NPL * Update tests Signed-off-by: Zhecheng Li Related: #2426 --- cmd/antrea-agent/agent.go | 2 +- pkg/agent/agent.go | 57 +++++--- pkg/agent/agent_linux.go | 2 +- pkg/agent/agent_test.go | 26 ++-- pkg/agent/agent_windows.go | 18 +-- pkg/agent/config/node_config.go | 22 +-- .../egress/ipassigner/ip_assigner_linux.go | 2 +- .../noderoute/node_route_controller.go | 126 +++++++++++++----- pkg/agent/controller/traceflow/packetin.go | 2 +- pkg/agent/memberlist/cluster.go | 2 +- pkg/agent/memberlist/cluster_test.go | 10 +- pkg/agent/openflow/client.go | 37 +++-- pkg/agent/openflow/pipeline_windows.go | 2 +- pkg/agent/querier/querier_test.go | 16 +-- pkg/agent/route/route_linux.go | 38 +++++- pkg/agent/route/route_windows.go | 4 +- pkg/agent/types/annotations.go | 4 +- pkg/agent/util/net.go | 49 ++++--- pkg/agent/util/net_test.go | 2 +- pkg/antctl/raw/helper.go | 16 ++- pkg/antctl/raw/supportbundle/command.go | 17 ++- pkg/util/k8s/node.go | 29 ++-- pkg/util/k8s/node_test.go | 8 +- test/integration/agent/openflow_test.go | 6 +- test/integration/agent/route_test.go | 22 +-- 25 files changed, 330 insertions(+), 189 deletions(-) diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go index 1007b33985e..03008cd8544 100644 --- a/cmd/antrea-agent/agent.go +++ b/cmd/antrea-agent/agent.go @@ -228,7 +228,7 @@ func run(o *Options) error { var egressController *egress.EgressController if features.DefaultFeatureGate.Enabled(features.Egress) { egressController, err = egress.NewEgressController( - ofClient, antreaClientProvider, crdClient, ifaceStore, routeClient, nodeConfig.Name, nodeConfig.NodeIPAddr.IP, + ofClient, antreaClientProvider, crdClient, ifaceStore, routeClient, nodeConfig.Name, nodeConfig.NodeIPv4Addr.IP, o.config.ClusterMembershipPort, egressInformer, nodeInformer, externalIPPoolInformer, ) if err != nil { diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index fe5733d6565..eff22088517 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -21,6 +21,7 @@ import ( "net" "os" "strconv" + "strings" "sync" "time" @@ -533,15 +534,17 @@ func (i *Initializer) configureGatewayInterface(gatewayIface *interfacestore.Int i.nodeConfig.GatewayConfig = &config.GatewayConfig{Name: i.hostGateway, MAC: gwMAC} gatewayIface.MAC = gwMAC + gatewayIface.IPs = []net.IP{} if i.networkConfig.TrafficEncapMode.IsNetworkPolicyOnly() { // Assign IP to gw as required by SpoofGuard. - // NodeIPAddr can be either IPv4 or IPv6. - if i.nodeConfig.NodeIPAddr.IP.To4() != nil { - i.nodeConfig.GatewayConfig.IPv4 = i.nodeConfig.NodeTransportIPAddr.IP - } else { - i.nodeConfig.GatewayConfig.IPv6 = i.nodeConfig.NodeTransportIPAddr.IP + if i.nodeConfig.NodeIPv4Addr != nil { + i.nodeConfig.GatewayConfig.IPv4 = i.nodeConfig.NodeTransportIPv4Addr.IP + gatewayIface.IPs = append(gatewayIface.IPs, i.nodeConfig.NodeTransportIPv4Addr.IP) + } + if i.nodeConfig.NodeIPv6Addr != nil { + i.nodeConfig.GatewayConfig.IPv6 = i.nodeConfig.NodeTransportIPv6Addr.IP + gatewayIface.IPs = append(gatewayIface.IPs, i.nodeConfig.NodeTransportIPv6Addr.IP) } - gatewayIface.IPs = []net.IP{i.nodeConfig.NodeTransportIPAddr.IP} // No need to assign local CIDR to gw0 because local CIDR is not managed by Antrea return nil } @@ -645,26 +648,34 @@ func (i *Initializer) initNodeLocalConfig() error { return err } - var nodeIPAddr, transportIPAddr *net.IPNet + var nodeIPv4Addr, nodeIPv6Addr, transportIPv4Addr, transportIPv6Addr *net.IPNet var localIntf *net.Interface // Find the interface configured with Node IP and use it for Pod traffic. - ipAddr, err := k8s.GetNodeAddr(node) + ipAddrs, err := k8s.GetNodeAddrs(node) if err != nil { - return fmt.Errorf("failed to obtain local IP address from K8s: %w", err) + return fmt.Errorf("failed to obtain local IP addresses from K8s: %w", err) } - nodeIPAddr, localIntf, err = getIPNetDeviceFromIP(ipAddr) + nodeIPv4Addr, nodeIPv6Addr, localIntf, err = getIPNetDeviceFromIP(ipAddrs) if err != nil { - return fmt.Errorf("failed to get local IPNet device with IP %v: %v", ipAddr, err) + return fmt.Errorf("failed to get local IPNet device with IP %v: %v", ipAddrs, err) } - transportIPAddr = nodeIPAddr + transportIPv4Addr = nodeIPv4Addr + transportIPv6Addr = nodeIPv6Addr if i.networkConfig.TransportIface != "" { // Find the configured transport interface, and update its IP address in Node's annotation. - transportIPAddr, localIntf, err = getTransportIPNetDeviceByName(i.networkConfig.TransportIface, i.ovsBridge) + transportIPv4Addr, transportIPv6Addr, localIntf, err = getTransportIPNetDeviceByName(i.networkConfig.TransportIface, i.ovsBridge) if err != nil { return fmt.Errorf("failed to get local IPNet device with transport interface %s: %v", i.networkConfig.TransportIface, err) } - klog.InfoS("Updating Node transport address annotation") - if err := i.patchNodeAnnotations(nodeName, types.NodeTransportAddressAnnotationKey, transportIPAddr.IP.String()); err != nil { + klog.InfoS("Updating Node transport addresses annotation") + var ips []string + if transportIPv4Addr != nil { + ips = append(ips, transportIPv4Addr.IP.String()) + } + if transportIPv6Addr != nil { + ips = append(ips, transportIPv6Addr.IP.String()) + } + if err := i.patchNodeAnnotations(nodeName, types.NodeTransportAddressAnnotationKey, strings.Join(ips, ",")); err != nil { return err } } else { @@ -686,12 +697,14 @@ func (i *Initializer) initNodeLocalConfig() error { } i.nodeConfig = &config.NodeConfig{ - Name: nodeName, - OVSBridge: i.ovsBridge, - DefaultTunName: defaultTunInterfaceName, - NodeIPAddr: nodeIPAddr, - NodeTransportIPAddr: transportIPAddr, - UplinkNetConfig: new(config.AdapterNetConfig)} + Name: nodeName, + OVSBridge: i.ovsBridge, + DefaultTunName: defaultTunInterfaceName, + NodeIPv4Addr: nodeIPv4Addr, + NodeIPv6Addr: nodeIPv6Addr, + NodeTransportIPv4Addr: transportIPv4Addr, + NodeTransportIPv6Addr: transportIPv6Addr, + UplinkNetConfig: new(config.AdapterNetConfig)} mtu, err := i.getNodeMTU(localIntf) if err != nil { @@ -864,7 +877,7 @@ func (i *Initializer) getNodeMTU(localIntf *net.Interface) (int, error) { } else if i.networkConfig.TunnelType == ovsconfig.GRETunnel { mtu -= config.GREOverhead } - if i.nodeConfig.NodeIPAddr.IP.To4() == nil { + if i.nodeConfig.NodeIPv6Addr != nil { mtu -= config.IPv6ExtraOverhead } } diff --git a/pkg/agent/agent_linux.go b/pkg/agent/agent_linux.go index 20e399cb4c8..1efcad7c23e 100644 --- a/pkg/agent/agent_linux.go +++ b/pkg/agent/agent_linux.go @@ -43,6 +43,6 @@ func (i *Initializer) getTunnelPortLocalIP() net.IP { return nil } -func GetTransportIPNetDeviceByName(ifaceName string, ovsBridgeName string) (*net.IPNet, *net.Interface, error) { +func GetTransportIPNetDeviceByName(ifaceName string, ovsBridgeName string) (*net.IPNet, *net.IPNet, *net.Interface, error) { return util.GetIPNetDeviceByName(ifaceName) } diff --git a/pkg/agent/agent_test.go b/pkg/agent/agent_test.go index b5741a52aca..204382d2fdd 100644 --- a/pkg/agent/agent_test.go +++ b/pkg/agent/agent_test.go @@ -278,14 +278,14 @@ func TestInitNodeLocalConfig(t *testing.T) { client := fake.NewSimpleClientset(node) ifaceStore := interfacestore.NewInterfaceStore() expectedNodeConfig := config.NodeConfig{ - Name: nodeName, - OVSBridge: ovsBridge, - DefaultTunName: defaultTunInterfaceName, - PodIPv4CIDR: podCIDR, - NodeIPAddr: nodeIPNet, - NodeTransportIPAddr: nodeIPNet, - NodeMTU: tt.expectedMTU, - UplinkNetConfig: new(config.AdapterNetConfig), + Name: nodeName, + OVSBridge: ovsBridge, + DefaultTunName: defaultTunInterfaceName, + PodIPv4CIDR: podCIDR, + NodeIPv4Addr: nodeIPNet, + NodeTransportIPv4Addr: nodeIPNet, + NodeMTU: tt.expectedMTU, + UplinkNetConfig: new(config.AdapterNetConfig), } initializer := &Initializer{ @@ -300,7 +300,7 @@ func TestInitNodeLocalConfig(t *testing.T) { } if tt.transportInterface != nil { initializer.networkConfig.TransportIface = tt.transportInterface.iface.Name - expectedNodeConfig.NodeTransportIPAddr = tt.transportInterface.ipNet + expectedNodeConfig.NodeTransportIPv4Addr = tt.transportInterface.ipNet defer mockGetTransportIPNetDeviceByName(tt.transportInterface.ipNet, tt.transportInterface.iface)() } defer mockGetIPNetDeviceFromIP(nodeIPNet, ipDevice)() @@ -317,8 +317,8 @@ func TestInitNodeLocalConfig(t *testing.T) { func mockGetIPNetDeviceFromIP(ipNet *net.IPNet, ipDevice *net.Interface) func() { prevGetIPNetDeviceFromIP := getIPNetDeviceFromIP - getIPNetDeviceFromIP = func(localIP net.IP) (*net.IPNet, *net.Interface, error) { - return ipNet, ipDevice, nil + getIPNetDeviceFromIP = func(localIP []net.IP) (*net.IPNet, *net.IPNet, *net.Interface, error) { + return ipNet, nil, ipDevice, nil } return func() { getIPNetDeviceFromIP = prevGetIPNetDeviceFromIP } } @@ -330,8 +330,8 @@ func mockNodeNameEnv(name string) func() { func mockGetTransportIPNetDeviceByName(ipNet *net.IPNet, ipDevice *net.Interface) func() { prevGetIPNetDeviceByName := getTransportIPNetDeviceByName - getTransportIPNetDeviceByName = func(ifName, brName string) (*net.IPNet, *net.Interface, error) { - return ipNet, ipDevice, nil + getTransportIPNetDeviceByName = func(ifName, brName string) (*net.IPNet, *net.IPNet, *net.Interface, error) { + return ipNet, nil, ipDevice, nil } return func() { getTransportIPNetDeviceByName = prevGetIPNetDeviceByName } } diff --git a/pkg/agent/agent_windows.go b/pkg/agent/agent_windows.go index 8c4d441423b..c526b9f431c 100644 --- a/pkg/agent/agent_windows.go +++ b/pkg/agent/agent_windows.go @@ -49,7 +49,7 @@ func (i *Initializer) prepareHostNetwork() error { // Get uplink network configuration. The uplink interface is the one used for transporting Pod traffic across Nodes. // Use the interface specified with "transportInterface" in the configuration if configured, otherwise the interface // configured with NodeIP is used as uplink. - _, adapter, err := util.GetIPNetDeviceFromIP(i.nodeConfig.NodeTransportIPAddr.IP) + _, _, adapter, err := util.GetIPNetDeviceFromIP([]net.IP{i.nodeConfig.NodeTransportIPv4Addr.IP}) if err != nil { return err } @@ -68,7 +68,7 @@ func (i *Initializer) prepareHostNetwork() error { } i.nodeConfig.UplinkNetConfig.Name = adapter.Name i.nodeConfig.UplinkNetConfig.MAC = adapter.HardwareAddr - i.nodeConfig.UplinkNetConfig.IP = i.nodeConfig.NodeTransportIPAddr + i.nodeConfig.UplinkNetConfig.IP = i.nodeConfig.NodeTransportIPv4Addr i.nodeConfig.UplinkNetConfig.Index = adapter.Index defaultGW, err := util.GetDefaultGatewayByInterfaceIndex(adapter.Index) if err != nil { @@ -96,7 +96,7 @@ func (i *Initializer) prepareHostNetwork() error { if subnetCIDR == nil { return fmt.Errorf("failed to find valid IPv4 PodCIDR") } - return util.PrepareHNSNetwork(subnetCIDR, i.nodeConfig.NodeTransportIPAddr, adapter) + return util.PrepareHNSNetwork(subnetCIDR, i.nodeConfig.NodeTransportIPv4Addr, adapter) } // prepareOVSBridge adds local port and uplink to ovs bridge. @@ -213,7 +213,7 @@ func (i *Initializer) initHostNetworkFlows() error { // getTunnelLocalIP returns local_ip of tunnel port func (i *Initializer) getTunnelPortLocalIP() net.IP { - return i.nodeConfig.NodeTransportIPAddr.IP + return i.nodeConfig.NodeTransportIPv4Addr.IP } // saveHostRoutes saves routes which are configured on uplink interface before @@ -271,17 +271,17 @@ func (i *Initializer) restoreHostRoutes() error { return nil } -func GetTransportIPNetDeviceByName(ifaceName string, ovsBridgeName string) (*net.IPNet, *net.Interface, error) { +func GetTransportIPNetDeviceByName(ifaceName string, ovsBridgeName string) (*net.IPNet, *net.IPNet, *net.Interface, error) { // Find transport Interface in the order: ifaceName -> "vEthernet (ifaceName)" -> br-int. Return immediately if // an interface using the specified name exists. Using "vEthernet (ifaceName)" or br-int is for restart agent case. for _, name := range []string{ifaceName, fmt.Sprintf("vEthernet (%s)", ifaceName), ovsBridgeName} { - ipNet, link, err := util.GetIPNetDeviceByName(name) + ipNet, _, link, err := util.GetIPNetDeviceByName(name) if err == nil { - return ipNet, link, nil + return ipNet, nil, link, nil } if !strings.Contains(err.Error(), "no such network interface") { - return nil, nil, err + return nil, nil, nil, err } } - return nil, nil, fmt.Errorf("unable to find local IP and device") + return nil, nil, nil, fmt.Errorf("unable to find local IP and device") } diff --git a/pkg/agent/config/node_config.go b/pkg/agent/config/node_config.go index 750b4ba2ee3..2089d359f9a 100644 --- a/pkg/agent/config/node_config.go +++ b/pkg/agent/config/node_config.go @@ -83,10 +83,14 @@ type NodeConfig struct { // The CIDR block from where to allocate IPv6 address to Pod. // It's nil for the net workPolicyOnly trafficEncapMode which doesn't do IPAM. PodIPv6CIDR *net.IPNet - // The Node's IP used in Kubernetes. It has the network mask information. - NodeIPAddr *net.IPNet - // The IP on the Node's transport interface. It is used for tunneling or routing the Pod traffic across Nodes. - NodeTransportIPAddr *net.IPNet + // The Node's IPv4 address used in Kubernetes. It has the network mask information. + NodeIPv4Addr *net.IPNet + // The Node's IPv6 address used in Kubernetes. It has the network mask information. + NodeIPv6Addr *net.IPNet + // The IPv4 address on the Node's transport interface. It is used for tunneling or routing the Pod traffic across Nodes. + NodeTransportIPv4Addr *net.IPNet + // The IPv6 address on the Node's transport interface. It is used for tunneling or routing the Pod traffic across Nodes. + NodeTransportIPv6Addr *net.IPNet // Set either via defaultMTU config in antrea.yaml or auto discovered. // Auto discovery will use MTU value of the Node's primary interface. // For Encap and Hybrid mode, Node MTU will be adjusted to account for encap header. @@ -98,8 +102,8 @@ type NodeConfig struct { } func (n *NodeConfig) String() string { - return fmt.Sprintf("NodeName: %s, OVSBridge: %s, PodIPv4CIDR: %s, PodIPv6CIDR: %s, NodeIP: %s, TransportIP: %s, Gateway: %s", - n.Name, n.OVSBridge, n.PodIPv4CIDR, n.PodIPv6CIDR, n.NodeIPAddr, n.NodeTransportIPAddr, n.GatewayConfig) + return fmt.Sprintf("NodeName: %s, OVSBridge: %s, PodIPv4CIDR: %s, PodIPv6CIDR: %s, NodeIPv4: %s, NodeIPv6: %s, TransportIPv4: %s, TransportIPv6: %s, Gateway: %s", + n.Name, n.OVSBridge, n.PodIPv4CIDR, n.PodIPv6CIDR, n.NodeIPv4Addr, n.NodeIPv6Addr, n.NodeTransportIPv4Addr, n.NodeTransportIPv6Addr, n.GatewayConfig) } // User provided network configuration parameters. @@ -112,15 +116,13 @@ type NetworkConfig struct { } // IsIPv4Enabled returns true if the cluster network supports IPv4. -// TODO: support dual-stack in networkPolicyOnly mode. func IsIPv4Enabled(nodeConfig *NodeConfig, trafficEncapMode TrafficEncapModeType) bool { return nodeConfig.PodIPv4CIDR != nil || - (trafficEncapMode.IsNetworkPolicyOnly() && nodeConfig.NodeIPAddr.IP.To4() != nil) + (trafficEncapMode.IsNetworkPolicyOnly() && nodeConfig.NodeIPv4Addr != nil) } // IsIPv6Enabled returns true if the cluster network supports IPv6. -// TODO: support dual-stack in networkPolicyOnly mode. func IsIPv6Enabled(nodeConfig *NodeConfig, trafficEncapMode TrafficEncapModeType) bool { return nodeConfig.PodIPv6CIDR != nil || - (trafficEncapMode.IsNetworkPolicyOnly() && nodeConfig.NodeIPAddr.IP.To4() == nil) + (trafficEncapMode.IsNetworkPolicyOnly() && nodeConfig.NodeIPv6Addr != nil) } diff --git a/pkg/agent/controller/egress/ipassigner/ip_assigner_linux.go b/pkg/agent/controller/egress/ipassigner/ip_assigner_linux.go index 0870e13f1da..4f2ef042092 100644 --- a/pkg/agent/controller/egress/ipassigner/ip_assigner_linux.go +++ b/pkg/agent/controller/egress/ipassigner/ip_assigner_linux.go @@ -49,7 +49,7 @@ type ipAssigner struct { // NewIPAssigner returns an *ipAssigner. func NewIPAssigner(nodeIPAddr net.IP, dummyDeviceName string) (*ipAssigner, error) { - _, egressInterface, err := util.GetIPNetDeviceFromIP(nodeIPAddr) + _, _, egressInterface, err := util.GetIPNetDeviceFromIP([]net.IP{nodeIPAddr}) if err != nil { return nil, fmt.Errorf("get IPNetDevice from ip %v error: %+v", nodeIPAddr, err) } diff --git a/pkg/agent/controller/noderoute/node_route_controller.go b/pkg/agent/controller/noderoute/node_route_controller.go index 0dd33bbe3a0..96087de4069 100644 --- a/pkg/agent/controller/noderoute/node_route_controller.go +++ b/pkg/agent/controller/noderoute/node_route_controller.go @@ -17,6 +17,7 @@ package noderoute import ( "fmt" "net" + "strings" "time" "github.com/containernetworking/plugins/pkg/ip" @@ -135,7 +136,7 @@ func nodeRouteInfoPodCIDRIndexFunc(obj interface{}) ([]string, error) { type nodeRouteInfo struct { nodeName string podCIDRs []*net.IPNet - nodeIP net.IP + nodeIPs []net.IP gatewayIP []net.IP nodeMAC net.HardwareAddr } @@ -215,17 +216,26 @@ func (c *Controller) removeStaleTunnelPorts() error { continue } - peerNodeIP, err := k8s.GetNodeAddr(node) + peerNodeIPs, err := k8s.GetNodeAddrs(node) if err != nil { klog.Errorf("Failed to retrieve IP address of Node %s: %v", node.Name, err) continue } + var peerNodeIPv4, peerNodeIPv6 net.IP + for _, ip := range peerNodeIPs { + if ip.To4() == nil { + peerNodeIPv6 = ip + } else { + peerNodeIPv4 = ip + } + } ifaceID := util.GenerateNodeTunnelInterfaceKey(node.Name) validConfiguration := interfaceConfig.PSK == c.networkConfig.IPSecPSK && - interfaceConfig.RemoteIP.Equal(peerNodeIP) && interfaceConfig.TunnelInterfaceConfig.Type == c.networkConfig.TunnelType - if validConfiguration { + v4RemoteIP := peerNodeIPv4 != nil && interfaceConfig.RemoteIP.Equal(peerNodeIPv4) + v6RemoteIP := peerNodeIPv6 != nil && interfaceConfig.RemoteIP.Equal(peerNodeIPv6) + if validConfiguration && (v4RemoteIP || v6RemoteIP) { desiredInterfaces[ifaceID] = true } } @@ -408,20 +418,35 @@ func (c *Controller) deleteNodeRoute(nodeName string) error { return nil } +func isIPsEqual(l0 []net.IP, l1 []net.IP) bool { + if len(l0) != len(l1) { + return false + } + l0Map := make(map[string]bool) + for _, ip := range l0 { + l0Map[ip.String()] = true + } + for _, ip := range l1 { + if !l0Map[ip.String()] { + return false + } + } + return true +} + func (c *Controller) addNodeRoute(nodeName string, node *corev1.Node) error { // It is only for Windows Noencap mode to get Node MAC. peerNodeMAC, err := getNodeMAC(node) if err != nil { return fmt.Errorf("error when retrieving MAC of Node %s: %v", nodeName, err) } - peerNodeIP, err := c.getNodeTransportAddress(node) + peerTransportNodeIPs, err := c.getNodeTransportAddrs(node) if err != nil { return err } nrInfo, installed, _ := c.installedNodes.GetByKey(nodeName) - - if installed && nrInfo.(*nodeRouteInfo).nodeMAC.String() == peerNodeMAC.String() && nrInfo.(*nodeRouteInfo).nodeIP.Equal(peerNodeIP) { + if installed && nrInfo.(*nodeRouteInfo).nodeMAC.String() == peerNodeMAC.String() && isIPsEqual(peerTransportNodeIPs, nrInfo.(*nodeRouteInfo).nodeIPs) { // Route is already added for this Node and both Node MAC and transport IP are not changed. return nil } @@ -434,7 +459,7 @@ func (c *Controller) addNodeRoute(nodeName string, node *corev1.Node) error { klog.Infof("Adding routes and flows to Node %s, podCIDRs: %v, addresses: %v", nodeName, podCIDRStrs, node.Status.Addresses) - var podCIDRs []*net.IPNet + var peerPodCIDRs []*net.IPNet peerConfig := make(map[*net.IPNet]net.IP, len(podCIDRStrs)) for _, podCIDR := range podCIDRStrs { klog.Infof("Adding routes and flows to Node %s, podCIDR: %s, addresses: %v", @@ -471,31 +496,53 @@ func (c *Controller) addNodeRoute(nodeName string, node *corev1.Node) error { } peerGatewayIP := ip.NextIP(peerPodCIDRAddr) peerConfig[peerPodCIDR] = peerGatewayIP - podCIDRs = append(podCIDRs, peerPodCIDR) + peerPodCIDRs = append(peerPodCIDRs, peerPodCIDR) } - ipsecTunOFPort := int32(0) - if c.networkConfig.EnableIPSecTunnel { - // Create a separate tunnel port for the Node, as OVS IPSec monitor needs to - // read PSK and remote IP from the Node's tunnel interface to create IPSec - // security policies. - if ipsecTunOFPort, err = c.createIPSecTunnelPort(nodeName, peerNodeIP); err != nil { - return err - } + peerNodeIPs, err := k8s.GetNodeAddrs(node) + if err != nil { + klog.Errorf("Failed to retrieve IP addresses of Node %s: %v", nodeName, err) + return nil } - err = c.ofClient.InstallNodeFlows( - nodeName, - peerConfig, - peerNodeIP, - uint32(ipsecTunOFPort), - peerNodeMAC) - if err != nil { - return fmt.Errorf("failed to install flows to Node %s: %v", nodeName, err) + for _, peerNodeIP := range peerNodeIPs { + ipsecTunOFPort := int32(0) + if c.networkConfig.EnableIPSecTunnel { + // Create a separate tunnel port for the Node, as OVS IPSec monitor needs to + // read PSK and remote IP from the Node's tunnel interface to create IPSec + // security policies. + if ipsecTunOFPort, err = c.createIPSecTunnelPort(nodeName, peerNodeIP); err != nil { + return err + } + } + + err = c.ofClient.InstallNodeFlows( + nodeName, + peerConfig, + peerNodeIP, + uint32(ipsecTunOFPort), + peerNodeMAC) + if err != nil { + return fmt.Errorf("failed to install flows to Node %s: %v", nodeName, err) + } } var peerGatewayIPs []net.IP + var peerNodeIPv4, peerNodeIPv6 net.IP + for _, ip := range peerNodeIPs { + if ip.To4() == nil { + peerNodeIPv6 = ip + } else { + peerNodeIPv4 = ip + } + } for peerPodCIDR, peerGatewayIP := range peerConfig { + var peerNodeIP net.IP + if peerGatewayIP.To4() == nil { + peerNodeIP = peerNodeIPv6 + } else { + peerNodeIP = peerNodeIPv4 + } if err := c.routeClient.AddRoutes(peerPodCIDR, nodeName, peerNodeIP, peerGatewayIP); err != nil { return err } @@ -503,11 +550,12 @@ func (c *Controller) addNodeRoute(nodeName string, node *corev1.Node) error { } c.installedNodes.Add(&nodeRouteInfo{ nodeName: nodeName, - podCIDRs: podCIDRs, - nodeIP: peerNodeIP, + podCIDRs: peerPodCIDRs, + nodeIPs: peerNodeIPs, gatewayIP: peerGatewayIPs, nodeMAC: peerNodeMAC, }) + return err } @@ -656,23 +704,27 @@ func getNodeMAC(node *corev1.Node) (net.HardwareAddr, error) { return mac, nil } -func (c *Controller) getNodeTransportAddress(node *corev1.Node) (net.IP, error) { +func (c *Controller) getNodeTransportAddrs(node *corev1.Node) ([]net.IP, error) { + var transportAddrs []net.IP if c.networkConfig.TransportIface != "" { - transportAddrStr := node.Annotations[types.NodeTransportAddressAnnotationKey] - if transportAddrStr != "" { - peerNodeAddr := net.ParseIP(transportAddrStr) - if peerNodeAddr == nil { - return nil, fmt.Errorf("invalid annotation for transport-address on Node %s: %s", node.Name, transportAddrStr) + transportAddrsStr := node.Annotations[types.NodeTransportAddressAnnotationKey] + if transportAddrsStr != "" { + for _, addr := range strings.Split(transportAddrsStr, ",") { + peerNodeAddr := net.ParseIP(addr) + if peerNodeAddr == nil { + return nil, fmt.Errorf("invalid annotation for transport-address on Node %s: %s", node.Name, transportAddrsStr) + } + transportAddrs = append(transportAddrs, peerNodeAddr) } - return peerNodeAddr, nil + return transportAddrs, nil } klog.InfoS("Transport address is not found, using NodeIP instead") } // Use NodeIP if the transport IP address is not set or not found. - peerNodeIP, err := k8s.GetNodeAddr(node) + peerNodeIPs, err := k8s.GetNodeAddrs(node) if err != nil { - klog.ErrorS(err, "Failed to retrieve Node IP address", "node", node.Name) + klog.ErrorS(err, "Failed to retrieve Node IP addresses", "node", node.Name) return nil, err } - return peerNodeIP, nil + return peerNodeIPs, nil } diff --git a/pkg/agent/controller/traceflow/packetin.go b/pkg/agent/controller/traceflow/packetin.go index 474baecde6f..f791203d1fd 100644 --- a/pkg/agent/controller/traceflow/packetin.go +++ b/pkg/agent/controller/traceflow/packetin.go @@ -231,7 +231,7 @@ func (c *Controller) parsePacketIn(pktIn *ofctrl.PacketIn) (*crdv1alpha1.Tracefl if tableID == uint8(openflow.L2ForwardingOutTable) { ob := new(crdv1alpha1.Observation) tunnelDstIP := "" - isIPv6 := c.nodeConfig.NodeIPAddr.IP.To4() == nil + isIPv6 := c.nodeConfig.NodeIPv6Addr != nil if match := getMatchTunnelDstField(matchers, isIPv6); match != nil { tunnelDstIP, err = getTunnelDstValue(match) if err != nil { diff --git a/pkg/agent/memberlist/cluster.go b/pkg/agent/memberlist/cluster.go index 27b38ba4a4f..dcea8b12507 100644 --- a/pkg/agent/memberlist/cluster.go +++ b/pkg/agent/memberlist/cluster.go @@ -252,7 +252,7 @@ func (c *Cluster) enqueueExternalIPPool(obj interface{}) { // newClusterMember gets the Node's IP and returns a cluster member ":" // representing that Node in the memberlist cluster. func (c *Cluster) newClusterMember(node *corev1.Node) (string, error) { - nodeAddr, err := k8s.GetNodeAddr(node) + nodeAddr, err := k8s.GetNodeAddrs(node) if err != nil { return "", fmt.Errorf("obtain IP address from K8s Node failed: %v", err) } diff --git a/pkg/agent/memberlist/cluster_test.go b/pkg/agent/memberlist/cluster_test.go index f08d5688e04..f5f45b8670f 100644 --- a/pkg/agent/memberlist/cluster_test.go +++ b/pkg/agent/memberlist/cluster_test.go @@ -58,7 +58,7 @@ func newFakeCluster(nodeConfig *config.NodeConfig, stopCh <-chan struct{}, i int crdInformerFactory := crdinformers.NewSharedInformerFactory(crdClient, 0) ipPoolInformer := crdInformerFactory.Crd().V1alpha2().ExternalIPPools() - cluster, err := NewCluster(port, nodeConfig.NodeIPAddr.IP, nodeConfig.Name, nodeInformer, ipPoolInformer) + cluster, err := NewCluster(port, nodeConfig.NodeIPv4Addr.IP, nodeConfig.Name, nodeInformer, ipPoolInformer) if err != nil { return nil, err } @@ -137,8 +137,8 @@ func TestCluster_Run(t *testing.T) { for i, tCase := range testCases { t.Run(tCase.name, func(t *testing.T) { nodeConfig := &config.NodeConfig{ - Name: localNodeName, - NodeIPAddr: &net.IPNet{IP: net.IPv4(127, 0, 0, 1), Mask: net.IPv4Mask(255, 255, 255, 255)}, + Name: localNodeName, + NodeIPv4Addr: &net.IPNet{IP: net.IPv4(127, 0, 0, 1), Mask: net.IPv4Mask(255, 255, 255, 255)}, } stopCh := make(chan struct{}) defer close(stopCh) @@ -177,8 +177,8 @@ func TestCluster_RunClusterEvents(t *testing.T) { nodeName := "localNodeName" nodeConfig := &config.NodeConfig{ - Name: nodeName, - NodeIPAddr: &net.IPNet{IP: net.IPv4(127, 0, 0, 1)}, + Name: nodeName, + NodeIPv4Addr: &net.IPNet{IP: net.IPv4(127, 0, 0, 1)}, } localNode := &v1.Node{ ObjectMeta: metav1.ObjectMeta{Name: nodeName}, diff --git a/pkg/agent/openflow/client.go b/pkg/agent/openflow/client.go index 247e587428b..7651f6af859 100644 --- a/pkg/agent/openflow/client.go +++ b/pkg/agent/openflow/client.go @@ -390,7 +390,7 @@ func (c *client) InstallNodeFlows(hostname string, // only work for IPv4 addresses. flows = append(flows, c.arpResponderFlow(peerGatewayIP, cookie.Node)) } - if c.encapMode.NeedsEncapToPeer(tunnelPeerIP, c.nodeConfig.NodeTransportIPAddr) { + if c.encapMode.NeedsEncapToPeer(tunnelPeerIP, c.nodeConfig.NodeTransportIPv4Addr) || c.encapMode.NeedsEncapToPeer(tunnelPeerIP, c.nodeConfig.NodeTransportIPv6Addr) { // tunnelPeerIP is the Node Internal Address. In a dual-stack setup, whether this address is an IPv4 address or an // IPv6 one is decided by the address family of Node Internal Address. flows = append(flows, c.l3FwdFlowToRemote(localGatewayMAC, *peerPodCIDR, tunnelPeerIP, cookie.Node)) @@ -724,21 +724,34 @@ func (c *client) Initialize(roundInfo types.RoundInfo, nodeConfig *config.NodeCo } func (c *client) InstallExternalFlows() error { - nodeIP := c.nodeConfig.NodeIPAddr.IP - localGatewayMAC := c.nodeConfig.GatewayConfig.MAC + install := func(nodeIP net.IP) error { + localGatewayMAC := c.nodeConfig.GatewayConfig.MAC - var flows []binding.Flow - if c.nodeConfig.PodIPv4CIDR != nil { - flows = c.externalFlows(nodeIP, *c.nodeConfig.PodIPv4CIDR, localGatewayMAC) - } - if c.nodeConfig.PodIPv6CIDR != nil { - flows = append(flows, c.externalFlows(nodeIP, *c.nodeConfig.PodIPv6CIDR, localGatewayMAC)...) + var flows []binding.Flow + if c.nodeConfig.PodIPv4CIDR != nil { + flows = c.externalFlows(nodeIP, *c.nodeConfig.PodIPv4CIDR, localGatewayMAC) + } + if c.nodeConfig.PodIPv6CIDR != nil { + flows = append(flows, c.externalFlows(nodeIP, *c.nodeConfig.PodIPv6CIDR, localGatewayMAC)...) + } + + if err := c.ofEntryOperations.AddAll(flows); err != nil { + return fmt.Errorf("failed to install flows for external communication: %v", err) + } + c.hostNetworkingFlows = append(c.hostNetworkingFlows, flows...) + return nil } - if err := c.ofEntryOperations.AddAll(flows); err != nil { - return fmt.Errorf("failed to install flows for external communication: %v", err) + if c.nodeConfig.NodeIPv4Addr != nil { + if err := install(c.nodeConfig.NodeIPv4Addr.IP); err != nil { + return err + } + } + if c.nodeConfig.NodeIPv6Addr != nil { + if err := install(c.nodeConfig.NodeIPv6Addr.IP); err != nil { + return err + } } - c.hostNetworkingFlows = append(c.hostNetworkingFlows, flows...) return nil } diff --git a/pkg/agent/openflow/pipeline_windows.go b/pkg/agent/openflow/pipeline_windows.go index 1f623165d92..a5a35dada4d 100644 --- a/pkg/agent/openflow/pipeline_windows.go +++ b/pkg/agent/openflow/pipeline_windows.go @@ -108,7 +108,7 @@ func (c *client) hostBridgeUplinkFlows(localSubnet net.IPNet, category cookie.Ca func (c *client) l3FwdFlowToRemoteViaRouting(localGatewayMAC net.HardwareAddr, remoteGatewayMAC net.HardwareAddr, category cookie.Category, peerIP net.IP, peerPodCIDR *net.IPNet) []binding.Flow { - if c.encapMode.NeedsDirectRoutingToPeer(peerIP, c.nodeConfig.NodeTransportIPAddr) && remoteGatewayMAC != nil { + if c.encapMode.NeedsDirectRoutingToPeer(peerIP, c.nodeConfig.NodeTransportIPv4Addr) && remoteGatewayMAC != nil { ipProto := getIPProtocol(peerIP) l3FwdTable := c.pipeline[l3ForwardingTable] // It enhances Windows Noencap mode performance by bypassing host network. diff --git a/pkg/agent/querier/querier_test.go b/pkg/agent/querier/querier_test.go index 2661398057b..6034d6ef581 100644 --- a/pkg/agent/querier/querier_test.go +++ b/pkg/agent/querier/querier_test.go @@ -74,9 +74,9 @@ func TestAgentQuerierGetAgentInfo(t *testing.T) { { name: "networkPolicyOnly-mode non-partial", nodeConfig: &config.NodeConfig{ - Name: "foo", - OVSBridge: "br-int", - NodeIPAddr: getIPNet("10.10.0.10"), + Name: "foo", + OVSBridge: "br-int", + NodeIPv4Addr: getIPNet("10.10.0.10"), }, apiPort: 10350, partial: false, @@ -120,11 +120,11 @@ func TestAgentQuerierGetAgentInfo(t *testing.T) { { name: "encap-mode non-partial", nodeConfig: &config.NodeConfig{ - Name: "foo", - OVSBridge: "br-int", - NodeIPAddr: getIPNet("10.10.0.10"), - PodIPv4CIDR: getIPNet("20.20.20.0/24"), - PodIPv6CIDR: getIPNet("2001:ab03:cd04:55ef::/64"), + Name: "foo", + OVSBridge: "br-int", + NodeIPv4Addr: getIPNet("10.10.0.10"), + PodIPv4CIDR: getIPNet("20.20.20.0/24"), + PodIPv6CIDR: getIPNet("2001:ab03:cd04:55ef::/64"), }, apiPort: 10350, partial: false, diff --git a/pkg/agent/route/route_linux.go b/pkg/agent/route/route_linux.go index 06cfca5b32d..452175d5b41 100644 --- a/pkg/agent/route/route_linux.go +++ b/pkg/agent/route/route_linux.go @@ -35,6 +35,7 @@ import ( "antrea.io/antrea/pkg/agent/util" "antrea.io/antrea/pkg/agent/util/ipset" "antrea.io/antrea/pkg/agent/util/iptables" + "antrea.io/antrea/pkg/agent/util/sysctl" "antrea.io/antrea/pkg/ovs/ovsconfig" "antrea.io/antrea/pkg/util/env" ) @@ -130,6 +131,18 @@ func (c *Client) Initialize(nodeConfig *config.NodeConfig, done func()) error { return fmt.Errorf("failed to initialize ip routes: %v", err) } + // Ensure IPv6 forwarding is enabled if it is a dual-stack or IPv6-only cluster. + if c.nodeConfig.NodeIPv6Addr != nil { + sysctlFilename := "ipv6/conf/all/forwarding" + v, err := sysctl.GetSysctlNet(sysctlFilename) + if err != nil { + return fmt.Errorf("failed to read value of sysctl file: %s", sysctlFilename) + } + if v != 1 { + return fmt.Errorf("IPv6 forwarding is not enabled") + } + } + return nil } @@ -438,9 +451,17 @@ func (c *Client) restoreIptablesData(podCIDR *net.IPNet, podIPSet string, snatMa func (c *Client) initIPRoutes() error { if c.networkConfig.TrafficEncapMode.IsNetworkPolicyOnly() { gwLink := util.GetNetLink(c.nodeConfig.GatewayConfig.Name) - _, gwIP, _ := net.ParseCIDR(fmt.Sprintf("%s/32", c.nodeConfig.NodeTransportIPAddr.IP.String())) - if err := netlink.AddrReplace(gwLink, &netlink.Addr{IPNet: gwIP}); err != nil { - return fmt.Errorf("failed to add address %s to gw %s: %v", gwIP, gwLink.Attrs().Name, err) + if c.nodeConfig.NodeTransportIPv4Addr != nil { + _, gwIP, _ := net.ParseCIDR(fmt.Sprintf("%s/32", c.nodeConfig.NodeTransportIPv4Addr.IP.String())) + if err := netlink.AddrReplace(gwLink, &netlink.Addr{IPNet: gwIP}); err != nil { + return fmt.Errorf("failed to add address %s to gw %s: %v", gwIP, gwLink.Attrs().Name, err) + } + } + if c.nodeConfig.NodeTransportIPv6Addr != nil { + _, gwIP, _ := net.ParseCIDR(fmt.Sprintf("%s/128", c.nodeConfig.NodeTransportIPv6Addr.IP.String())) + if err := netlink.AddrReplace(gwLink, &netlink.Addr{IPNet: gwIP}); err != nil { + return fmt.Errorf("failed to add address %s to gw %s: %v", gwIP, gwLink.Attrs().Name, err) + } } } return nil @@ -565,6 +586,13 @@ func (c *Client) listIPv6NeighborsOnGateway() (map[string]*netlink.Neigh, error) // AddRoutes adds routes to a new podCIDR. It overrides the routes if they already exist. func (c *Client) AddRoutes(podCIDR *net.IPNet, nodeName string, nodeIP, nodeGwIP net.IP) error { + var nodeTransportIPAddr *net.IPNet + if podCIDR.IP.To4() == nil { + nodeTransportIPAddr = c.nodeConfig.NodeTransportIPv6Addr + } else { + nodeTransportIPAddr = c.nodeConfig.NodeTransportIPv4Addr + } + podCIDRStr := podCIDR.String() ipsetName := getIPSetName(podCIDR.IP) // Add this podCIDR to antreaPodIPSet so that packets to them won't be masqueraded when they leave the host. @@ -576,7 +604,7 @@ func (c *Client) AddRoutes(podCIDR *net.IPNet, nodeName string, nodeIP, nodeGwIP Dst: podCIDR, } var routes []*netlink.Route - if c.networkConfig.TrafficEncapMode.NeedsEncapToPeer(nodeIP, c.nodeConfig.NodeTransportIPAddr) { + if c.networkConfig.TrafficEncapMode.NeedsEncapToPeer(nodeIP, nodeTransportIPAddr) { if podCIDR.IP.To4() == nil { // "on-link" is not identified in IPv6 route entries, so split the configuration into 2 entries. routes = []*netlink.Route{ @@ -590,7 +618,7 @@ func (c *Client) AddRoutes(podCIDR *net.IPNet, nodeName string, nodeIP, nodeGwIP } route.LinkIndex = c.nodeConfig.GatewayConfig.LinkIndex route.Gw = nodeGwIP - } else if c.networkConfig.TrafficEncapMode.NeedsDirectRoutingToPeer(nodeIP, c.nodeConfig.NodeTransportIPAddr) { + } else if c.networkConfig.TrafficEncapMode.NeedsDirectRoutingToPeer(nodeIP, nodeTransportIPAddr) { // NoEncap traffic to Node on the same subnet. // Set the peerNodeIP as next hop. route.Gw = nodeIP diff --git a/pkg/agent/route/route_windows.go b/pkg/agent/route/route_windows.go index 605e51eadf3..c81a1d9f605 100644 --- a/pkg/agent/route/route_windows.go +++ b/pkg/agent/route/route_windows.go @@ -117,10 +117,10 @@ func (c *Client) AddRoutes(podCIDR *net.IPNet, nodeName string, peerNodeIP, peer DestinationSubnet: podCIDR, RouteMetric: util.DefaultMetric, } - if c.networkConfig.TrafficEncapMode.NeedsEncapToPeer(peerNodeIP, c.nodeConfig.NodeTransportIPAddr) { + if c.networkConfig.TrafficEncapMode.NeedsEncapToPeer(peerNodeIP, c.nodeConfig.NodeTransportIPv4Addr) { route.LinkIndex = c.nodeConfig.GatewayConfig.LinkIndex route.GatewayAddress = peerGwIP - } else if c.networkConfig.TrafficEncapMode.NeedsDirectRoutingToPeer(peerNodeIP, c.nodeConfig.NodeTransportIPAddr) { + } else if c.networkConfig.TrafficEncapMode.NeedsDirectRoutingToPeer(peerNodeIP, c.nodeConfig.NodeTransportIPv4Addr) { // NoEncap traffic to Node on the same subnet. // Set the peerNodeIP as next hop. route.LinkIndex = c.bridgeInfIndex diff --git a/pkg/agent/types/annotations.go b/pkg/agent/types/annotations.go index 4353eb11d9b..586a1b51b26 100644 --- a/pkg/agent/types/annotations.go +++ b/pkg/agent/types/annotations.go @@ -18,6 +18,6 @@ const ( // NodeMACAddressAnnotationKey represents the key of the Node's MAC address in the Annotations of the Node. NodeMACAddressAnnotationKey string = "node.antrea.io/mac-address" - // NodeTransportAddressAnnotationKey represents the key of the interface's IP address on which the Node transfers Pod traffic in the Annotations of the Node. - NodeTransportAddressAnnotationKey string = "node.antrea.io/transport-address" + // NodeTransportAddressAnnotationKey represents the key of the interface's IP addresses on which the Node transfers Pod traffic in the Annotations of the Node. + NodeTransportAddressAnnotationKey string = "node.antrea.io/transport-addresses" ) diff --git a/pkg/agent/util/net.go b/pkg/agent/util/net.go index 9fcd7fabc1e..26706c141a0 100644 --- a/pkg/agent/util/net.go +++ b/pkg/agent/util/net.go @@ -100,47 +100,66 @@ func dialUnix(address string) (net.Conn, error) { return net.Dial("unix", address) } -// GetIPNetDeviceFromIP returns a local IP/mask and associated device from IP. -func GetIPNetDeviceFromIP(localIP net.IP) (*net.IPNet, *net.Interface, error) { +// GetIPNetDeviceFromIP returns local IPs/masks and associated device from IP. +func GetIPNetDeviceFromIP(localIPs []net.IP) (v4IPNet *net.IPNet, v6IPNet *net.IPNet, iface *net.Interface, err error) { linkList, err := net.Interfaces() if err != nil { - return nil, nil, err + return nil, nil, nil, err } - for _, link := range linkList { - addrList, err := link.Addrs() + for i := range linkList { + addrList, err := linkList[i].Addrs() if err != nil { continue } for _, addr := range addrList { if ipNet, ok := addr.(*net.IPNet); ok { - if ipNet.IP.Equal(localIP) { - return ipNet, &link, nil + for _, ip := range localIPs { + if ipNet.IP.Equal(ip) { + iface = &linkList[i] + if ip.To4() == nil { + v6IPNet = ipNet + } else { + v4IPNet = ipNet + } + } } } } + if iface != nil { + break + } + } + if iface == nil { + return nil, nil, nil, fmt.Errorf("unable to find local IPs and device") } - return nil, nil, fmt.Errorf("unable to find local IP and device") + return v4IPNet, v6IPNet, iface, nil } -func GetIPNetDeviceByName(ifaceName string) (*net.IPNet, *net.Interface, error) { - link, err := net.InterfaceByName(ifaceName) +func GetIPNetDeviceByName(ifaceName string) (v4IPNet *net.IPNet, v6IPNet *net.IPNet, link *net.Interface, err error) { + link, err = net.InterfaceByName(ifaceName) if err != nil { - return nil, nil, err + return nil, nil, nil, err } addrList, err := link.Addrs() if err != nil { - return nil, nil, err + return nil, nil, nil, err } for _, addr := range addrList { if ipNet, ok := addr.(*net.IPNet); ok { if ipNet.IP.IsGlobalUnicast() { - return ipNet, link, nil + if ipNet.IP.To4() != nil { + v6IPNet = ipNet + } else { + v4IPNet = ipNet + } } } - continue } - return nil, nil, fmt.Errorf("unable to find local IP and device") + if v4IPNet != nil || v6IPNet != nil { + return v4IPNet, v6IPNet, link, nil + } + return nil, nil, nil, fmt.Errorf("unable to find local IP and device") } func GetIPv4Addr(ips []net.IP) net.IP { diff --git a/pkg/agent/util/net_test.go b/pkg/agent/util/net_test.go index 4042564bfeb..2884cdc30a1 100644 --- a/pkg/agent/util/net_test.go +++ b/pkg/agent/util/net_test.go @@ -55,7 +55,7 @@ func TestGetDefaultLocalNodeAddr(t *testing.T) { defer conn.Close() ip := conn.LocalAddr().(*net.UDPAddr).IP - _, dev, err := GetIPNetDeviceFromIP(ip) + _, _, dev, err := GetIPNetDeviceFromIP([]net.IP{ip}) if err != nil { t.Error(err) } diff --git a/pkg/antctl/raw/helper.go b/pkg/antctl/raw/helper.go index 0a6b78edadf..da7b3ebe537 100644 --- a/pkg/antctl/raw/helper.go +++ b/pkg/antctl/raw/helper.go @@ -97,12 +97,15 @@ func CreateAgentClientCfg(k8sClientset kubernetes.Interface, antreaClientset ant if agentInfo == nil { return nil, fmt.Errorf("no Antrea Agent found for Node name %s", nodeName) } - nodeIP, err := k8s.GetNodeAddr(node) + nodeIPs, err := k8s.GetNodeAddrs(node) if err != nil { return nil, fmt.Errorf("error when parsing IP of Node %s", nodeName) } cfg := rest.CopyConfig(cfgTmpl) - cfg.Host = fmt.Sprintf("https://%s", net.JoinHostPort(nodeIP.String(), fmt.Sprint(agentInfo.APIPort))) + if len(nodeIPs) == 0 { + return nil, fmt.Errorf("there is no NodeIP on agent Node") + } + cfg.Host = fmt.Sprintf("https://%s", net.JoinHostPort(nodeIPs[0].String(), fmt.Sprint(agentInfo.APIPort))) return cfg, nil } @@ -116,13 +119,16 @@ func CreateControllerClientCfg(k8sClientset kubernetes.Interface, antreaClientse if err != nil { return nil, fmt.Errorf("error when searching the Node of the controller: %w", err) } - var controllerNodeIP net.IP - controllerNodeIP, err = k8s.GetNodeAddr(controllerNode) + var controllerNodeIPs []net.IP + controllerNodeIPs, err = k8s.GetNodeAddrs(controllerNode) if err != nil { return nil, fmt.Errorf("error when parsing controller IP: %w", err) } cfg := rest.CopyConfig(cfgTmpl) - cfg.Host = fmt.Sprintf("https://%s", net.JoinHostPort(controllerNodeIP.String(), fmt.Sprint(controllerInfo.APIPort))) + if len(controllerNodeIPs) == 0 { + return nil, fmt.Errorf("there is no NodeIP on controller Node") + } + cfg.Host = fmt.Sprintf("https://%s", net.JoinHostPort(controllerNodeIPs[0].String(), fmt.Sprint(controllerInfo.APIPort))) return cfg, nil } diff --git a/pkg/antctl/raw/supportbundle/command.go b/pkg/antctl/raw/supportbundle/command.go index 5a9ade8fb98..ba707166a97 100644 --- a/pkg/antctl/raw/supportbundle/command.go +++ b/pkg/antctl/raw/supportbundle/command.go @@ -300,13 +300,17 @@ func createAgentClients(k8sClientset kubernetes.Interface, antreaClientset antre if !ok { continue } - ip, err := k8s.GetNodeAddr(&node) + ips, err := k8s.GetNodeAddrs(&node) if err != nil { klog.Warningf("Error when parsing IP of Node %s", node.Name) continue } cfg := rest.CopyConfig(cfgTmpl) - cfg.Host = net.JoinHostPort(ip.String(), port) + if len(ips) == 0 { + klog.Warningf("There is no Node Addresses") + continue + } + cfg.Host = net.JoinHostPort(ips[0].String(), port) client, err := rest.RESTClientFor(cfg) if err != nil { klog.Warningf("Error when creating agent client for node: %s", node.Name) @@ -327,14 +331,17 @@ func createControllerClient(k8sClientset kubernetes.Interface, antreaClientset a if err != nil { return nil, fmt.Errorf("error when searching the Node of the controller: %w", err) } - var controllerNodeIP net.IP - controllerNodeIP, err = k8s.GetNodeAddr(controllerNode) + var controllerNodeIPs []net.IP + controllerNodeIPs, err = k8s.GetNodeAddrs(controllerNode) if err != nil { return nil, fmt.Errorf("error when parsing controllre IP: %w", err) } cfg := rest.CopyConfig(cfgTmpl) - cfg.Host = net.JoinHostPort(controllerNodeIP.String(), fmt.Sprint(controllerInfo.APIPort)) + if len(controllerNodeIPs) == 0 { + return nil, fmt.Errorf("error there is no NodeIP") + } + cfg.Host = net.JoinHostPort(controllerNodeIPs[0].String(), fmt.Sprint(controllerInfo.APIPort)) controllerClient, err := rest.RESTClientFor(cfg) if err != nil { klog.Warningf("Error when creating controller client for node: %s", controllerInfo.NodeRef.Name) diff --git a/pkg/util/k8s/node.go b/pkg/util/k8s/node.go index 0f4b48796e2..38520205ddb 100644 --- a/pkg/util/k8s/node.go +++ b/pkg/util/k8s/node.go @@ -21,27 +21,28 @@ import ( v1 "k8s.io/api/core/v1" ) -// GetNodeAddr gets the available IP address of a Node. GetNodeAddr will first try to get the NodeInternalIP, then try +// GetNodeAddrs gets the available IP addresses of a Node. GetNodeAddrs will first try to get the NodeInternalIP, then try // to get the NodeExternalIP. -// Note: Although K8s supports dual-stack, there is only a single Internal address per Node because of issue ( -// kubernetes/kubernetes#91940 ). The Node might have multiple addresses after the issue is fixed, and one per address -// family. And we should change the return type at that time. -func GetNodeAddr(node *v1.Node) (net.IP, error) { - addresses := make(map[v1.NodeAddressType]string) +func GetNodeAddrs(node *v1.Node) ([]net.IP, error) { + addresses := make(map[v1.NodeAddressType][]string) for _, addr := range node.Status.Addresses { - addresses[addr.Type] = addr.Address + addresses[addr.Type] = append(addresses[addr.Type], addr.Address) } - var ipAddrStr string + var ipAddrStrs []string if internalIP, ok := addresses[v1.NodeInternalIP]; ok { - ipAddrStr = internalIP + ipAddrStrs = internalIP } else if externalIP, ok := addresses[v1.NodeExternalIP]; ok { - ipAddrStr = externalIP + ipAddrStrs = externalIP } else { return nil, fmt.Errorf("Node %s has neither external ip nor internal ip", node.Name) } - ipAddr := net.ParseIP(ipAddrStr) - if ipAddr == nil { - return nil, fmt.Errorf("<%v> is not a valid ip address", ipAddrStr) + var nodeAddrs []net.IP + for i := range ipAddrStrs { + addr := net.ParseIP(ipAddrStrs[i]) + if addr == nil { + return nil, fmt.Errorf("'%s' is not a valid IP address", ipAddrStrs[i]) + } + nodeAddrs = append(nodeAddrs, addr) } - return ipAddr, nil + return nodeAddrs, nil } diff --git a/pkg/util/k8s/node_test.go b/pkg/util/k8s/node_test.go index 07362337a30..87935f2be40 100644 --- a/pkg/util/k8s/node_test.go +++ b/pkg/util/k8s/node_test.go @@ -28,7 +28,7 @@ func TestGetNodeAddr(t *testing.T) { tests := []struct { name string node *corev1.Node - expectedAddr net.IP + expectedAddr []net.IP expectedErr error }{ { @@ -54,7 +54,7 @@ func TestGetNodeAddr(t *testing.T) { }, }, }, - expectedAddr: net.ParseIP("192.168.10.10"), + expectedAddr: []net.IP{net.ParseIP("192.168.10.10")}, expectedErr: nil, }, { @@ -76,7 +76,7 @@ func TestGetNodeAddr(t *testing.T) { }, }, }, - expectedAddr: net.ParseIP("1.1.1.1"), + expectedAddr: []net.IP{net.ParseIP("1.1.1.1")}, expectedErr: nil, }, { @@ -100,7 +100,7 @@ func TestGetNodeAddr(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - addr, err := GetNodeAddr(tt.node) + addr, err := GetNodeAddrs(tt.node) assert.Equal(t, tt.expectedErr, err) assert.Equal(t, tt.expectedAddr, addr) }) diff --git a/test/integration/agent/openflow_test.go b/test/integration/agent/openflow_test.go index 8adc762d2d6..b004a4e0430 100644 --- a/test/integration/agent/openflow_test.go +++ b/test/integration/agent/openflow_test.go @@ -216,7 +216,7 @@ func TestReplayFlowsNetworkPolicyFlows(t *testing.T) { } func testExternalFlows(t *testing.T, config *testConfig) { - nodeIP := config.nodeConfig.NodeIPAddr.IP + nodeIP := config.nodeConfig.NodeIPv4Addr.IP var localSubnet *net.IPNet if config.nodeConfig.PodIPv4CIDR != nil { localSubnet = config.nodeConfig.PodIPv4CIDR @@ -855,7 +855,7 @@ func prepareConfiguration() *testConfig { MAC: gwMAC, } nodeConfig := &config1.NodeConfig{ - NodeIPAddr: nodeSubnet, + NodeIPv4Addr: nodeSubnet, GatewayConfig: gatewayConfig, PodIPv4CIDR: podIPv4CIDR, } @@ -900,7 +900,7 @@ func prepareIPv6Configuration() *testConfig { MAC: gwMAC, } nodeConfig := &config1.NodeConfig{ - NodeIPAddr: nodeSubnet, + NodeIPv4Addr: nodeSubnet, GatewayConfig: gatewayConfig, PodIPv6CIDR: podIPv6CIDR, } diff --git a/test/integration/agent/route_test.go b/test/integration/agent/route_test.go index 7bd1ef8a39e..b1f5edf4e90 100644 --- a/test/integration/agent/route_test.go +++ b/test/integration/agent/route_test.go @@ -48,14 +48,14 @@ func ExecOutputTrim(cmd string) (string, error) { } var ( - _, podCIDR, _ = net.ParseCIDR("10.10.10.0/24") - nodeIP, nodeIntf, _ = util.GetIPNetDeviceFromIP(func() net.IP { + _, podCIDR, _ = net.ParseCIDR("10.10.10.0/24") + nodeIPv4, _, nodeIntf, _ = util.GetIPNetDeviceFromIP(func() []net.IP { conn, _ := net.Dial("udp", "8.8.8.8:80") defer conn.Close() - return conn.LocalAddr().(*net.UDPAddr).IP + return []net.IP{conn.LocalAddr().(*net.UDPAddr).IP} }()) nodeLink, _ = netlink.LinkByName(nodeIntf.Name) - localPeerIP = ip.NextIP(nodeIP.IP) + localPeerIP = ip.NextIP(nodeIPv4.IP) remotePeerIP = net.ParseIP("50.50.50.1") _, serviceCIDR, _ = net.ParseCIDR("200.200.0.0/16") gwIP = net.ParseIP("10.10.10.1") @@ -63,11 +63,11 @@ var ( gwName = "antrea-gw0" gwConfig = &config.GatewayConfig{IPv4: gwIP, MAC: gwMAC, Name: gwName} nodeConfig = &config.NodeConfig{ - Name: "test", - PodIPv4CIDR: podCIDR, - NodeIPAddr: nodeIP, - NodeTransportIPAddr: nodeIP, - GatewayConfig: gwConfig, + Name: "test", + PodIPv4CIDR: podCIDR, + NodeIPv4Addr: nodeIPv4, + NodeTransportIPv4Addr: nodeIPv4, + GatewayConfig: gwConfig, } ) @@ -549,7 +549,7 @@ func TestRouteTablePolicyOnly(t *testing.T) { gwIPOut, err := ExecOutputTrim(fmt.Sprintf("ip addr show %s", gwName)) assert.NoError(t, err) gwIP := net.IPNet{ - IP: nodeConfig.NodeIPAddr.IP, + IP: nodeConfig.NodeIPv4Addr.IP, Mask: net.CIDRMask(32, 32), } assert.Contains(t, gwIPOut, gwIP.String()) @@ -608,7 +608,7 @@ func TestIPv6RoutesAndNeighbors(t *testing.T) { Name: "test", PodIPv4CIDR: podCIDR, PodIPv6CIDR: ipv6Subnet, - NodeIPAddr: nodeIP, + NodeIPv4Addr: nodeIPv4, GatewayConfig: dualGWConfig, } err = routeClient.Initialize(dualNodeConfig, func() {})