From c2ae008fefe4375f79ef389653c43f12cc3cc9ad Mon Sep 17 00:00:00 2001 From: Zhecheng Li Date: Fri, 16 Jul 2021 19:11:41 +0800 Subject: [PATCH] [IPv6] Support no-encap mode in dual-stack setup Signed-off-by: Zhecheng Li --- cmd/antrea-agent/agent.go | 2 +- pkg/agent/agent.go | 29 +++---- pkg/agent/agent_test.go | 6 +- pkg/agent/config/node_config.go | 16 ++-- .../egress/ipassigner/ip_assigner_linux.go | 2 +- .../noderoute/node_route_controller.go | 78 ++++++++++++------- pkg/agent/controller/traceflow/packetin.go | 2 +- pkg/agent/memberlist/cluster.go | 2 +- pkg/agent/memberlist/cluster_test.go | 10 +-- pkg/agent/openflow/client.go | 37 ++++++--- pkg/agent/openflow/pipeline_windows.go | 3 +- pkg/agent/querier/querier_test.go | 16 ++-- pkg/agent/route/route_linux.go | 31 ++++++-- pkg/agent/route/route_windows.go | 5 +- pkg/agent/util/net.go | 29 +++++-- pkg/agent/util/net_test.go | 2 +- pkg/antctl/raw/helper.go | 16 ++-- pkg/antctl/raw/supportbundle/command.go | 17 ++-- pkg/util/k8s/node.go | 29 +++---- pkg/util/k8s/node_test.go | 8 +- test/integration/agent/openflow_test.go | 6 +- test/integration/agent/route_test.go | 14 ++-- 22 files changed, 226 insertions(+), 134 deletions(-) diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go index 8789033da43..e0ec3af695d 100644 --- a/cmd/antrea-agent/agent.go +++ b/cmd/antrea-agent/agent.go @@ -226,7 +226,7 @@ func run(o *Options) error { var egressController *egress.EgressController if features.DefaultFeatureGate.Enabled(features.Egress) { egressController, err = egress.NewEgressController( - ofClient, antreaClientProvider, crdClient, ifaceStore, routeClient, nodeConfig.Name, nodeConfig.NodeIPAddr.IP, + ofClient, antreaClientProvider, crdClient, ifaceStore, routeClient, nodeConfig.Name, nodeConfig.NodeIPv4Addr.IP, o.config.ClusterMembershipPort, egressInformer, nodeInformer, externalIPPoolInformer, ) if err != nil { diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index d059c9b30f1..bda2e506e1c 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -530,15 +530,17 @@ func (i *Initializer) configureGatewayInterface(gatewayIface *interfacestore.Int i.nodeConfig.GatewayConfig = &config.GatewayConfig{Name: i.hostGateway, MAC: gwMAC} gatewayIface.MAC = gwMAC + gatewayIface.IPs = []net.IP{} if i.networkConfig.TrafficEncapMode.IsNetworkPolicyOnly() { - // Assign IP to gw as required by SpoofGuard. - // NodeIPAddr can be either IPv4 or IPv6. - if i.nodeConfig.NodeIPAddr.IP.To4() != nil { - i.nodeConfig.GatewayConfig.IPv4 = i.nodeConfig.NodeIPAddr.IP - } else { - i.nodeConfig.GatewayConfig.IPv6 = i.nodeConfig.NodeIPAddr.IP + // Assign IPs to gw as required by SpoofGuard. + if i.nodeConfig.NodeIPv4Addr != nil { + i.nodeConfig.GatewayConfig.IPv4 = i.nodeConfig.NodeIPv4Addr.IP + gatewayIface.IPs = append(gatewayIface.IPs, i.nodeConfig.NodeIPv4Addr.IP) + } + if i.nodeConfig.NodeIPv6Addr != nil { + i.nodeConfig.GatewayConfig.IPv6 = i.nodeConfig.NodeIPv6Addr.IP + gatewayIface.IPs = append(gatewayIface.IPs, i.nodeConfig.NodeIPv6Addr.IP) } - gatewayIface.IPs = []net.IP{i.nodeConfig.NodeIPAddr.IP} // No need to assign local CIDR to gw0 because local CIDR is not managed by Antrea return nil } @@ -642,13 +644,13 @@ func (i *Initializer) initNodeLocalConfig() error { return err } - ipAddr, err := k8s.GetNodeAddr(node) + ipAddrs, err := k8s.GetNodeAddrs(node) if err != nil { - return fmt.Errorf("failed to obtain local IP address from K8s: %w", err) + return fmt.Errorf("failed to obtain local IP addresses from K8s: %w", err) } - localAddr, localIntf, err := getIPNetDeviceFromIP(ipAddr) + localV4Addr, localV6Addr, localIntf, err := getIPNetDeviceFromIP(ipAddrs) if err != nil { - return fmt.Errorf("failed to get local IPNet device with IP %v: %v", ipAddr, err) + return fmt.Errorf("failed to get local IPNet device with IP %v: %v", ipAddrs, err) } // Update the Node's MAC address in the annotations of the Node. The MAC address will be used for direct routing by @@ -675,7 +677,8 @@ func (i *Initializer) initNodeLocalConfig() error { Name: nodeName, OVSBridge: i.ovsBridge, DefaultTunName: defaultTunInterfaceName, - NodeIPAddr: localAddr, + NodeIPv4Addr: localV4Addr, + NodeIPv6Addr: localV6Addr, UplinkNetConfig: new(config.AdapterNetConfig)} mtu, err := i.getNodeMTU(localIntf) @@ -849,7 +852,7 @@ func (i *Initializer) getNodeMTU(localIntf *net.Interface) (int, error) { } else if i.networkConfig.TunnelType == ovsconfig.GRETunnel { mtu -= config.GREOverhead } - if i.nodeConfig.NodeIPAddr.IP.To4() == nil { + if i.nodeConfig.NodeIPv6Addr != nil { mtu -= config.IPv6ExtraOverhead } } diff --git a/pkg/agent/agent_test.go b/pkg/agent/agent_test.go index 298af4ca3d9..63c62dd518b 100644 --- a/pkg/agent/agent_test.go +++ b/pkg/agent/agent_test.go @@ -235,7 +235,7 @@ func TestInitNodeLocalConfig(t *testing.T) { OVSBridge: ovsBridge, DefaultTunName: defaultTunInterfaceName, PodIPv4CIDR: podCIDR, - NodeIPAddr: nodeIPNet, + NodeIPv4Addr: nodeIPNet, NodeMTU: tt.expectedMTU, UplinkNetConfig: new(config.AdapterNetConfig), } @@ -249,8 +249,8 @@ func TestInitNodeLocalConfig(t *testing.T) { func mockGetIPNetDeviceFromIP(ipNet *net.IPNet, ipDevice *net.Interface) func() { prevGetIPNetDeviceFromIP := getIPNetDeviceFromIP - getIPNetDeviceFromIP = func(localIP net.IP) (*net.IPNet, *net.Interface, error) { - return ipNet, ipDevice, nil + getIPNetDeviceFromIP = func(localIP []net.IP) (*net.IPNet, *net.IPNet, *net.Interface, error) { + return ipNet, nil, ipDevice, nil } return func() { getIPNetDeviceFromIP = prevGetIPNetDeviceFromIP } } diff --git a/pkg/agent/config/node_config.go b/pkg/agent/config/node_config.go index 786d325c9f1..3238587d107 100644 --- a/pkg/agent/config/node_config.go +++ b/pkg/agent/config/node_config.go @@ -83,8 +83,10 @@ type NodeConfig struct { // The CIDR block from where to allocate IPv6 address to Pod. // It's nil for the net workPolicyOnly trafficEncapMode which doesn't do IPAM. PodIPv6CIDR *net.IPNet - // The Node's IP used in Kubernetes. It has the network mask information. - NodeIPAddr *net.IPNet + // The Node's IPv4 IP used in Kubernetes. It has the network mask information. + NodeIPv4Addr *net.IPNet + // The Node's IPv6 IP used in Kubernetes. It has the network mask information. + NodeIPv6Addr *net.IPNet // Set either via defaultMTU config in antrea.yaml or auto discovered. // Auto discovery will use MTU value of the Node's primary interface. // For Encap and Hybrid mode, Node MTU will be adjusted to account for encap header. @@ -96,8 +98,8 @@ type NodeConfig struct { } func (n *NodeConfig) String() string { - return fmt.Sprintf("NodeName: %s, OVSBridge: %s, PodIPv4CIDR: %s, PodIPv6CIDR: %s, NodeIP: %s, Gateway: %s", - n.Name, n.OVSBridge, n.PodIPv4CIDR, n.PodIPv6CIDR, n.NodeIPAddr, n.GatewayConfig) + return fmt.Sprintf("NodeName: %s, OVSBridge: %s, PodIPv4CIDR: %s, PodIPv6CIDR: %s, NodeIPv4: %s, NodeIPv6: %s, Gateway: %s", + n.Name, n.OVSBridge, n.PodIPv4CIDR, n.PodIPv6CIDR, n.NodeIPv4Addr, n.NodeIPv6Addr, n.GatewayConfig) } // User provided network configuration parameters. @@ -109,15 +111,13 @@ type NetworkConfig struct { } // IsIPv4Enabled returns true if the cluster network supports IPv4. -// TODO: support dual-stack in networkPolicyOnly mode. func IsIPv4Enabled(nodeConfig *NodeConfig, trafficEncapMode TrafficEncapModeType) bool { return nodeConfig.PodIPv4CIDR != nil || - (trafficEncapMode.IsNetworkPolicyOnly() && nodeConfig.NodeIPAddr.IP.To4() != nil) + (trafficEncapMode.IsNetworkPolicyOnly() && nodeConfig.NodeIPv4Addr != nil) } // IsIPv6Enabled returns true if the cluster network supports IPv6. -// TODO: support dual-stack in networkPolicyOnly mode. func IsIPv6Enabled(nodeConfig *NodeConfig, trafficEncapMode TrafficEncapModeType) bool { return nodeConfig.PodIPv6CIDR != nil || - (trafficEncapMode.IsNetworkPolicyOnly() && nodeConfig.NodeIPAddr.IP.To4() == nil) + (trafficEncapMode.IsNetworkPolicyOnly() && nodeConfig.NodeIPv6Addr != nil) } diff --git a/pkg/agent/controller/egress/ipassigner/ip_assigner_linux.go b/pkg/agent/controller/egress/ipassigner/ip_assigner_linux.go index 0870e13f1da..4f2ef042092 100644 --- a/pkg/agent/controller/egress/ipassigner/ip_assigner_linux.go +++ b/pkg/agent/controller/egress/ipassigner/ip_assigner_linux.go @@ -49,7 +49,7 @@ type ipAssigner struct { // NewIPAssigner returns an *ipAssigner. func NewIPAssigner(nodeIPAddr net.IP, dummyDeviceName string) (*ipAssigner, error) { - _, egressInterface, err := util.GetIPNetDeviceFromIP(nodeIPAddr) + _, _, egressInterface, err := util.GetIPNetDeviceFromIP([]net.IP{nodeIPAddr}) if err != nil { return nil, fmt.Errorf("get IPNetDevice from ip %v error: %+v", nodeIPAddr, err) } diff --git a/pkg/agent/controller/noderoute/node_route_controller.go b/pkg/agent/controller/noderoute/node_route_controller.go index 7d2dd79dd9a..0d87559caa6 100644 --- a/pkg/agent/controller/noderoute/node_route_controller.go +++ b/pkg/agent/controller/noderoute/node_route_controller.go @@ -135,7 +135,7 @@ func nodeRouteInfoPodCIDRIndexFunc(obj interface{}) ([]string, error) { type nodeRouteInfo struct { nodeName string podCIDRs []*net.IPNet - nodeIP net.IP + nodeIPs []net.IP gatewayIP []net.IP nodeMAC net.HardwareAddr } @@ -215,17 +215,26 @@ func (c *Controller) removeStaleTunnelPorts() error { continue } - peerNodeIP, err := k8s.GetNodeAddr(node) + peerNodeIPs, err := k8s.GetNodeAddrs(node) if err != nil { klog.Errorf("Failed to retrieve IP address of Node %s: %v", node.Name, err) continue } + var peerNodeIPv4, peerNodeIPv6 net.IP + for _, ip := range peerNodeIPs { + if ip.To4() == nil { + peerNodeIPv6 = ip + } else { + peerNodeIPv4 = ip + } + } ifaceID := util.GenerateNodeTunnelInterfaceKey(node.Name) validConfiguration := interfaceConfig.PSK == c.networkConfig.IPSecPSK && - interfaceConfig.RemoteIP.Equal(peerNodeIP) && interfaceConfig.TunnelInterfaceConfig.Type == c.networkConfig.TunnelType - if validConfiguration { + v4RemoteIP := peerNodeIPv4 != nil && interfaceConfig.RemoteIP.Equal(peerNodeIPv4) + v6RemoteIP := peerNodeIPv6 != nil && interfaceConfig.RemoteIP.Equal(peerNodeIPv6) + if validConfiguration && (v4RemoteIP || v6RemoteIP) { desiredInterfaces[ifaceID] = true } } @@ -430,7 +439,7 @@ func (c *Controller) addNodeRoute(nodeName string, node *corev1.Node) error { klog.Infof("Adding routes and flows to Node %s, podCIDRs: %v, addresses: %v", nodeName, podCIDRStrs, node.Status.Addresses) - var podCIDRs []*net.IPNet + var peerPodCIDRs []*net.IPNet peerConfig := make(map[*net.IPNet]net.IP, len(podCIDRStrs)) for _, podCIDR := range podCIDRStrs { klog.Infof("Adding routes and flows to Node %s, podCIDR: %s, addresses: %v", @@ -467,37 +476,53 @@ func (c *Controller) addNodeRoute(nodeName string, node *corev1.Node) error { } peerGatewayIP := ip.NextIP(peerPodCIDRAddr) peerConfig[peerPodCIDR] = peerGatewayIP - podCIDRs = append(podCIDRs, peerPodCIDR) + peerPodCIDRs = append(peerPodCIDRs, peerPodCIDR) } - peerNodeIP, err := k8s.GetNodeAddr(node) + peerNodeIPs, err := k8s.GetNodeAddrs(node) if err != nil { - klog.Errorf("Failed to retrieve IP address of Node %s: %v", nodeName, err) + klog.Errorf("Failed to retrieve IP addresses of Node %s: %v", nodeName, err) return nil } - ipsecTunOFPort := int32(0) - if c.networkConfig.EnableIPSecTunnel { - // Create a separate tunnel port for the Node, as OVS IPSec monitor needs to - // read PSK and remote IP from the Node's tunnel interface to create IPSec - // security policies. - if ipsecTunOFPort, err = c.createIPSecTunnelPort(nodeName, peerNodeIP); err != nil { - return err + for _, peerNodeIP := range peerNodeIPs { + ipsecTunOFPort := int32(0) + if c.networkConfig.EnableIPSecTunnel { + // Create a separate tunnel port for the Node, as OVS IPSec monitor needs to + // read PSK and remote IP from the Node's tunnel interface to create IPSec + // security policies. + if ipsecTunOFPort, err = c.createIPSecTunnelPort(nodeName, peerNodeIP); err != nil { + return err + } } - } - err = c.ofClient.InstallNodeFlows( - nodeName, - peerConfig, - peerNodeIP, - uint32(ipsecTunOFPort), - peerNodeMAC) - if err != nil { - return fmt.Errorf("failed to install flows to Node %s: %v", nodeName, err) + err = c.ofClient.InstallNodeFlows( + nodeName, + peerConfig, + peerNodeIP, + uint32(ipsecTunOFPort), + peerNodeMAC) + if err != nil { + return fmt.Errorf("failed to install flows to Node %s: %v", nodeName, err) + } } var peerGatewayIPs []net.IP + var peerNodeIPv4, peerNodeIPv6 net.IP + for _, ip := range peerNodeIPs { + if ip.To4() == nil { + peerNodeIPv6 = ip + } else { + peerNodeIPv4 = ip + } + } for peerPodCIDR, peerGatewayIP := range peerConfig { + var peerNodeIP net.IP + if peerGatewayIP.To4() == nil { + peerNodeIP = peerNodeIPv6 + } else { + peerNodeIP = peerNodeIPv4 + } if err := c.routeClient.AddRoutes(peerPodCIDR, nodeName, peerNodeIP, peerGatewayIP); err != nil { return err } @@ -505,11 +530,12 @@ func (c *Controller) addNodeRoute(nodeName string, node *corev1.Node) error { } c.installedNodes.Add(&nodeRouteInfo{ nodeName: nodeName, - podCIDRs: podCIDRs, - nodeIP: peerNodeIP, + podCIDRs: peerPodCIDRs, + nodeIPs: peerNodeIPs, gatewayIP: peerGatewayIPs, nodeMAC: peerNodeMAC, }) + return err } diff --git a/pkg/agent/controller/traceflow/packetin.go b/pkg/agent/controller/traceflow/packetin.go index 474baecde6f..02ba83c6f55 100644 --- a/pkg/agent/controller/traceflow/packetin.go +++ b/pkg/agent/controller/traceflow/packetin.go @@ -231,7 +231,7 @@ func (c *Controller) parsePacketIn(pktIn *ofctrl.PacketIn) (*crdv1alpha1.Tracefl if tableID == uint8(openflow.L2ForwardingOutTable) { ob := new(crdv1alpha1.Observation) tunnelDstIP := "" - isIPv6 := c.nodeConfig.NodeIPAddr.IP.To4() == nil + isIPv6 := c.nodeConfig.NodeIPv4Addr == nil if match := getMatchTunnelDstField(matchers, isIPv6); match != nil { tunnelDstIP, err = getTunnelDstValue(match) if err != nil { diff --git a/pkg/agent/memberlist/cluster.go b/pkg/agent/memberlist/cluster.go index 27b38ba4a4f..dcea8b12507 100644 --- a/pkg/agent/memberlist/cluster.go +++ b/pkg/agent/memberlist/cluster.go @@ -252,7 +252,7 @@ func (c *Cluster) enqueueExternalIPPool(obj interface{}) { // newClusterMember gets the Node's IP and returns a cluster member ":" // representing that Node in the memberlist cluster. func (c *Cluster) newClusterMember(node *corev1.Node) (string, error) { - nodeAddr, err := k8s.GetNodeAddr(node) + nodeAddr, err := k8s.GetNodeAddrs(node) if err != nil { return "", fmt.Errorf("obtain IP address from K8s Node failed: %v", err) } diff --git a/pkg/agent/memberlist/cluster_test.go b/pkg/agent/memberlist/cluster_test.go index f08d5688e04..f5f45b8670f 100644 --- a/pkg/agent/memberlist/cluster_test.go +++ b/pkg/agent/memberlist/cluster_test.go @@ -58,7 +58,7 @@ func newFakeCluster(nodeConfig *config.NodeConfig, stopCh <-chan struct{}, i int crdInformerFactory := crdinformers.NewSharedInformerFactory(crdClient, 0) ipPoolInformer := crdInformerFactory.Crd().V1alpha2().ExternalIPPools() - cluster, err := NewCluster(port, nodeConfig.NodeIPAddr.IP, nodeConfig.Name, nodeInformer, ipPoolInformer) + cluster, err := NewCluster(port, nodeConfig.NodeIPv4Addr.IP, nodeConfig.Name, nodeInformer, ipPoolInformer) if err != nil { return nil, err } @@ -137,8 +137,8 @@ func TestCluster_Run(t *testing.T) { for i, tCase := range testCases { t.Run(tCase.name, func(t *testing.T) { nodeConfig := &config.NodeConfig{ - Name: localNodeName, - NodeIPAddr: &net.IPNet{IP: net.IPv4(127, 0, 0, 1), Mask: net.IPv4Mask(255, 255, 255, 255)}, + Name: localNodeName, + NodeIPv4Addr: &net.IPNet{IP: net.IPv4(127, 0, 0, 1), Mask: net.IPv4Mask(255, 255, 255, 255)}, } stopCh := make(chan struct{}) defer close(stopCh) @@ -177,8 +177,8 @@ func TestCluster_RunClusterEvents(t *testing.T) { nodeName := "localNodeName" nodeConfig := &config.NodeConfig{ - Name: nodeName, - NodeIPAddr: &net.IPNet{IP: net.IPv4(127, 0, 0, 1)}, + Name: nodeName, + NodeIPv4Addr: &net.IPNet{IP: net.IPv4(127, 0, 0, 1)}, } localNode := &v1.Node{ ObjectMeta: metav1.ObjectMeta{Name: nodeName}, diff --git a/pkg/agent/openflow/client.go b/pkg/agent/openflow/client.go index 7e7970a176d..4d1424d1ef6 100644 --- a/pkg/agent/openflow/client.go +++ b/pkg/agent/openflow/client.go @@ -390,7 +390,7 @@ func (c *client) InstallNodeFlows(hostname string, // only work for IPv4 addresses. flows = append(flows, c.arpResponderFlow(peerGatewayIP, cookie.Node)) } - if c.encapMode.NeedsEncapToPeer(tunnelPeerIP, c.nodeConfig.NodeIPAddr) { + if c.encapMode.NeedsEncapToPeer(tunnelPeerIP, c.nodeConfig.NodeIPv4Addr) || c.encapMode.NeedsEncapToPeer(tunnelPeerIP, c.nodeConfig.NodeIPv6Addr) { // tunnelPeerIP is the Node Internal Address. In a dual-stack setup, whether this address is an IPv4 address or an // IPv6 one is decided by the address family of Node Internal Address. flows = append(flows, c.l3FwdFlowToRemote(localGatewayMAC, *peerPodCIDR, tunnelPeerIP, cookie.Node)) @@ -724,21 +724,34 @@ func (c *client) Initialize(roundInfo types.RoundInfo, nodeConfig *config.NodeCo } func (c *client) InstallExternalFlows() error { - nodeIP := c.nodeConfig.NodeIPAddr.IP - localGatewayMAC := c.nodeConfig.GatewayConfig.MAC + install := func(nodeIP net.IP) error { + localGatewayMAC := c.nodeConfig.GatewayConfig.MAC - var flows []binding.Flow - if c.nodeConfig.PodIPv4CIDR != nil { - flows = c.externalFlows(nodeIP, *c.nodeConfig.PodIPv4CIDR, localGatewayMAC) - } - if c.nodeConfig.PodIPv6CIDR != nil { - flows = append(flows, c.externalFlows(nodeIP, *c.nodeConfig.PodIPv6CIDR, localGatewayMAC)...) + var flows []binding.Flow + if c.nodeConfig.PodIPv4CIDR != nil { + flows = c.externalFlows(nodeIP, *c.nodeConfig.PodIPv4CIDR, localGatewayMAC) + } + if c.nodeConfig.PodIPv6CIDR != nil { + flows = append(flows, c.externalFlows(nodeIP, *c.nodeConfig.PodIPv6CIDR, localGatewayMAC)...) + } + + if err := c.ofEntryOperations.AddAll(flows); err != nil { + return fmt.Errorf("failed to install flows for external communication: %v", err) + } + c.hostNetworkingFlows = append(c.hostNetworkingFlows, flows...) + return nil } - if err := c.ofEntryOperations.AddAll(flows); err != nil { - return fmt.Errorf("failed to install flows for external communication: %v", err) + if c.nodeConfig.NodeIPv4Addr != nil { + if err := install(c.nodeConfig.NodeIPv4Addr.IP); err != nil { + return err + } + } + if c.nodeConfig.NodeIPv6Addr != nil { + if err := install(c.nodeConfig.NodeIPv6Addr.IP); err != nil { + return err + } } - c.hostNetworkingFlows = append(c.hostNetworkingFlows, flows...) return nil } diff --git a/pkg/agent/openflow/pipeline_windows.go b/pkg/agent/openflow/pipeline_windows.go index 395268196b4..10524f0c159 100644 --- a/pkg/agent/openflow/pipeline_windows.go +++ b/pkg/agent/openflow/pipeline_windows.go @@ -297,7 +297,8 @@ func (c *client) hostBridgeUplinkFlows(localSubnet net.IPNet, category cookie.Ca func (c *client) l3FwdFlowToRemoteViaRouting(localGatewayMAC net.HardwareAddr, remoteGatewayMAC net.HardwareAddr, category cookie.Category, peerIP net.IP, peerPodCIDR *net.IPNet) []binding.Flow { - if c.encapMode.NeedsDirectRoutingToPeer(peerIP, c.nodeConfig.NodeIPAddr) && remoteGatewayMAC != nil { + // Currently, IPv6 is not supported on Windows. + if c.encapMode.NeedsDirectRoutingToPeer(peerIP, c.nodeConfig.NodeIPv4Addr) && remoteGatewayMAC != nil { // It enhances Windows Noencap mode performance by bypassing host network. flows := []binding.Flow{c.pipeline[l2ForwardingCalcTable].BuildFlow(priorityNormal). MatchDstMAC(remoteGatewayMAC). diff --git a/pkg/agent/querier/querier_test.go b/pkg/agent/querier/querier_test.go index 2661398057b..6034d6ef581 100644 --- a/pkg/agent/querier/querier_test.go +++ b/pkg/agent/querier/querier_test.go @@ -74,9 +74,9 @@ func TestAgentQuerierGetAgentInfo(t *testing.T) { { name: "networkPolicyOnly-mode non-partial", nodeConfig: &config.NodeConfig{ - Name: "foo", - OVSBridge: "br-int", - NodeIPAddr: getIPNet("10.10.0.10"), + Name: "foo", + OVSBridge: "br-int", + NodeIPv4Addr: getIPNet("10.10.0.10"), }, apiPort: 10350, partial: false, @@ -120,11 +120,11 @@ func TestAgentQuerierGetAgentInfo(t *testing.T) { { name: "encap-mode non-partial", nodeConfig: &config.NodeConfig{ - Name: "foo", - OVSBridge: "br-int", - NodeIPAddr: getIPNet("10.10.0.10"), - PodIPv4CIDR: getIPNet("20.20.20.0/24"), - PodIPv6CIDR: getIPNet("2001:ab03:cd04:55ef::/64"), + Name: "foo", + OVSBridge: "br-int", + NodeIPv4Addr: getIPNet("10.10.0.10"), + PodIPv4CIDR: getIPNet("20.20.20.0/24"), + PodIPv6CIDR: getIPNet("2001:ab03:cd04:55ef::/64"), }, apiPort: 10350, partial: false, diff --git a/pkg/agent/route/route_linux.go b/pkg/agent/route/route_linux.go index 45acd8e9ee1..e576e11d8a8 100644 --- a/pkg/agent/route/route_linux.go +++ b/pkg/agent/route/route_linux.go @@ -35,6 +35,7 @@ import ( "antrea.io/antrea/pkg/agent/util" "antrea.io/antrea/pkg/agent/util/ipset" "antrea.io/antrea/pkg/agent/util/iptables" + "antrea.io/antrea/pkg/agent/util/sysctl" "antrea.io/antrea/pkg/ovs/ovsconfig" "antrea.io/antrea/pkg/util/env" ) @@ -130,6 +131,11 @@ func (c *Client) Initialize(nodeConfig *config.NodeConfig, done func()) error { return fmt.Errorf("failed to initialize ip routes: %v", err) } + // Ensure IPv6 forwarding is enabled if it is a dual-stack or IPv6-only cluster. + if c.nodeConfig.NodeIPv6Addr != nil && sysctl.EnsureSysctlNetValue("ipv6/conf/all/forwarding", 1) != nil { + return fmt.Errorf("failed to enable IPv6 forwarding") + } + return nil } @@ -438,9 +444,17 @@ func (c *Client) restoreIptablesData(podCIDR *net.IPNet, podIPSet string, snatMa func (c *Client) initIPRoutes() error { if c.networkConfig.TrafficEncapMode.IsNetworkPolicyOnly() { gwLink := util.GetNetLink(c.nodeConfig.GatewayConfig.Name) - _, gwIP, _ := net.ParseCIDR(fmt.Sprintf("%s/32", c.nodeConfig.NodeIPAddr.IP.String())) - if err := netlink.AddrReplace(gwLink, &netlink.Addr{IPNet: gwIP}); err != nil { - return fmt.Errorf("failed to add address %s to gw %s: %v", gwIP, gwLink.Attrs().Name, err) + if c.nodeConfig.NodeIPv4Addr != nil { + _, gwIP, _ := net.ParseCIDR(fmt.Sprintf("%s/32", c.nodeConfig.NodeIPv4Addr.IP.String())) + if err := netlink.AddrReplace(gwLink, &netlink.Addr{IPNet: gwIP}); err != nil { + return fmt.Errorf("failed to add address %s to gw %s: %v", gwIP, gwLink.Attrs().Name, err) + } + } + if c.nodeConfig.NodeIPv6Addr != nil { + _, gwIP, _ := net.ParseCIDR(fmt.Sprintf("%s/128", c.nodeConfig.NodeIPv6Addr.IP.String())) + if err := netlink.AddrReplace(gwLink, &netlink.Addr{IPNet: gwIP}); err != nil { + return fmt.Errorf("failed to add address %s to gw %s: %v", gwIP, gwLink.Attrs().Name, err) + } } } return nil @@ -565,6 +579,13 @@ func (c *Client) listIPv6NeighborsOnGateway() (map[string]*netlink.Neigh, error) // AddRoutes adds routes to a new podCIDR. It overrides the routes if they already exist. func (c *Client) AddRoutes(podCIDR *net.IPNet, nodeName string, nodeIP, nodeGwIP net.IP) error { + var nodeIPAddr *net.IPNet + if podCIDR.IP.To4() == nil { + nodeIPAddr = c.nodeConfig.NodeIPv6Addr + } else { + nodeIPAddr = c.nodeConfig.NodeIPv4Addr + } + podCIDRStr := podCIDR.String() ipsetName := getIPSetName(podCIDR.IP) // Add this podCIDR to antreaPodIPSet so that packets to them won't be masqueraded when they leave the host. @@ -576,7 +597,7 @@ func (c *Client) AddRoutes(podCIDR *net.IPNet, nodeName string, nodeIP, nodeGwIP Dst: podCIDR, } var routes []*netlink.Route - if c.networkConfig.TrafficEncapMode.NeedsEncapToPeer(nodeIP, c.nodeConfig.NodeIPAddr) { + if c.networkConfig.TrafficEncapMode.NeedsEncapToPeer(nodeIP, nodeIPAddr) { if podCIDR.IP.To4() == nil { // "on-link" is not identified in IPv6 route entries, so split the configuration into 2 entries. routes = []*netlink.Route{ @@ -590,7 +611,7 @@ func (c *Client) AddRoutes(podCIDR *net.IPNet, nodeName string, nodeIP, nodeGwIP } route.LinkIndex = c.nodeConfig.GatewayConfig.LinkIndex route.Gw = nodeGwIP - } else if c.networkConfig.TrafficEncapMode.NeedsDirectRoutingToPeer(nodeIP, c.nodeConfig.NodeIPAddr) { + } else if c.networkConfig.TrafficEncapMode.NeedsDirectRoutingToPeer(nodeIP, nodeIPAddr) { // NoEncap traffic to Node on the same subnet. // Set the peerNodeIP as next hop. route.Gw = nodeIP diff --git a/pkg/agent/route/route_windows.go b/pkg/agent/route/route_windows.go index a889bd7999d..f6d48d37992 100644 --- a/pkg/agent/route/route_windows.go +++ b/pkg/agent/route/route_windows.go @@ -109,10 +109,11 @@ func (c *Client) AddRoutes(podCIDR *net.IPNet, nodeName string, peerNodeIP, peer DestinationSubnet: podCIDR, RouteMetric: util.DefaultMetric, } - if c.networkConfig.TrafficEncapMode.NeedsEncapToPeer(peerNodeIP, c.nodeConfig.NodeIPAddr) { + // Currently, IPv6 is not supported on Windows. + if c.networkConfig.TrafficEncapMode.NeedsEncapToPeer(peerNodeIP, c.nodeConfig.NodeIPv4Addr) { route.LinkIndex = c.nodeConfig.GatewayConfig.LinkIndex route.GatewayAddress = peerGwIP - } else if c.networkConfig.TrafficEncapMode.NeedsDirectRoutingToPeer(peerNodeIP, c.nodeConfig.NodeIPAddr) { + } else if c.networkConfig.TrafficEncapMode.NeedsDirectRoutingToPeer(peerNodeIP, c.nodeConfig.NodeIPv4Addr) { // NoEncap traffic to Node on the same subnet. // Set the peerNodeIP as next hop. route.LinkIndex = c.bridgeInfIndex diff --git a/pkg/agent/util/net.go b/pkg/agent/util/net.go index 13d15240570..88f12a12a2d 100644 --- a/pkg/agent/util/net.go +++ b/pkg/agent/util/net.go @@ -100,27 +100,40 @@ func dialUnix(address string) (net.Conn, error) { return net.Dial("unix", address) } -// GetIPNetDeviceFromIP returns a local IP/mask and associated device from IP. -func GetIPNetDeviceFromIP(localIP net.IP) (*net.IPNet, *net.Interface, error) { +// GetIPNetDeviceFromIP returns local IPs/masks and associated devices from IP. +func GetIPNetDeviceFromIP(localIPs []net.IP) (v4IPnet *net.IPNet, v6IPnet *net.IPNet, iface *net.Interface, err error) { linkList, err := net.Interfaces() if err != nil { - return nil, nil, err + return nil, nil, nil, err } - for _, link := range linkList { - addrList, err := link.Addrs() + for _, l := range linkList { + addrList, err := l.Addrs() if err != nil { continue } for _, addr := range addrList { if ipNet, ok := addr.(*net.IPNet); ok { - if ipNet.IP.Equal(localIP) { - return ipNet, &link, nil + for _, ip := range localIPs { + if ipNet.IP.Equal(ip) { + iface = &l + if ip.To4() == nil { + v6IPnet = ipNet + } else { + v4IPnet = ipNet + } + } } } } + if iface != nil { + break + } + } + if iface == nil { + return nil, nil, nil, fmt.Errorf("unable to find local IPs and device") } - return nil, nil, fmt.Errorf("unable to find local IP and device") + return v4IPnet, v6IPnet, iface, nil } func GetIPv4Addr(ips []net.IP) net.IP { diff --git a/pkg/agent/util/net_test.go b/pkg/agent/util/net_test.go index 4042564bfeb..2884cdc30a1 100644 --- a/pkg/agent/util/net_test.go +++ b/pkg/agent/util/net_test.go @@ -55,7 +55,7 @@ func TestGetDefaultLocalNodeAddr(t *testing.T) { defer conn.Close() ip := conn.LocalAddr().(*net.UDPAddr).IP - _, dev, err := GetIPNetDeviceFromIP(ip) + _, _, dev, err := GetIPNetDeviceFromIP([]net.IP{ip}) if err != nil { t.Error(err) } diff --git a/pkg/antctl/raw/helper.go b/pkg/antctl/raw/helper.go index 0a6b78edadf..da7b3ebe537 100644 --- a/pkg/antctl/raw/helper.go +++ b/pkg/antctl/raw/helper.go @@ -97,12 +97,15 @@ func CreateAgentClientCfg(k8sClientset kubernetes.Interface, antreaClientset ant if agentInfo == nil { return nil, fmt.Errorf("no Antrea Agent found for Node name %s", nodeName) } - nodeIP, err := k8s.GetNodeAddr(node) + nodeIPs, err := k8s.GetNodeAddrs(node) if err != nil { return nil, fmt.Errorf("error when parsing IP of Node %s", nodeName) } cfg := rest.CopyConfig(cfgTmpl) - cfg.Host = fmt.Sprintf("https://%s", net.JoinHostPort(nodeIP.String(), fmt.Sprint(agentInfo.APIPort))) + if len(nodeIPs) == 0 { + return nil, fmt.Errorf("there is no NodeIP on agent Node") + } + cfg.Host = fmt.Sprintf("https://%s", net.JoinHostPort(nodeIPs[0].String(), fmt.Sprint(agentInfo.APIPort))) return cfg, nil } @@ -116,13 +119,16 @@ func CreateControllerClientCfg(k8sClientset kubernetes.Interface, antreaClientse if err != nil { return nil, fmt.Errorf("error when searching the Node of the controller: %w", err) } - var controllerNodeIP net.IP - controllerNodeIP, err = k8s.GetNodeAddr(controllerNode) + var controllerNodeIPs []net.IP + controllerNodeIPs, err = k8s.GetNodeAddrs(controllerNode) if err != nil { return nil, fmt.Errorf("error when parsing controller IP: %w", err) } cfg := rest.CopyConfig(cfgTmpl) - cfg.Host = fmt.Sprintf("https://%s", net.JoinHostPort(controllerNodeIP.String(), fmt.Sprint(controllerInfo.APIPort))) + if len(controllerNodeIPs) == 0 { + return nil, fmt.Errorf("there is no NodeIP on controller Node") + } + cfg.Host = fmt.Sprintf("https://%s", net.JoinHostPort(controllerNodeIPs[0].String(), fmt.Sprint(controllerInfo.APIPort))) return cfg, nil } diff --git a/pkg/antctl/raw/supportbundle/command.go b/pkg/antctl/raw/supportbundle/command.go index 5a9ade8fb98..ba707166a97 100644 --- a/pkg/antctl/raw/supportbundle/command.go +++ b/pkg/antctl/raw/supportbundle/command.go @@ -300,13 +300,17 @@ func createAgentClients(k8sClientset kubernetes.Interface, antreaClientset antre if !ok { continue } - ip, err := k8s.GetNodeAddr(&node) + ips, err := k8s.GetNodeAddrs(&node) if err != nil { klog.Warningf("Error when parsing IP of Node %s", node.Name) continue } cfg := rest.CopyConfig(cfgTmpl) - cfg.Host = net.JoinHostPort(ip.String(), port) + if len(ips) == 0 { + klog.Warningf("There is no Node Addresses") + continue + } + cfg.Host = net.JoinHostPort(ips[0].String(), port) client, err := rest.RESTClientFor(cfg) if err != nil { klog.Warningf("Error when creating agent client for node: %s", node.Name) @@ -327,14 +331,17 @@ func createControllerClient(k8sClientset kubernetes.Interface, antreaClientset a if err != nil { return nil, fmt.Errorf("error when searching the Node of the controller: %w", err) } - var controllerNodeIP net.IP - controllerNodeIP, err = k8s.GetNodeAddr(controllerNode) + var controllerNodeIPs []net.IP + controllerNodeIPs, err = k8s.GetNodeAddrs(controllerNode) if err != nil { return nil, fmt.Errorf("error when parsing controllre IP: %w", err) } cfg := rest.CopyConfig(cfgTmpl) - cfg.Host = net.JoinHostPort(controllerNodeIP.String(), fmt.Sprint(controllerInfo.APIPort)) + if len(controllerNodeIPs) == 0 { + return nil, fmt.Errorf("error there is no NodeIP") + } + cfg.Host = net.JoinHostPort(controllerNodeIPs[0].String(), fmt.Sprint(controllerInfo.APIPort)) controllerClient, err := rest.RESTClientFor(cfg) if err != nil { klog.Warningf("Error when creating controller client for node: %s", controllerInfo.NodeRef.Name) diff --git a/pkg/util/k8s/node.go b/pkg/util/k8s/node.go index 0f4b48796e2..38520205ddb 100644 --- a/pkg/util/k8s/node.go +++ b/pkg/util/k8s/node.go @@ -21,27 +21,28 @@ import ( v1 "k8s.io/api/core/v1" ) -// GetNodeAddr gets the available IP address of a Node. GetNodeAddr will first try to get the NodeInternalIP, then try +// GetNodeAddrs gets the available IP addresses of a Node. GetNodeAddrs will first try to get the NodeInternalIP, then try // to get the NodeExternalIP. -// Note: Although K8s supports dual-stack, there is only a single Internal address per Node because of issue ( -// kubernetes/kubernetes#91940 ). The Node might have multiple addresses after the issue is fixed, and one per address -// family. And we should change the return type at that time. -func GetNodeAddr(node *v1.Node) (net.IP, error) { - addresses := make(map[v1.NodeAddressType]string) +func GetNodeAddrs(node *v1.Node) ([]net.IP, error) { + addresses := make(map[v1.NodeAddressType][]string) for _, addr := range node.Status.Addresses { - addresses[addr.Type] = addr.Address + addresses[addr.Type] = append(addresses[addr.Type], addr.Address) } - var ipAddrStr string + var ipAddrStrs []string if internalIP, ok := addresses[v1.NodeInternalIP]; ok { - ipAddrStr = internalIP + ipAddrStrs = internalIP } else if externalIP, ok := addresses[v1.NodeExternalIP]; ok { - ipAddrStr = externalIP + ipAddrStrs = externalIP } else { return nil, fmt.Errorf("Node %s has neither external ip nor internal ip", node.Name) } - ipAddr := net.ParseIP(ipAddrStr) - if ipAddr == nil { - return nil, fmt.Errorf("<%v> is not a valid ip address", ipAddrStr) + var nodeAddrs []net.IP + for i := range ipAddrStrs { + addr := net.ParseIP(ipAddrStrs[i]) + if addr == nil { + return nil, fmt.Errorf("'%s' is not a valid IP address", ipAddrStrs[i]) + } + nodeAddrs = append(nodeAddrs, addr) } - return ipAddr, nil + return nodeAddrs, nil } diff --git a/pkg/util/k8s/node_test.go b/pkg/util/k8s/node_test.go index 07362337a30..87935f2be40 100644 --- a/pkg/util/k8s/node_test.go +++ b/pkg/util/k8s/node_test.go @@ -28,7 +28,7 @@ func TestGetNodeAddr(t *testing.T) { tests := []struct { name string node *corev1.Node - expectedAddr net.IP + expectedAddr []net.IP expectedErr error }{ { @@ -54,7 +54,7 @@ func TestGetNodeAddr(t *testing.T) { }, }, }, - expectedAddr: net.ParseIP("192.168.10.10"), + expectedAddr: []net.IP{net.ParseIP("192.168.10.10")}, expectedErr: nil, }, { @@ -76,7 +76,7 @@ func TestGetNodeAddr(t *testing.T) { }, }, }, - expectedAddr: net.ParseIP("1.1.1.1"), + expectedAddr: []net.IP{net.ParseIP("1.1.1.1")}, expectedErr: nil, }, { @@ -100,7 +100,7 @@ func TestGetNodeAddr(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - addr, err := GetNodeAddr(tt.node) + addr, err := GetNodeAddrs(tt.node) assert.Equal(t, tt.expectedErr, err) assert.Equal(t, tt.expectedAddr, addr) }) diff --git a/test/integration/agent/openflow_test.go b/test/integration/agent/openflow_test.go index 8adc762d2d6..b004a4e0430 100644 --- a/test/integration/agent/openflow_test.go +++ b/test/integration/agent/openflow_test.go @@ -216,7 +216,7 @@ func TestReplayFlowsNetworkPolicyFlows(t *testing.T) { } func testExternalFlows(t *testing.T, config *testConfig) { - nodeIP := config.nodeConfig.NodeIPAddr.IP + nodeIP := config.nodeConfig.NodeIPv4Addr.IP var localSubnet *net.IPNet if config.nodeConfig.PodIPv4CIDR != nil { localSubnet = config.nodeConfig.PodIPv4CIDR @@ -855,7 +855,7 @@ func prepareConfiguration() *testConfig { MAC: gwMAC, } nodeConfig := &config1.NodeConfig{ - NodeIPAddr: nodeSubnet, + NodeIPv4Addr: nodeSubnet, GatewayConfig: gatewayConfig, PodIPv4CIDR: podIPv4CIDR, } @@ -900,7 +900,7 @@ func prepareIPv6Configuration() *testConfig { MAC: gwMAC, } nodeConfig := &config1.NodeConfig{ - NodeIPAddr: nodeSubnet, + NodeIPv4Addr: nodeSubnet, GatewayConfig: gatewayConfig, PodIPv6CIDR: podIPv6CIDR, } diff --git a/test/integration/agent/route_test.go b/test/integration/agent/route_test.go index e7edec83f53..d3ccfbb6d50 100644 --- a/test/integration/agent/route_test.go +++ b/test/integration/agent/route_test.go @@ -48,14 +48,14 @@ func ExecOutputTrim(cmd string) (string, error) { } var ( - _, podCIDR, _ = net.ParseCIDR("10.10.10.0/24") - nodeIP, nodeIntf, _ = util.GetIPNetDeviceFromIP(func() net.IP { + _, podCIDR, _ = net.ParseCIDR("10.10.10.0/24") + nodeIPv4, _, nodeIntf, _ = util.GetIPNetDeviceFromIP(func() []net.IP { conn, _ := net.Dial("udp", "8.8.8.8:80") defer conn.Close() - return conn.LocalAddr().(*net.UDPAddr).IP + return []net.IP{conn.LocalAddr().(*net.UDPAddr).IP} }()) nodeLink, _ = netlink.LinkByName(nodeIntf.Name) - localPeerIP = ip.NextIP(nodeIP.IP) + localPeerIP = ip.NextIP(nodeIPv4.IP) remotePeerIP = net.ParseIP("50.50.50.1") _, serviceCIDR, _ = net.ParseCIDR("200.200.0.0/16") gwIP = net.ParseIP("10.10.10.1") @@ -65,7 +65,7 @@ var ( nodeConfig = &config.NodeConfig{ Name: "test", PodIPv4CIDR: podCIDR, - NodeIPAddr: nodeIP, + NodeIPv4Addr: nodeIPv4, GatewayConfig: gwConfig, } ) @@ -548,7 +548,7 @@ func TestRouteTablePolicyOnly(t *testing.T) { gwIPOut, err := ExecOutputTrim(fmt.Sprintf("ip addr show %s", gwName)) assert.NoError(t, err) gwIP := net.IPNet{ - IP: nodeConfig.NodeIPAddr.IP, + IP: nodeConfig.NodeIPv4Addr.IP, Mask: net.CIDRMask(32, 32), } assert.Contains(t, gwIPOut, gwIP.String()) @@ -607,7 +607,7 @@ func TestIPv6RoutesAndNeighbors(t *testing.T) { Name: "test", PodIPv4CIDR: podCIDR, PodIPv6CIDR: ipv6Subnet, - NodeIPAddr: nodeIP, + NodeIPv4Addr: nodeIPv4, GatewayConfig: dualGWConfig, } err = routeClient.Initialize(dualNodeConfig, func() {})