[IPv6] Consume Node.Spec.CIDRs to support dual-stack configuration (#971

)

1. Consume Node.Spec.CIDRs to support IPv4/IPv6 dual-stack Pod Subnets
2. Change NodeConfig.PodCIDR as a slice
3. Change GatewayConfig.IP as a slice to support multiple addresses for antrea-gw0
4. Change InterfaceConfig.IP as a slice to support multiple address for a Pod

pkg/agent/agent.go

@@ -285,7 +285,7 @@ func (i *Initializer) initOpenFlowPipeline() error {
 
 	// Set up flow entries for gateway interface, including classifier, skip spoof guard check,
 	// L3 forwarding and L2 forwarding
-	if err := i.ofClient.InstallGatewayFlows(gateway.IP, gateway.MAC, gatewayOFPort); err != nil {
+	if err := i.ofClient.InstallGatewayFlows(gateway.IPs, gateway.MAC, gatewayOFPort); err != nil {
 		klog.Errorf("Failed to setup openflow entries for gateway: %v", err)
 		return err
 	}
@@ -454,29 +454,21 @@ func (i *Initializer) configureGatewayInterface(gatewayIface *interfacestore.Int
 	gatewayIface.MAC = gwMAC
 	if i.networkConfig.TrafficEncapMode.IsNetworkPolicyOnly() {
 		// Assign IP to gw as required by SpoofGuard.
-		i.nodeConfig.GatewayConfig.IP = i.nodeConfig.NodeIPAddr.IP
-		gatewayIface.IP = i.nodeConfig.NodeIPAddr.IP
+		i.nodeConfig.GatewayConfig.IPs = []net.IP{i.nodeConfig.NodeIPAddr.IP}
+		gatewayIface.IPs = []net.IP{i.nodeConfig.NodeIPAddr.IP}
 		// No need to assign local CIDR to gw0 because local CIDR is not managed by Antrea
 		return nil
 	}
 
-	// Configure host gateway IP using the first address of node localSubnet.
-	localSubnet := i.nodeConfig.PodCIDR
-	subnetID := localSubnet.IP.Mask(localSubnet.Mask)
-	gwIP := &net.IPNet{IP: ip.NextIP(subnetID), Mask: localSubnet.Mask}
-
-	// Check IP address configuration on existing interface first, return if the interface has the desired address.
-	// We perform this check unconditionally, even if the OVS port does not exist when this function is called
-	// (i.e. portExists is false). Indeed, it may be possible for the interface to exist even if the OVS bridge does
-	// not exist.
-	// Configure the IP address on the interface if it does not exist.
-	if err := util.ConfigureLinkAddress(gwLinkIdx, gwIP); err != nil {
-		return err
+	i.nodeConfig.GatewayConfig.LinkIndex = gwLinkIdx
+	// Allocate the gateway IP address from the Pod CIDRs if it exists. The gateway IP should be the first address
+	// in the Subnet and configure on the host gateway.
+	for _, podCIDR := range []*net.IPNet{i.nodeConfig.PodIPv4CIDR, i.nodeConfig.PodIPv6CIDR} {
+		if err := i.allocateGatewayAddress(podCIDR, gatewayIface); err != nil {
+			return err
+		}
 	}
 
-	i.nodeConfig.GatewayConfig.LinkIndex = gwLinkIdx
-	i.nodeConfig.GatewayConfig.IP = gwIP.IP
-	gatewayIface.IP = gwIP.IP
 	return nil
 }
 
@@ -595,7 +587,34 @@ func (i *Initializer) initNodeLocalConfig() error {
 		return nil
 	}
 
-	// Spec.PodCIDR can be empty due to misconfiguration
+	// Parse all PodCIDRs first, so that we could support IPv4/IPv6 dual-stack configurations.
+	if node.Spec.PodCIDRs != nil {
+		for _, podCIDR := range node.Spec.PodCIDRs {
+			_, localSubnet, err := net.ParseCIDR(podCIDR)
+			if err != nil {
+				klog.Errorf("Failed to parse subnet from CIDR string %s: %v", node.Spec.PodCIDR, err)
+				return err
+			}
+			if localSubnet.IP.To4() != nil {
+				if i.nodeConfig.PodIPv4CIDR != nil {
+					klog.Warningf("One IPv4 PodCIDR is already configured on this Node, ignore the IPv4 Subnet CIDR %s", localSubnet.String())
+				} else {
+					i.nodeConfig.PodIPv4CIDR = localSubnet
+					klog.V(2).Infof("Configure IPv4 Subnet CIDR %s on this Node", localSubnet.String())
+				}
+				continue
+			}
+			if i.nodeConfig.PodIPv6CIDR != nil {
+				klog.Warningf("One IPv6 PodCIDR is already configured on this Node, ignore the IPv6 subnet CIDR %s", localSubnet.String())
+			} else {
+				i.nodeConfig.PodIPv6CIDR = localSubnet
+				klog.V(2).Infof("Configure IPv6 Subnet CIDR %s on this Node", localSubnet.String())
+			}
+		}
+		return nil
+	}
+
+	// Spec.PodCIDR can be empty due to misconfiguration.
 	if node.Spec.PodCIDR == "" {
 		klog.Errorf("Spec.PodCIDR is empty for Node %s. Please make sure --allocate-node-cidrs is enabled "+
 			"for kube-controller-manager and --cluster-cidr specifies a sufficient CIDR range", nodeName)
@@ -606,7 +625,7 @@ func (i *Initializer) initNodeLocalConfig() error {
 		klog.Errorf("Failed to parse subnet from CIDR string %s: %v", node.Spec.PodCIDR, err)
 		return err
 	}
-	i.nodeConfig.PodCIDR = localSubnet
+	i.nodeConfig.PodIPv4CIDR = localSubnet
 	return nil
 }
 
@@ -701,3 +720,24 @@ func (i *Initializer) getNodeMTU(localIntf *net.Interface) (int, error) {
 	}
 	return mtu, nil
 }
+
+func (i *Initializer) allocateGatewayAddress(localSubnet *net.IPNet, gatewayIface *interfacestore.InterfaceConfig) error {
+	if localSubnet == nil {
+		return nil
+	}
+	subnetID := localSubnet.IP.Mask(localSubnet.Mask)
+	gwIP := &net.IPNet{IP: ip.NextIP(subnetID), Mask: localSubnet.Mask}
+
+	// Check IP address configuration on existing interface first, return if the interface has the desired address.
+	// We perform this check unconditionally, even if the OVS port does not exist when this function is called
+	// (i.e. portExists is false). Indeed, it may be possible for the interface to exist even if the OVS bridge does
+	// not exist.
+	// Configure the IP address on the interface if it does not exist.
+	if err := util.ConfigureLinkAddress(i.nodeConfig.GatewayConfig.LinkIndex, gwIP); err != nil {
+		return err
+	}
+
+	i.nodeConfig.GatewayConfig.IPs = append(i.nodeConfig.GatewayConfig.IPs, gwIP.IP)
+	gatewayIface.IPs = append(gatewayIface.IPs, gwIP.IP)
+	return nil
+}

pkg/agent/agent_test.go

@@ -70,10 +70,10 @@ func TestInitstore(t *testing.T) {
 
 	ovsPort1 := ovsconfig.OVSPortData{UUID: uuid1, Name: "p1", IFName: "p1", OFPort: 11,
 		ExternalIDs: convertExternalIDMap(cniserver.BuildOVSPortExternalIDs(
-			interfacestore.NewContainerInterface("p1", uuid1, "pod1", "ns1", p1NetMAC, p1NetIP)))}
+			interfacestore.NewContainerInterface("p1", uuid1, "pod1", "ns1", p1NetMAC, []net.IP{p1NetIP})))}
 	ovsPort2 := ovsconfig.OVSPortData{UUID: uuid2, Name: "p2", IFName: "p2", OFPort: 12,
 		ExternalIDs: convertExternalIDMap(cniserver.BuildOVSPortExternalIDs(
-			interfacestore.NewContainerInterface("p2", uuid2, "pod2", "ns2", p2NetMAC, p2NetIP)))}
+			interfacestore.NewContainerInterface("p2", uuid2, "pod2", "ns2", p2NetMAC, []net.IP{p2NetIP})))}
 	initOVSPorts := []ovsconfig.OVSPortData{ovsPort1, ovsPort2}
 
 	mockOVSBridgeClient.EXPECT().GetPortList().Return(initOVSPorts, ovsconfig.NewTransactionError(fmt.Errorf("Failed to list OVS ports"), true))
@@ -90,7 +90,7 @@ func TestInitstore(t *testing.T) {
 	container1, found1 := store.GetContainerInterface(uuid1)
 	if !found1 {
 		t.Errorf("Failed to load OVS port into local store")
-	} else if container1.OFPort != 11 || container1.IP.String() != p1IP || container1.MAC.String() != p1MAC || container1.InterfaceName != "p1" {
+	} else if container1.OFPort != 11 || container1.GetIPv4Addr() == nil || container1.GetIPv4Addr().String() != p1IP || container1.MAC.String() != p1MAC || container1.InterfaceName != "p1" {
 		t.Errorf("Failed to load OVS port configuration into local store")
 	}
 	_, found2 := store.GetContainerInterface(uuid2)

pkg/agent/agent_windows.go

@@ -17,6 +17,7 @@
 package agent
 
 import (
+	"fmt"
 	"net"
 	"strings"
 
@@ -33,7 +34,10 @@ import (
 // setupExternalConnectivity installs OpenFlow entries to SNAT Pod traffic using Node IP, and then Pod could communicate
 // to the external IP address.
 func (i *Initializer) setupExternalConnectivity() error {
-	subnetCIDR := i.nodeConfig.PodCIDR
+	subnetCIDR := i.nodeConfig.PodIPv4CIDR
+	if subnetCIDR == nil {
+		return fmt.Errorf("Failed to find valid IPv4 PodCIDR")
+	}
 	nodeIP := i.nodeConfig.NodeIPAddr.IP
 	// Install OpenFlow entries on the OVS to enable Pod traffic to communicate to external IP addresses.
 	if err := i.ofClient.InstallExternalFlows(nodeIP, *subnetCIDR); err != nil {
@@ -81,7 +85,11 @@ func (i *Initializer) prepareHostNetwork() error {
 		return err
 	}
 	// Create HNS network.
-	return util.PrepareHNSNetwork(i.nodeConfig.PodCIDR, i.nodeConfig.NodeIPAddr, adapter)
+	subnetCIDR := i.nodeConfig.PodIPv4CIDR
+	if subnetCIDR == nil {
+		return fmt.Errorf("Failed to find valid IPv4 PodCIDR")
+	}
+	return util.PrepareHNSNetwork(subnetCIDR, i.nodeConfig.NodeIPAddr, adapter)
 }
 
 // prepareOVSBridge adds local port and uplink to ovs bridge.

pkg/agent/apiserver/handlers/ovstracing/handler.go

@@ -83,13 +83,13 @@ func getPeerAddress(aq querier.AgentQuerier, peer *tracingPeer) (net.IP, *interf
 			err := handlers.NewHandlerError(fmt.Errorf("OVS port %s not found", peer.ovsPort), http.StatusNotFound)
 			return nil, nil, err
 		}
-		return intf.IP, intf, nil
+		return intf.GetIPv4Addr(), intf, nil
 	}
 
 	interfaces := aq.GetInterfaceStore().GetContainerInterfacesByPod(peer.name, peer.namespace)
 	if len(interfaces) > 0 {
 		// Local Pod.
-		return interfaces[0].IP, interfaces[0], nil
+		return interfaces[0].GetIPv4Addr(), interfaces[0], nil
 	}
 
 	// Try getting the Pod from K8s API.

pkg/agent/apiserver/handlers/ovstracing/handler_test.go

@@ -46,7 +46,7 @@ var (
 	testNodeConfig = &config.NodeConfig{
 		GatewayConfig: &config.GatewayConfig{
 			Name: "antrea-gw0",
-			IP:   net.ParseIP("10.1.1.1"),
+			IPs:  []net.IP{net.ParseIP("10.1.1.1")},
 			MAC:  gatewayMAC},
 	}
 
@@ -55,19 +55,19 @@ var (
 	inPodInterface   = &interfacestore.InterfaceConfig{
 		Type:          interfacestore.ContainerInterface,
 		InterfaceName: "inPod",
-		IP:            net.ParseIP("10.1.1.11"),
+		IPs:           []net.IP{net.ParseIP("10.1.1.11")},
 		MAC:           podMAC,
 	}
 	srcPodInterface = &interfacestore.InterfaceConfig{
 		Type:          interfacestore.ContainerInterface,
 		InterfaceName: "srcPod",
-		IP:            net.ParseIP("10.1.1.12"),
+		IPs:           []net.IP{net.ParseIP("10.1.1.12")},
 		MAC:           podMAC,
 	}
 	dstPodInterface = &interfacestore.InterfaceConfig{
 		Type:          interfacestore.ContainerInterface,
 		InterfaceName: "dstPod",
-		IP:            net.ParseIP("10.1.1.13"),
+		IPs:           []net.IP{net.ParseIP("10.1.1.13")},
 		MAC:           podMAC,
 	}
 )

pkg/agent/apiserver/handlers/podinterface/handler.go

@@ -40,7 +40,7 @@ func generateResponse(i *interfacestore.InterfaceConfig) Response {
 		PodName:       i.ContainerInterfaceConfig.PodName,
 		PodNamespace:  i.ContainerInterfaceConfig.PodNamespace,
 		InterfaceName: i.InterfaceName,
-		IP:            i.IP.String(),
+		IP:            i.GetIPv4Addr().String(),
 		MAC:           i.MAC.String(),
 		PortUUID:      i.OVSPortConfig.PortUUID,
 		OFPort:        i.OVSPortConfig.OFPort,

pkg/agent/apiserver/handlers/podinterface/handler_test.go

@@ -91,7 +91,7 @@ var responses = []Response{
 var testInterfaceConfigs = []*interfacestore.InterfaceConfig{
 	{
 		InterfaceName: "interface0",
-		IP:            net.ParseIP(ipStrs[0]),
+		IPs:           []net.IP{net.ParseIP(ipStrs[0])},
 		MAC:           macs[0],
 		OVSPortConfig: &interfacestore.OVSPortConfig{
 			PortUUID: "portuuid0",
@@ -105,7 +105,7 @@ var testInterfaceConfigs = []*interfacestore.InterfaceConfig{
 	},
 	{
 		InterfaceName: "interface1",
-		IP:            net.ParseIP(ipStrs[1]),
+		IPs:           []net.IP{net.ParseIP(ipStrs[1])},
 		MAC:           macs[1],
 		OVSPortConfig: &interfacestore.OVSPortConfig{
 			PortUUID: "portuuid1",
@@ -119,7 +119,7 @@ var testInterfaceConfigs = []*interfacestore.InterfaceConfig{
 	},
 	{
 		InterfaceName: "interface2",
-		IP:            net.ParseIP(ipStrs[2]),
+		IPs:           []net.IP{net.ParseIP(ipStrs[2])},
 		MAC:           macs[2],
 		OVSPortConfig: &interfacestore.OVSPortConfig{
 			PortUUID: "portuuid2",

pkg/agent/cniserver/pod_configuration.go

@@ -18,6 +18,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net"
+	"strings"
 
 	cnitypes "github.com/containernetworking/cni/pkg/types"
 	"github.com/containernetworking/cni/pkg/types/current"
@@ -136,7 +137,7 @@ func buildContainerConfig(
 		podName,
 		podNamespace,
 		containerMAC,
-		containerIP)
+		[]net.IP{containerIP})
 }
 
 // BuildOVSPortExternalIDs parses OVS port external_ids from InterfaceConfig.
@@ -145,12 +146,20 @@ func BuildOVSPortExternalIDs(containerConfig *interfacestore.InterfaceConfig) ma
 	externalIDs := make(map[string]interface{})
 	externalIDs[ovsExternalIDMAC] = containerConfig.MAC.String()
 	externalIDs[ovsExternalIDContainerID] = containerConfig.ContainerID
-	externalIDs[ovsExternalIDIP] = containerConfig.IP.String()
+	externalIDs[ovsExternalIDIP] = getContainerIPsString(containerConfig.IPs)
 	externalIDs[ovsExternalIDPodName] = containerConfig.PodName
 	externalIDs[ovsExternalIDPodNamespace] = containerConfig.PodNamespace
 	return externalIDs
 }
 
+func getContainerIPsString(ips []net.IP) string {
+	var containerIPs []string
+	for _, ip := range ips {
+		containerIPs = append(containerIPs, ip.String())
+	}
+	return strings.Join(containerIPs, ",")
+}
+
 // ParseOVSPortInterfaceConfig reads the Pod properties saved in the OVS port
 // external_ids, initializes and returns an InterfaceConfig struct.
 // nill will be returned, if the OVS port does not have external IDs or it is
@@ -166,7 +175,12 @@ func ParseOVSPortInterfaceConfig(portData *ovsconfig.OVSPortData, portConfig *in
 		klog.V(2).Infof("OVS port %s has no %s in external_ids", portData.Name, ovsExternalIDContainerID)
 		return nil
 	}
-	containerIP := net.ParseIP(portData.ExternalIDs[ovsExternalIDIP])
+	containerIPStrs := strings.Split(portData.ExternalIDs[ovsExternalIDIP], ",")
+	var containerIPs []net.IP
+	for _, ipStr := range containerIPStrs {
+		containerIPs = append(containerIPs, net.ParseIP(ipStr))
+	}
+
 	containerMAC, err := net.ParseMAC(portData.ExternalIDs[ovsExternalIDMAC])
 	if err != nil {
 		klog.Errorf("Failed to parse MAC address from OVS external config %s: %v",
@@ -181,7 +195,7 @@ func ParseOVSPortInterfaceConfig(portData *ovsconfig.OVSPortData, portConfig *in
 		podName,
 		podNamespace,
 		containerMAC,
-		containerIP)
+		containerIPs)
 	interfaceConfig.OVSPortConfig = portConfig
 	return interfaceConfig
 }
@@ -362,13 +376,14 @@ func (pc *podConfigurator) validateOVSInterfaceConfig(containerID string, contai
 
 		for _, ipc := range ips {
 			if ipc.Version == "4" {
-				if containerConfig.IP.Equal(ipc.Address.IP) {
+				ipv4Addr := util.GetIPv4Addr(containerConfig.IPs)
+				if ipv4Addr != nil && ipv4Addr.Equal(ipc.Address.IP) {
 					return nil
 				}
 			}
 		}
 		return fmt.Errorf("interface IP %s does not match container %s IP",
-			containerConfig.IP.String(), containerID)
+			getContainerIPsString(containerConfig.IPs), containerID)
 	} else {
 		return fmt.Errorf("container %s interface not found from local cache", containerID)
 	}
@@ -420,7 +435,7 @@ func (pc *podConfigurator) reconcile(pods []corev1.Pod) error {
 			klog.V(4).Infof("Syncing interface %s for Pod %s", containerConfig.InterfaceName, namespacedName)
 			if err := pc.ofClient.InstallPodFlows(
 				containerConfig.InterfaceName,
-				containerConfig.IP,
+				containerConfig.IPs,
 				containerConfig.MAC,
 				pc.gatewayMAC,
 				uint32(containerConfig.OFPort),
@@ -486,7 +501,7 @@ func (pc *podConfigurator) connectInterfaceToOVS(
 	}
 
 	klog.V(2).Infof("Setting up Openflow entries for container %s", containerID)
-	err = pc.ofClient.InstallPodFlows(ovsPortName, containerConfig.IP, containerConfig.MAC, pc.gatewayMAC, uint32(ofPort))
+	err = pc.ofClient.InstallPodFlows(ovsPortName, containerConfig.IPs, containerConfig.MAC, pc.gatewayMAC, uint32(ofPort))
 	if err != nil {
 		return nil, fmt.Errorf("failed to add Openflow entries for container %s: %v", containerID, err)
 	}
@@ -551,11 +566,13 @@ func (pc *podConfigurator) disconnectInterceptedInterface(podName, podNamespace,
 		klog.V(2).Infof("Did not find the port for container %s in local cache", containerID)
 		return nil
 	}
-	if err := pc.routeClient.UnMigrateRoutesFromGw(&net.IPNet{
-		IP:   containerConfig.IP,
-		Mask: net.CIDRMask(32, 32),
-	}, ""); err != nil {
-		return fmt.Errorf("connectInterceptedInterface failed to migrate: %w", err)
+	for _, ip := range containerConfig.IPs {
+		if err := pc.routeClient.UnMigrateRoutesFromGw(&net.IPNet{
+			IP:   ip,
+			Mask: net.CIDRMask(32, 32),
+		}, ""); err != nil {
+			return fmt.Errorf("connectInterceptedInterface failed to migrate: %w", err)
+		}
 	}
 	return pc.disconnectInterfaceFromOVS(containerConfig)
 	// TODO recover pre-connect state? repatch vethpair to original bridge etc ?? to make first CNI happy??

pkg/agent/cniserver/server.go

@@ -219,9 +219,12 @@ func (s *CNIServer) checkRequestMessage(request *cnipb.CniCmdRequest) (*CNIConfi
 }
 
 func (s *CNIServer) updateLocalIPAMSubnet(cniConfig *CNIConfig) {
-	cniConfig.NetworkConfig.IPAM.Gateway = s.nodeConfig.GatewayConfig.IP.String()
-	cniConfig.NetworkConfig.IPAM.Subnet = s.nodeConfig.PodCIDR.String()
-	cniConfig.NetworkConfiguration, _ = json.Marshal(cniConfig.NetworkConfig)
+	gwIPv4 := util.GetIPv4Addr(s.nodeConfig.GatewayConfig.IPs)
+	cniConfig.NetworkConfig.IPAM.Gateway = gwIPv4.String()
+	if s.nodeConfig.PodIPv4CIDR != nil {
+		cniConfig.NetworkConfig.IPAM.Subnet = s.nodeConfig.PodIPv4CIDR.String()
+		cniConfig.NetworkConfiguration, _ = json.Marshal(cniConfig.NetworkConfig)
+	}
 }
 
 func (s *CNIServer) generateCNIErrorResponse(cniErrorCode cnipb.ErrorCode, cniErrorMsg string) *cnipb.CniCmdResponse {
@@ -400,7 +403,8 @@ func (s *CNIServer) CmdAdd(ctx context.Context, request *cnipb.CniCmdRequest) (*
 	result.IPs = ipamResult.IPs
 	result.Routes = ipamResult.Routes
 	// Ensure interface gateway setting and mapping relations between result.Interfaces and result.IPs
-	updateResultIfaceConfig(result, s.nodeConfig.GatewayConfig.IP)
+	gwIPv4 := util.GetIPv4Addr(s.nodeConfig.GatewayConfig.IPs)
+	updateResultIfaceConfig(result, gwIPv4)
 	// Setup pod interfaces and connect to ovs bridge
 	podName := string(cniConfig.K8S_POD_NAME)
 	podNamespace := string(cniConfig.K8S_POD_NAMESPACE)