[IPv6] Implement L3 connectivity for IPv6 traffic

1. Use IPv6 in iptables and ipset configuration.
2. Identifiy IPv6 address and configure in OpenFlow.
3. Use Node Internal address for tunnel.

Makefile

@@ -114,6 +114,7 @@ docker-test-integration:
 		-v $(DOCKER_CACHE)/gopath:/tmp/gopath \
 		-v $(DOCKER_CACHE)/gocache:/tmp/gocache \
 		-v $(CURDIR):/usr/src/github.com/vmware-tanzu/antrea:ro \
+		-v /lib/modules:/lib/modules \
 		antrea/test test-integration $(USERID) $(GRPID)
 
 .PHONY: docker-tidy

go.mod

@@ -14,7 +14,7 @@ require (
 	github.com/cheggaaa/pb/v3 v3.0.4
 	github.com/containernetworking/cni v0.7.1
 	github.com/containernetworking/plugins v0.8.2-0.20190724153215-ded2f1757770
-	github.com/contiv/libOpenflow v0.0.0-20200424005919-3a6722c98962
+	github.com/contiv/libOpenflow v0.0.0-20200728044739-7c6534390721
 	github.com/contiv/ofnet v0.0.0-00010101000000-000000000000
 	github.com/coreos/go-iptables v0.4.5
 	github.com/davecgh/go-spew v1.1.1
@@ -60,7 +60,7 @@ require (
 replace (
 	// antrea/plugins/octant/go.mod also has this replacement since replace statement in dependencies
 	// were ignored. We need to change antrea/plugins/octant/go.mod if there is any change here.
-	github.com/contiv/ofnet => github.com/wenyingd/ofnet v0.0.0-20200609044910-a72f3e66744e
+	github.com/contiv/ofnet => github.com/wenyingd/ofnet v0.0.0-20200728094531-d5b4d75f2cc3
 	// fake.NewSimpleClientset is quite slow when it's initialized with massive objects due to
 	// https://github.com/kubernetes/kubernetes/issues/89574. It takes more than tens of minutes to
 	// init a fake client with 200k objects, which makes it hard to run the NetworkPolicy scale test.

go.sum

@@ -72,8 +72,8 @@ github.com/containernetworking/cni v0.7.1 h1:fE3r16wpSEyaqY4Z4oFrLMmIGfBYIKpPrHK
 github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
 github.com/containernetworking/plugins v0.8.2-0.20190724153215-ded2f1757770 h1:AuFzUXPFhLlTfxhHMGdCIr7wZOQ0J0dEZLoffU0KfRM=
 github.com/containernetworking/plugins v0.8.2-0.20190724153215-ded2f1757770/go.mod h1:AlmXjbiLJBqvZ4vxkWAqjx1CKHVEoQf0/Ugrvc6Cv70=
-github.com/contiv/libOpenflow v0.0.0-20200424005919-3a6722c98962 h1:J5j3JV+Z9/Hg0ufQwk37uk8ZCJ+YUxEFxekBpIMI/JE=
-github.com/contiv/libOpenflow v0.0.0-20200424005919-3a6722c98962/go.mod h1:DtsPlJOByJZ+MO9YITEGUlbJ/jfh/ef0qeNyBYaeNR4=
+github.com/contiv/libOpenflow v0.0.0-20200728044739-7c6534390721 h1:kYk8terl1B/Nw1YsQwL/EQ6DKa0tpVeh2M8gSmbSAHI=
+github.com/contiv/libOpenflow v0.0.0-20200728044739-7c6534390721/go.mod h1:DtsPlJOByJZ+MO9YITEGUlbJ/jfh/ef0qeNyBYaeNR4=
 github.com/contiv/libovsdb v0.0.0-20170227191248-d0061a53e358 h1:AiA9SKyNXulsU7aAnyka3UFHYOIH00A9HvdIRnDXlg0=
 github.com/contiv/libovsdb v0.0.0-20170227191248-d0061a53e358/go.mod h1:+qKEHaNVPj+wrn5st7TEFH9wcUWCJq5ZBvVKPQwzAeg=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@@ -380,8 +380,8 @@ github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYp
 github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI=
 github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7ZovXvuNyL3XQl8UFofeikI1NW1Gypu7k=
 github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
-github.com/wenyingd/ofnet v0.0.0-20200609044910-a72f3e66744e h1:NM4NTe6Z+mF5IYlYAiEdRlY8XcMY4P6VlYqgsBhpojQ=
-github.com/wenyingd/ofnet v0.0.0-20200609044910-a72f3e66744e/go.mod h1:+g6SfqhTVqeGEmUJ0l4WtCgsL4dflTUJE4k+TPCKqXo=
+github.com/wenyingd/ofnet v0.0.0-20200728094531-d5b4d75f2cc3 h1:Fe47GJc0+uCgKDCxTxXlNw+dTDhCOx390tZZaERYhlg=
+github.com/wenyingd/ofnet v0.0.0-20200728094531-d5b4d75f2cc3/go.mod h1:oF9872TvzJqLzLKDGVMItRLWJHlnwXluuIuNbOP5WKM=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=

pkg/agent/controller/noderoute/node_route_controller.go

@@ -150,12 +150,11 @@ func (c *Controller) removeStaleGatewayRoutes() error {
 	// running, so the route to local Pods will be desired as well.
 	var desiredPodCIDRs []string
 	for _, node := range nodes {
-		// PodCIDR is allocated by K8s NodeIpamController asynchronously so it's possible we see a Node
-		// with no PodCIDR set when it just joins the cluster.
-		if node.Spec.PodCIDR == "" {
+		podCIDRs := getPodCIDRsOnNode(node)
+		if len(podCIDRs) == 0 {
 			continue
 		}
-		desiredPodCIDRs = append(desiredPodCIDRs, node.Spec.PodCIDR)
+		desiredPodCIDRs = append(desiredPodCIDRs, podCIDRs...)
 	}
 
 	// routeClient will remove orphaned routes whose destinations are not in desiredPodCIDRs.
@@ -353,16 +352,17 @@ func (c *Controller) syncNodeRoute(nodeName string) error {
 func (c *Controller) deleteNodeRoute(nodeName string) error {
 	klog.Infof("Deleting routes and flows to Node %s", nodeName)
 
-	podCIDR, installed := c.installedNodes.Load(nodeName)
+	podCIDRs, installed := c.installedNodes.Load(nodeName)
 	if !installed {
 		// Route is not added for this Node.
 		return nil
 	}
 
-	if err := c.routeClient.DeleteRoutes(podCIDR.(*net.IPNet)); err != nil {
-		return fmt.Errorf("failed to delete the route to Node %s: %v", nodeName, err)
+	for _, podCIDR := range podCIDRs.([]*net.IPNet) {
+		if err := c.routeClient.DeleteRoutes(podCIDR); err != nil {
+			return fmt.Errorf("failed to delete the route to Node %s: %v", nodeName, err)
+		}
 	}
-
 	if err := c.ofClient.UninstallNodeFlows(nodeName); err != nil {
 		return fmt.Errorf("failed to uninstall flows to Node %s: %v", nodeName, err)
 	}
@@ -390,25 +390,14 @@ func (c *Controller) addNodeRoute(nodeName string, node *v1.Node) error {
 		return nil
 	}
 
-	klog.Infof("Adding routes and flows to Node %s, podCIDR: %s, addresses: %v",
-		nodeName, node.Spec.PodCIDR, node.Status.Addresses)
+	klog.Infof("Adding routes and flows to Node %s, podCIDRs: %v, addresses: %v",
+		nodeName, node.Spec.PodCIDRs, node.Status.Addresses)
 
-	if node.Spec.PodCIDR == "" {
-		klog.Errorf("PodCIDR is empty for Node %s", nodeName)
-		// Does not help to return an error and trigger controller retries.
-		return nil
-	}
-	peerPodCIDRAddr, peerPodCIDR, err := net.ParseCIDR(node.Spec.PodCIDR)
-	if err != nil {
-		klog.Errorf("Failed to parse PodCIDR %s for Node %s", node.Spec.PodCIDR, nodeName)
-		return nil
-	}
 	peerNodeIP, err := GetNodeAddr(node)
 	if err != nil {
 		klog.Errorf("Failed to retrieve IP address of Node %s: %v", nodeName, err)
 		return nil
 	}
-	peerGatewayIP := ip.NextIP(peerPodCIDRAddr)
 
 	ipsecTunOFPort := int32(0)
 	if c.networkConfig.EnableIPSecTunnel {
@@ -420,25 +409,56 @@ func (c *Controller) addNodeRoute(nodeName string, node *v1.Node) error {
 		}
 	}
 
+	podCIDRStrs := getPodCIDRsOnNode(node)
+	if len(podCIDRStrs) == 0 {
+		// If no valid PodCIDR is configured in Node.Spec, return immediately.
+		return nil
+	}
+	var podCIDRs []*net.IPNet
+	peerConfig := make(map[*net.IPNet]net.IP, len(podCIDRStrs))
+	for _, podCIDR := range podCIDRStrs {
+		peerPodCIDRAddr, peerPodCIDR, err := net.ParseCIDR(podCIDR)
+		if err != nil {
+			klog.Errorf("Failed to parse PodCIDR %s for Node %s", podCIDR, nodeName)
+			return nil
+		}
+		peerGatewayIP := ip.NextIP(peerPodCIDRAddr)
+		peerConfig[peerPodCIDR] = peerGatewayIP
+		podCIDRs = append(podCIDRs, peerPodCIDR)
+	}
 	err = c.ofClient.InstallNodeFlows(
 		nodeName,
 		c.nodeConfig.GatewayConfig.MAC,
-		*peerPodCIDR,
-		peerGatewayIP,
+		peerConfig,
 		peerNodeIP,
 		config.DefaultTunOFPort,
 		uint32(ipsecTunOFPort))
 	if err != nil {
 		return fmt.Errorf("failed to install flows to Node %s: %v", nodeName, err)
 	}
 
-	if err := c.routeClient.AddRoutes(peerPodCIDR, peerNodeIP, peerGatewayIP); err != nil {
-		return err
+	for peerPodCIDR, peerGatewayIP := range peerConfig {
+		if err := c.routeClient.AddRoutes(peerPodCIDR, peerNodeIP, peerGatewayIP); err != nil {
+			return err
+		}
 	}
-	c.installedNodes.Store(nodeName, peerPodCIDR)
+	c.installedNodes.Store(nodeName, podCIDRs)
 	return err
 }
 
+func getPodCIDRsOnNode(node *v1.Node) []string {
+	if node.Spec.PodCIDRs != nil {
+		return node.Spec.PodCIDRs
+	}
+
+	if node.Spec.PodCIDR == "" {
+		klog.Errorf("PodCIDR is empty for Node %s", node.Name)
+		// Does not help to return an error and trigger controller retries.
+		return nil
+	}
+	return []string{node.Spec.PodCIDR}
+}
+
 // createIPSecTunnelPort creates an IPSec tunnel port for the remote Node if the
 // tunnel does not exist, and returns the ofport number.
 func (c *Controller) createIPSecTunnelPort(nodeName string, nodeIP net.IP) (int32, error) {

pkg/agent/openflow/client.go

@@ -71,8 +71,8 @@ type Client interface {
 	InstallNodeFlows(
 		hostname string,
 		localGatewayMAC net.HardwareAddr,
-		peerPodCIDR net.IPNet,
-		peerGatewayIP, tunnelPeerIP net.IP,
+		peerConfigs map[*net.IPNet]net.IP,
+		tunnelPeerIP net.IP,
 		tunOFPort, ipsecTunOFPort uint32) error
 
 	// UninstallNodeFlows removes the connection to the remote Node specified with the
@@ -283,20 +283,25 @@ func (c *client) deleteFlows(cache *flowCategoryCache, flowCacheKey string) erro
 
 func (c *client) InstallNodeFlows(hostname string,
 	localGatewayMAC net.HardwareAddr,
-	peerPodCIDR net.IPNet,
-	peerGatewayIP, tunnelPeerIP net.IP,
+	peerConfigs map[*net.IPNet]net.IP,
+	tunnelPeerIP net.IP,
 	tunOFPort, ipsecTunOFPort uint32) error {
 	c.replayMutex.RLock()
 	defer c.replayMutex.RUnlock()
 
-	flows := []binding.Flow{
-		c.arpResponderFlow(peerGatewayIP, cookie.Node),
-	}
-	if c.encapMode.NeedsEncapToPeer(tunnelPeerIP, c.nodeConfig.NodeIPAddr) {
-		flows = append(flows, c.l3FwdFlowToRemote(localGatewayMAC, peerPodCIDR, tunnelPeerIP, tunOFPort, cookie.Node))
-	} else {
-		flows = append(flows, c.l3FwdFlowToRemoteViaGW(localGatewayMAC, peerPodCIDR, cookie.Node))
+	var flows []binding.Flow
+
+	for peerPodCIDR, peerGatewayIP := range peerConfigs {
+		if peerGatewayIP.To4() != nil {
+			flows = append(flows, c.arpResponderFlow(peerGatewayIP, cookie.Node))
+		}
+		if c.encapMode.NeedsEncapToPeer(tunnelPeerIP, c.nodeConfig.NodeIPAddr) {
+			flows = append(flows, c.l3FwdFlowToRemote(localGatewayMAC, *peerPodCIDR, tunnelPeerIP, tunOFPort, cookie.Node))
+		} else {
+			flows = append(flows, c.l3FwdFlowToRemoteViaGW(localGatewayMAC, *peerPodCIDR, cookie.Node))
+		}
 	}
+
 	if ipsecTunOFPort != 0 {
 		// When IPSec tunnel is enabled, packets received from the remote Node are
 		// input from the Node's IPSec tunnel port, not the default tunnel port. So,

pkg/agent/openflow/client_test.go

@@ -43,7 +43,10 @@ func installNodeFlows(ofClient Client, cacheKey string) (int, error) {
 	gwMAC, _ := net.ParseMAC("AA:BB:CC:DD:EE:FF")
 	gwIP, IPNet, _ := net.ParseCIDR("10.0.1.1/24")
 	peerNodeIP := net.ParseIP("192.168.1.1")
-	err := ofClient.InstallNodeFlows(hostName, gwMAC, *IPNet, gwIP, peerNodeIP, config.DefaultTunOFPort, 0)
+	peerConfig := map[*net.IPNet]net.IP{
+		IPNet: gwIP,
+	}
+	err := ofClient.InstallNodeFlows(hostName, gwMAC, peerConfig, peerNodeIP, config.DefaultTunOFPort, 0)
 	client := ofClient.(*client)
 	fCacheI, ok := client.nodeFlowCache.Load(hostName)
 	if ok {

pkg/agent/openflow/pipeline.go

@@ -712,7 +712,8 @@ func (c *client) l3FwdFlowToRemote(
 	tunnelPeer net.IP,
 	tunOFPort uint32,
 	category cookie.Category) binding.Flow {
-	return c.pipeline[l3ForwardingTable].BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIP).
+	ipProto := parseIPProtocol(peerSubnet.IP)
+	return c.pipeline[l3ForwardingTable].BuildFlow(priorityNormal).MatchProtocol(ipProto).
 		MatchDstIPNet(peerSubnet).
 		Action().DecTTL().
 		// Rewrite src MAC to local gateway MAC and rewrite dst MAC to virtual MAC.
@@ -736,8 +737,9 @@ func (c *client) l3FwdFlowToRemoteViaGW(
 	localGatewayMAC net.HardwareAddr,
 	peerSubnet net.IPNet,
 	category cookie.Category) binding.Flow {
+	ipProto := parseIPProtocol(peerSubnet.IP)
 	l3FwdTable := c.pipeline[l3ForwardingTable]
-	return l3FwdTable.BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIP).
+	return l3FwdTable.BuildFlow(priorityNormal).MatchProtocol(ipProto).
 		MatchDstIPNet(peerSubnet).
 		Action().DecTTL().
 		Action().SetDstMAC(localGatewayMAC).

pkg/agent/route/interfaces.go

@@ -26,7 +26,8 @@ type Interface interface {
 	// It should be idempotent and can be safely called on every startup.
 	Initialize(nodeConfig *config.NodeConfig) error
 
-	// Reconcile should remove orphaned routes and related configuration based on the desired podCIDRs.
+	// Reconcile should remove orphaned routes and related configuration based on the desired podCIDRs. If IPv6 is enabled
+	// in the cluster, Reconcile should also remove the orphaned IPv6 neighbors.
 	Reconcile(podCIDRs []string) error
 
 	// AddRoutes should add routes to the provided podCIDR.