From 76bd7c926f06680e3e2c828b232c4e25242e531e Mon Sep 17 00:00:00 2001 From: Wenying Dong Date: Fri, 7 Aug 2020 00:03:51 -0700 Subject: [PATCH] [IPv6] Consume Node.Spec.CIDRs to support dual-stack configuration (#971) 1. Consume Node.Spec.CIDRs to support IPv4/IPv6 dual-stack Pod Subnets 2. Change NodeConfig.PodCIDR as a slice 3. Change GatewayConfig.IP as a slice to support multiple addresses for antrea-gw0 4. Change InterfaceConfig.IP as a slice to support multiple address for a Pod --- pkg/agent/agent.go | 80 ++++++--- pkg/agent/agent_test.go | 6 +- pkg/agent/agent_windows.go | 12 +- .../apiserver/handlers/ovstracing/handler.go | 4 +- .../handlers/ovstracing/handler_test.go | 8 +- .../handlers/podinterface/handler.go | 2 +- .../handlers/podinterface/handler_test.go | 6 +- pkg/agent/cniserver/pod_configuration.go | 43 +++-- pkg/agent/cniserver/server.go | 12 +- pkg/agent/cniserver/server_test.go | 49 ++++-- pkg/agent/config/node_config.go | 17 +- .../controller/networkpolicy/reconciler.go | 5 +- .../networkpolicy/reconciler_test.go | 12 +- .../traceflow/traceflow_controller.go | 4 +- .../connections/connections_test.go | 2 +- .../flowexporter/connections/conntrack.go | 11 +- .../connections/conntrack_linux_test.go | 4 +- pkg/agent/interfacestore/interface_cache.go | 8 +- pkg/agent/interfacestore/types.go | 12 +- pkg/agent/openflow/client.go | 35 ++-- pkg/agent/openflow/client_test.go | 2 +- pkg/agent/openflow/pipeline.go | 127 ++++++++------ pkg/agent/openflow/testing/mock_openflow.go | 4 +- pkg/agent/querier/querier.go | 4 +- pkg/agent/querier/querier_test.go | 8 +- pkg/agent/route/route_linux.go | 6 +- pkg/agent/route/route_windows.go | 7 +- pkg/agent/util/net.go | 9 + pkg/ovs/openflow/interfaces.go | 17 +- pkg/ovs/openflow/ofctrl_builder.go | 14 ++ test/integration/agent/cniserver_test.go | 8 +- test/integration/agent/flowexporter_test.go | 2 +- test/integration/agent/openflow_test.go | 156 +++++++++++------- test/integration/agent/route_test.go | 4 +- 34 files changed, 463 insertions(+), 237 deletions(-) diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index 8bfa6a0d195..8b672a07619 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -286,7 +286,7 @@ func (i *Initializer) initOpenFlowPipeline() error { // Set up flow entries for gateway interface, including classifier, skip spoof guard check, // L3 forwarding and L2 forwarding - if err := i.ofClient.InstallGatewayFlows(gateway.IP, gateway.MAC, gatewayOFPort); err != nil { + if err := i.ofClient.InstallGatewayFlows(gateway.IPs, gateway.MAC, gatewayOFPort); err != nil { klog.Errorf("Failed to setup openflow entries for gateway: %v", err) return err } @@ -455,29 +455,21 @@ func (i *Initializer) configureGatewayInterface(gatewayIface *interfacestore.Int gatewayIface.MAC = gwMAC if i.networkConfig.TrafficEncapMode.IsNetworkPolicyOnly() { // Assign IP to gw as required by SpoofGuard. - i.nodeConfig.GatewayConfig.IP = i.nodeConfig.NodeIPAddr.IP - gatewayIface.IP = i.nodeConfig.NodeIPAddr.IP + i.nodeConfig.GatewayConfig.IPs = []net.IP{i.nodeConfig.NodeIPAddr.IP} + gatewayIface.IPs = []net.IP{i.nodeConfig.NodeIPAddr.IP} // No need to assign local CIDR to gw0 because local CIDR is not managed by Antrea return nil } - // Configure host gateway IP using the first address of node localSubnet. - localSubnet := i.nodeConfig.PodCIDR - subnetID := localSubnet.IP.Mask(localSubnet.Mask) - gwIP := &net.IPNet{IP: ip.NextIP(subnetID), Mask: localSubnet.Mask} - - // Check IP address configuration on existing interface first, return if the interface has the desired address. - // We perform this check unconditionally, even if the OVS port does not exist when this function is called - // (i.e. portExists is false). Indeed, it may be possible for the interface to exist even if the OVS bridge does - // not exist. - // Configure the IP address on the interface if it does not exist. - if err := util.ConfigureLinkAddress(gwLinkIdx, gwIP); err != nil { - return err + i.nodeConfig.GatewayConfig.LinkIndex = gwLinkIdx + // Allocate the gateway IP address from the Pod CIDRs if it exists. The gateway IP should be the first address + // in the Subnet and configure on the host gateway. + for _, podCIDR := range []*net.IPNet{i.nodeConfig.PodIPv4CIDR, i.nodeConfig.PodIPv6CIDR} { + if err := i.allocateGatewayAddress(podCIDR, gatewayIface); err != nil { + return err + } } - i.nodeConfig.GatewayConfig.LinkIndex = gwLinkIdx - i.nodeConfig.GatewayConfig.IP = gwIP.IP - gatewayIface.IP = gwIP.IP return nil } @@ -596,7 +588,34 @@ func (i *Initializer) initNodeLocalConfig() error { return nil } - // Spec.PodCIDR can be empty due to misconfiguration + // Parse all PodCIDRs first, so that we could support IPv4/IPv6 dual-stack configurations. + if node.Spec.PodCIDRs != nil { + for _, podCIDR := range node.Spec.PodCIDRs { + _, localSubnet, err := net.ParseCIDR(podCIDR) + if err != nil { + klog.Errorf("Failed to parse subnet from CIDR string %s: %v", node.Spec.PodCIDR, err) + return err + } + if localSubnet.IP.To4() != nil { + if i.nodeConfig.PodIPv4CIDR != nil { + klog.Warningf("One IPv4 PodCIDR is already configured on this Node, ignore the IPv4 Subnet CIDR %s", localSubnet.String()) + } else { + i.nodeConfig.PodIPv4CIDR = localSubnet + klog.V(2).Infof("Configure IPv4 Subnet CIDR %s on this Node", localSubnet.String()) + } + continue + } + if i.nodeConfig.PodIPv6CIDR != nil { + klog.Warningf("One IPv6 PodCIDR is already configured on this Node, ignore the IPv6 subnet CIDR %s", localSubnet.String()) + } else { + i.nodeConfig.PodIPv6CIDR = localSubnet + klog.V(2).Infof("Configure IPv6 Subnet CIDR %s on this Node", localSubnet.String()) + } + } + return nil + } + + // Spec.PodCIDR can be empty due to misconfiguration. if node.Spec.PodCIDR == "" { klog.Errorf("Spec.PodCIDR is empty for Node %s. Please make sure --allocate-node-cidrs is enabled "+ "for kube-controller-manager and --cluster-cidr specifies a sufficient CIDR range", nodeName) @@ -607,7 +626,7 @@ func (i *Initializer) initNodeLocalConfig() error { klog.Errorf("Failed to parse subnet from CIDR string %s: %v", node.Spec.PodCIDR, err) return err } - i.nodeConfig.PodCIDR = localSubnet + i.nodeConfig.PodIPv4CIDR = localSubnet return nil } @@ -732,3 +751,24 @@ func (i *Initializer) getNodeMTU(localIntf *net.Interface) (int, error) { } return mtu, nil } + +func (i *Initializer) allocateGatewayAddress(localSubnet *net.IPNet, gatewayIface *interfacestore.InterfaceConfig) error { + if localSubnet == nil { + return nil + } + subnetID := localSubnet.IP.Mask(localSubnet.Mask) + gwIP := &net.IPNet{IP: ip.NextIP(subnetID), Mask: localSubnet.Mask} + + // Check IP address configuration on existing interface first, return if the interface has the desired address. + // We perform this check unconditionally, even if the OVS port does not exist when this function is called + // (i.e. portExists is false). Indeed, it may be possible for the interface to exist even if the OVS bridge does + // not exist. + // Configure the IP address on the interface if it does not exist. + if err := util.ConfigureLinkAddress(i.nodeConfig.GatewayConfig.LinkIndex, gwIP); err != nil { + return err + } + + i.nodeConfig.GatewayConfig.IPs = append(i.nodeConfig.GatewayConfig.IPs, gwIP.IP) + gatewayIface.IPs = append(gatewayIface.IPs, gwIP.IP) + return nil +} diff --git a/pkg/agent/agent_test.go b/pkg/agent/agent_test.go index 260e316b055..cb983d9bc78 100644 --- a/pkg/agent/agent_test.go +++ b/pkg/agent/agent_test.go @@ -70,10 +70,10 @@ func TestInitstore(t *testing.T) { ovsPort1 := ovsconfig.OVSPortData{UUID: uuid1, Name: "p1", IFName: "p1", OFPort: 11, ExternalIDs: convertExternalIDMap(cniserver.BuildOVSPortExternalIDs( - interfacestore.NewContainerInterface("p1", uuid1, "pod1", "ns1", p1NetMAC, p1NetIP)))} + interfacestore.NewContainerInterface("p1", uuid1, "pod1", "ns1", p1NetMAC, []net.IP{p1NetIP})))} ovsPort2 := ovsconfig.OVSPortData{UUID: uuid2, Name: "p2", IFName: "p2", OFPort: 12, ExternalIDs: convertExternalIDMap(cniserver.BuildOVSPortExternalIDs( - interfacestore.NewContainerInterface("p2", uuid2, "pod2", "ns2", p2NetMAC, p2NetIP)))} + interfacestore.NewContainerInterface("p2", uuid2, "pod2", "ns2", p2NetMAC, []net.IP{p2NetIP})))} initOVSPorts := []ovsconfig.OVSPortData{ovsPort1, ovsPort2} mockOVSBridgeClient.EXPECT().GetPortList().Return(initOVSPorts, ovsconfig.NewTransactionError(fmt.Errorf("Failed to list OVS ports"), true)) @@ -90,7 +90,7 @@ func TestInitstore(t *testing.T) { container1, found1 := store.GetContainerInterface(uuid1) if !found1 { t.Errorf("Failed to load OVS port into local store") - } else if container1.OFPort != 11 || container1.IP.String() != p1IP || container1.MAC.String() != p1MAC || container1.InterfaceName != "p1" { + } else if container1.OFPort != 11 || container1.GetIPv4Addr() == nil || container1.GetIPv4Addr().String() != p1IP || container1.MAC.String() != p1MAC || container1.InterfaceName != "p1" { t.Errorf("Failed to load OVS port configuration into local store") } _, found2 := store.GetContainerInterface(uuid2) diff --git a/pkg/agent/agent_windows.go b/pkg/agent/agent_windows.go index 9e6857c4227..4bebd1f69eb 100644 --- a/pkg/agent/agent_windows.go +++ b/pkg/agent/agent_windows.go @@ -17,6 +17,7 @@ package agent import ( + "fmt" "net" "strings" @@ -33,7 +34,10 @@ import ( // setupExternalConnectivity installs OpenFlow entries to SNAT Pod traffic using Node IP, and then Pod could communicate // to the external IP address. func (i *Initializer) setupExternalConnectivity() error { - subnetCIDR := i.nodeConfig.PodCIDR + subnetCIDR := i.nodeConfig.PodIPv4CIDR + if subnetCIDR == nil { + return fmt.Errorf("Failed to find valid IPv4 PodCIDR") + } nodeIP := i.nodeConfig.NodeIPAddr.IP // Install OpenFlow entries on the OVS to enable Pod traffic to communicate to external IP addresses. if err := i.ofClient.InstallExternalFlows(nodeIP, *subnetCIDR); err != nil { @@ -81,7 +85,11 @@ func (i *Initializer) prepareHostNetwork() error { return err } // Create HNS network. - return util.PrepareHNSNetwork(i.nodeConfig.PodCIDR, i.nodeConfig.NodeIPAddr, adapter) + subnetCIDR := i.nodeConfig.PodIPv4CIDR + if subnetCIDR == nil { + return fmt.Errorf("Failed to find valid IPv4 PodCIDR") + } + return util.PrepareHNSNetwork(subnetCIDR, i.nodeConfig.NodeIPAddr, adapter) } // prepareOVSBridge adds local port and uplink to ovs bridge. diff --git a/pkg/agent/apiserver/handlers/ovstracing/handler.go b/pkg/agent/apiserver/handlers/ovstracing/handler.go index fbbfc1bf0bc..25c23ba20e0 100644 --- a/pkg/agent/apiserver/handlers/ovstracing/handler.go +++ b/pkg/agent/apiserver/handlers/ovstracing/handler.go @@ -83,13 +83,13 @@ func getPeerAddress(aq querier.AgentQuerier, peer *tracingPeer) (net.IP, *interf err := handlers.NewHandlerError(fmt.Errorf("OVS port %s not found", peer.ovsPort), http.StatusNotFound) return nil, nil, err } - return intf.IP, intf, nil + return intf.GetIPv4Addr(), intf, nil } interfaces := aq.GetInterfaceStore().GetContainerInterfacesByPod(peer.name, peer.namespace) if len(interfaces) > 0 { // Local Pod. - return interfaces[0].IP, interfaces[0], nil + return interfaces[0].GetIPv4Addr(), interfaces[0], nil } // Try getting the Pod from K8s API. diff --git a/pkg/agent/apiserver/handlers/ovstracing/handler_test.go b/pkg/agent/apiserver/handlers/ovstracing/handler_test.go index 61fc21ca924..b370bf823e0 100644 --- a/pkg/agent/apiserver/handlers/ovstracing/handler_test.go +++ b/pkg/agent/apiserver/handlers/ovstracing/handler_test.go @@ -46,7 +46,7 @@ var ( testNodeConfig = &config.NodeConfig{ GatewayConfig: &config.GatewayConfig{ Name: "antrea-gw0", - IP: net.ParseIP("10.1.1.1"), + IPs: []net.IP{net.ParseIP("10.1.1.1")}, MAC: gatewayMAC}, } @@ -55,19 +55,19 @@ var ( inPodInterface = &interfacestore.InterfaceConfig{ Type: interfacestore.ContainerInterface, InterfaceName: "inPod", - IP: net.ParseIP("10.1.1.11"), + IPs: []net.IP{net.ParseIP("10.1.1.11")}, MAC: podMAC, } srcPodInterface = &interfacestore.InterfaceConfig{ Type: interfacestore.ContainerInterface, InterfaceName: "srcPod", - IP: net.ParseIP("10.1.1.12"), + IPs: []net.IP{net.ParseIP("10.1.1.12")}, MAC: podMAC, } dstPodInterface = &interfacestore.InterfaceConfig{ Type: interfacestore.ContainerInterface, InterfaceName: "dstPod", - IP: net.ParseIP("10.1.1.13"), + IPs: []net.IP{net.ParseIP("10.1.1.13")}, MAC: podMAC, } ) diff --git a/pkg/agent/apiserver/handlers/podinterface/handler.go b/pkg/agent/apiserver/handlers/podinterface/handler.go index 712f21260d0..97a3c248e5d 100644 --- a/pkg/agent/apiserver/handlers/podinterface/handler.go +++ b/pkg/agent/apiserver/handlers/podinterface/handler.go @@ -40,7 +40,7 @@ func generateResponse(i *interfacestore.InterfaceConfig) Response { PodName: i.ContainerInterfaceConfig.PodName, PodNamespace: i.ContainerInterfaceConfig.PodNamespace, InterfaceName: i.InterfaceName, - IP: i.IP.String(), + IP: i.GetIPv4Addr().String(), MAC: i.MAC.String(), PortUUID: i.OVSPortConfig.PortUUID, OFPort: i.OVSPortConfig.OFPort, diff --git a/pkg/agent/apiserver/handlers/podinterface/handler_test.go b/pkg/agent/apiserver/handlers/podinterface/handler_test.go index ea41161ccec..5c127d93a2e 100644 --- a/pkg/agent/apiserver/handlers/podinterface/handler_test.go +++ b/pkg/agent/apiserver/handlers/podinterface/handler_test.go @@ -91,7 +91,7 @@ var responses = []Response{ var testInterfaceConfigs = []*interfacestore.InterfaceConfig{ { InterfaceName: "interface0", - IP: net.ParseIP(ipStrs[0]), + IPs: []net.IP{net.ParseIP(ipStrs[0])}, MAC: macs[0], OVSPortConfig: &interfacestore.OVSPortConfig{ PortUUID: "portuuid0", @@ -105,7 +105,7 @@ var testInterfaceConfigs = []*interfacestore.InterfaceConfig{ }, { InterfaceName: "interface1", - IP: net.ParseIP(ipStrs[1]), + IPs: []net.IP{net.ParseIP(ipStrs[1])}, MAC: macs[1], OVSPortConfig: &interfacestore.OVSPortConfig{ PortUUID: "portuuid1", @@ -119,7 +119,7 @@ var testInterfaceConfigs = []*interfacestore.InterfaceConfig{ }, { InterfaceName: "interface2", - IP: net.ParseIP(ipStrs[2]), + IPs: []net.IP{net.ParseIP(ipStrs[2])}, MAC: macs[2], OVSPortConfig: &interfacestore.OVSPortConfig{ PortUUID: "portuuid2", diff --git a/pkg/agent/cniserver/pod_configuration.go b/pkg/agent/cniserver/pod_configuration.go index 1bb5aa2c4fd..99c799a1888 100644 --- a/pkg/agent/cniserver/pod_configuration.go +++ b/pkg/agent/cniserver/pod_configuration.go @@ -18,6 +18,7 @@ import ( "encoding/json" "fmt" "net" + "strings" cnitypes "github.com/containernetworking/cni/pkg/types" "github.com/containernetworking/cni/pkg/types/current" @@ -136,7 +137,7 @@ func buildContainerConfig( podName, podNamespace, containerMAC, - containerIP) + []net.IP{containerIP}) } // BuildOVSPortExternalIDs parses OVS port external_ids from InterfaceConfig. @@ -145,12 +146,20 @@ func BuildOVSPortExternalIDs(containerConfig *interfacestore.InterfaceConfig) ma externalIDs := make(map[string]interface{}) externalIDs[ovsExternalIDMAC] = containerConfig.MAC.String() externalIDs[ovsExternalIDContainerID] = containerConfig.ContainerID - externalIDs[ovsExternalIDIP] = containerConfig.IP.String() + externalIDs[ovsExternalIDIP] = getContainerIPsString(containerConfig.IPs) externalIDs[ovsExternalIDPodName] = containerConfig.PodName externalIDs[ovsExternalIDPodNamespace] = containerConfig.PodNamespace return externalIDs } +func getContainerIPsString(ips []net.IP) string { + var containerIPs []string + for _, ip := range ips { + containerIPs = append(containerIPs, ip.String()) + } + return strings.Join(containerIPs, ",") +} + // ParseOVSPortInterfaceConfig reads the Pod properties saved in the OVS port // external_ids, initializes and returns an InterfaceConfig struct. // nill will be returned, if the OVS port does not have external IDs or it is @@ -166,7 +175,12 @@ func ParseOVSPortInterfaceConfig(portData *ovsconfig.OVSPortData, portConfig *in klog.V(2).Infof("OVS port %s has no %s in external_ids", portData.Name, ovsExternalIDContainerID) return nil } - containerIP := net.ParseIP(portData.ExternalIDs[ovsExternalIDIP]) + containerIPStrs := strings.Split(portData.ExternalIDs[ovsExternalIDIP], ",") + var containerIPs []net.IP + for _, ipStr := range containerIPStrs { + containerIPs = append(containerIPs, net.ParseIP(ipStr)) + } + containerMAC, err := net.ParseMAC(portData.ExternalIDs[ovsExternalIDMAC]) if err != nil { klog.Errorf("Failed to parse MAC address from OVS external config %s: %v", @@ -181,7 +195,7 @@ func ParseOVSPortInterfaceConfig(portData *ovsconfig.OVSPortData, portConfig *in podName, podNamespace, containerMAC, - containerIP) + containerIPs) interfaceConfig.OVSPortConfig = portConfig return interfaceConfig } @@ -362,13 +376,14 @@ func (pc *podConfigurator) validateOVSInterfaceConfig(containerID string, contai for _, ipc := range ips { if ipc.Version == "4" { - if containerConfig.IP.Equal(ipc.Address.IP) { + ipv4Addr := util.GetIPv4Addr(containerConfig.IPs) + if ipv4Addr != nil && ipv4Addr.Equal(ipc.Address.IP) { return nil } } } return fmt.Errorf("interface IP %s does not match container %s IP", - containerConfig.IP.String(), containerID) + getContainerIPsString(containerConfig.IPs), containerID) } else { return fmt.Errorf("container %s interface not found from local cache", containerID) } @@ -420,7 +435,7 @@ func (pc *podConfigurator) reconcile(pods []corev1.Pod) error { klog.V(4).Infof("Syncing interface %s for Pod %s", containerConfig.InterfaceName, namespacedName) if err := pc.ofClient.InstallPodFlows( containerConfig.InterfaceName, - containerConfig.IP, + containerConfig.IPs, containerConfig.MAC, pc.gatewayMAC, uint32(containerConfig.OFPort), @@ -486,7 +501,7 @@ func (pc *podConfigurator) connectInterfaceToOVS( } klog.V(2).Infof("Setting up Openflow entries for container %s", containerID) - err = pc.ofClient.InstallPodFlows(ovsPortName, containerConfig.IP, containerConfig.MAC, pc.gatewayMAC, uint32(ofPort)) + err = pc.ofClient.InstallPodFlows(ovsPortName, containerConfig.IPs, containerConfig.MAC, pc.gatewayMAC, uint32(ofPort)) if err != nil { return nil, fmt.Errorf("failed to add Openflow entries for container %s: %v", containerID, err) } @@ -551,11 +566,13 @@ func (pc *podConfigurator) disconnectInterceptedInterface(podName, podNamespace, klog.V(2).Infof("Did not find the port for container %s in local cache", containerID) return nil } - if err := pc.routeClient.UnMigrateRoutesFromGw(&net.IPNet{ - IP: containerConfig.IP, - Mask: net.CIDRMask(32, 32), - }, ""); err != nil { - return fmt.Errorf("connectInterceptedInterface failed to migrate: %w", err) + for _, ip := range containerConfig.IPs { + if err := pc.routeClient.UnMigrateRoutesFromGw(&net.IPNet{ + IP: ip, + Mask: net.CIDRMask(32, 32), + }, ""); err != nil { + return fmt.Errorf("connectInterceptedInterface failed to migrate: %w", err) + } } return pc.disconnectInterfaceFromOVS(containerConfig) // TODO recover pre-connect state? repatch vethpair to original bridge etc ?? to make first CNI happy?? diff --git a/pkg/agent/cniserver/server.go b/pkg/agent/cniserver/server.go index ad817b349af..6efde7a6efe 100644 --- a/pkg/agent/cniserver/server.go +++ b/pkg/agent/cniserver/server.go @@ -219,9 +219,12 @@ func (s *CNIServer) checkRequestMessage(request *cnipb.CniCmdRequest) (*CNIConfi } func (s *CNIServer) updateLocalIPAMSubnet(cniConfig *CNIConfig) { - cniConfig.NetworkConfig.IPAM.Gateway = s.nodeConfig.GatewayConfig.IP.String() - cniConfig.NetworkConfig.IPAM.Subnet = s.nodeConfig.PodCIDR.String() - cniConfig.NetworkConfiguration, _ = json.Marshal(cniConfig.NetworkConfig) + gwIPv4 := util.GetIPv4Addr(s.nodeConfig.GatewayConfig.IPs) + cniConfig.NetworkConfig.IPAM.Gateway = gwIPv4.String() + if s.nodeConfig.PodIPv4CIDR != nil { + cniConfig.NetworkConfig.IPAM.Subnet = s.nodeConfig.PodIPv4CIDR.String() + cniConfig.NetworkConfiguration, _ = json.Marshal(cniConfig.NetworkConfig) + } } func (s *CNIServer) generateCNIErrorResponse(cniErrorCode cnipb.ErrorCode, cniErrorMsg string) *cnipb.CniCmdResponse { @@ -400,7 +403,8 @@ func (s *CNIServer) CmdAdd(ctx context.Context, request *cnipb.CniCmdRequest) (* result.IPs = ipamResult.IPs result.Routes = ipamResult.Routes // Ensure interface gateway setting and mapping relations between result.Interfaces and result.IPs - updateResultIfaceConfig(result, s.nodeConfig.GatewayConfig.IP) + gwIPv4 := util.GetIPv4Addr(s.nodeConfig.GatewayConfig.IPs) + updateResultIfaceConfig(result, gwIPv4) // Setup pod interfaces and connect to ovs bridge podName := string(cniConfig.K8S_POD_NAME) podNamespace := string(cniConfig.K8S_POD_NAMESPACE) diff --git a/pkg/agent/cniserver/server_test.go b/pkg/agent/cniserver/server_test.go index f3dbf667ee8..f759b03ac19 100644 --- a/pkg/agent/cniserver/server_test.go +++ b/pkg/agent/cniserver/server_test.go @@ -19,6 +19,7 @@ import ( "encoding/json" "fmt" "net" + "strings" "testing" cnitypes "github.com/containernetworking/cni/pkg/types" @@ -79,11 +80,11 @@ func TestLoadNetConfig(t *testing.T) { assert.Equal(networkCfg.Name, netCfg.Name) assert.Equal(networkCfg.IPAM.Type, netCfg.IPAM.Type) assert.Equal( - netCfg.IPAM.Subnet, testNodeConfig.PodCIDR.String(), - "Network configuration (PodCIDR) was not updated", + netCfg.IPAM.Subnet, testNodeConfig.PodIPv4CIDR.String(), + "Network configuration (PodCIDRs) was not updated", ) assert.Equal( - netCfg.IPAM.Gateway, testNodeConfig.GatewayConfig.IP.String(), + netCfg.IPAM.Gateway, testNodeConfig.GatewayConfig.IPs[0].String(), "Network configuration (Gateway IP) was not updated", ) } @@ -337,7 +338,7 @@ func TestUpdateResultIfaceConfig(t *testing.T) { // return a Result with 2 v4 addresses. testIps := []string{"10.1.2.100/24, ,4", "192.168.1.100/24, 192.168.2.253, 4"} - require.Equal(gwIP, testNodeConfig.GatewayConfig.IP) + require.Equal(gwIP, testNodeConfig.GatewayConfig.IPs[0]) t.Run("Gateways updated", func(t *testing.T) { assert := assert.New(t) @@ -435,7 +436,7 @@ func TestRemoveInterface(t *testing.T) { podName, testPodNamespace, containerMAC, - containerIP) + []net.IP{containerIP}) containerConfig.OVSPortConfig = &interfacestore.OVSPortConfig{PortUUID: fakePortUUID, OFPort: 0} } @@ -481,11 +482,14 @@ func TestRemoveInterface(t *testing.T) { func TestBuildOVSPortExternalIDs(t *testing.T) { containerID := uuid.New().String() containerMAC, _ := net.ParseMAC("aa:bb:cc:dd:ee:ff") - containerIP := net.ParseIP("10.1.2.100") - containerConfig := interfacestore.NewContainerInterface("pod1-abcd", containerID, "test-1", "t1", containerMAC, containerIP) + containerIP1 := net.ParseIP("10.1.2.100") + containerIP2 := net.ParseIP("2001:fd1a::2") + containerIPs := []net.IP{containerIP1, containerIP2} + containerConfig := interfacestore.NewContainerInterface("pod1-abcd", containerID, "test-1", "t1", containerMAC, containerIPs) externalIds := BuildOVSPortExternalIDs(containerConfig) parsedIP, existed := externalIds[ovsExternalIDIP] - if !existed || parsedIP != "10.1.2.100" { + parsedIPStr := parsedIP.(string) + if !existed || !strings.Contains(parsedIPStr, "10.1.2.100") || !strings.Contains(parsedIPStr, "2001:fd1a::2") { t.Errorf("Failed to parse container configuration") } parsedMac, existed := externalIds[ovsExternalIDMAC] @@ -496,6 +500,31 @@ func TestBuildOVSPortExternalIDs(t *testing.T) { if !existed || parsedID != containerID { t.Errorf("Failed to parse container configuration") } + portExternalIDs := make(map[string]string) + for k, v := range externalIds { + val := v.(string) + portExternalIDs[k] = val + } + mockPort := &ovsconfig.OVSPortData{ + Name: "testPort", + ExternalIDs: portExternalIDs, + } + portConfig := &interfacestore.OVSPortConfig{ + PortUUID: "12345678", + OFPort: int32(1), + } + ifaceConfig := ParseOVSPortInterfaceConfig(mockPort, portConfig) + assert.Equal(t, len(containerIPs), len(ifaceConfig.IPs)) + for _, ip1 := range containerIPs { + existed := false + for _, ip2 := range ifaceConfig.IPs { + if ip2.Equal(ip1) { + existed = true + break + } + } + assert.True(t, existed, fmt.Sprintf("IP %s should exist in the restored InterfaceConfig", ip1.String())) + } } func translateRawPrevResult(prevResult *current.Result, cniVersion string) (map[string]interface{}, error) { @@ -569,6 +598,6 @@ func init() { gwIP = net.ParseIP("192.168.1.1") _, nodePodCIDR, _ := net.ParseCIDR("192.168.1.0/24") gwMAC, _ := net.ParseMAC("00:00:00:00:00:01") - gateway := &config.GatewayConfig{Name: "", IP: gwIP, MAC: gwMAC} - testNodeConfig = &config.NodeConfig{Name: nodeName, PodCIDR: nodePodCIDR, GatewayConfig: gateway} + gateway := &config.GatewayConfig{Name: "", IPs: []net.IP{gwIP}, MAC: gwMAC} + testNodeConfig = &config.NodeConfig{Name: nodeName, PodIPv4CIDR: nodePodCIDR, GatewayConfig: gateway} } diff --git a/pkg/agent/config/node_config.go b/pkg/agent/config/node_config.go index 63306f8a205..08570c51c14 100644 --- a/pkg/agent/config/node_config.go +++ b/pkg/agent/config/node_config.go @@ -45,14 +45,14 @@ const ( type GatewayConfig struct { // Name is the name of host gateway, e.g. antrea-gw0. Name string - IP net.IP + IPs []net.IP MAC net.HardwareAddr // LinkIndex is the link index of host gateway. LinkIndex int } func (g *GatewayConfig) String() string { - return fmt.Sprintf("Name %s: IP %s, MAC %s", g.Name, g.IP, g.MAC) + return fmt.Sprintf("Name %s: IP %s, MAC %s", g.Name, g.IPs, g.MAC) } type AdapterNetConfig struct { @@ -74,9 +74,12 @@ type NodeConfig struct { // The name of the default tunnel interface. Defaults to "antrea-tun0", but can // be overridden by the discovered tunnel interface name from the OVS bridge. DefaultTunName string - // The CIDR block to allocate Pod IPs out of. - // It's nil for the networkPolicyOnly trafficEncapMode which doesn't do IPAM. - PodCIDR *net.IPNet + // The CIDR block from which to allocate IPv4 address to Pod. + // It's nil for the net workPolicyOnly trafficEncapMode which doesn't do IPAM. + PodIPv4CIDR *net.IPNet + // The CIDR block from where to allocate IPv6 address to Pod. + // It's nil for the net workPolicyOnly trafficEncapMode which doesn't do IPAM. + PodIPv6CIDR *net.IPNet // The Node's IP used in Kubernetes. It has the network mask information. NodeIPAddr *net.IPNet // Set either via defaultMTU config in antrea.yaml or auto discovered. @@ -90,8 +93,8 @@ type NodeConfig struct { } func (n *NodeConfig) String() string { - return fmt.Sprintf("NodeName: %s, OVSBridge: %s, PodCIDR: %s, NodeIP: %s, Gateway: %s", - n.Name, n.OVSBridge, n.PodCIDR, n.NodeIPAddr, n.GatewayConfig) + return fmt.Sprintf("NodeName: %s, OVSBridge: %s, PodIPv4CIDR %s, PodIPv6CIDR: %s, NodeIP: %s, Gateway: %s", + n.Name, n.OVSBridge, n.PodIPv4CIDR, n.PodIPv6CIDR, n.NodeIPAddr, n.GatewayConfig) } // User provided network configuration parameters. diff --git a/pkg/agent/controller/networkpolicy/reconciler.go b/pkg/agent/controller/networkpolicy/reconciler.go index 6c220c2358f..5536ee4dc09 100644 --- a/pkg/agent/controller/networkpolicy/reconciler.go +++ b/pkg/agent/controller/networkpolicy/reconciler.go @@ -746,8 +746,9 @@ func (r *reconciler) getPodIPs(members v1beta2.GroupMemberSet) sets.String { continue } for _, iface := range ifaces { - klog.V(2).Infof("Got IP %v for Pod %s/%s", iface.IP, m.Pod.Namespace, m.Pod.Name) - ips.Insert(iface.IP.String()) + ipv4Addr := iface.GetIPv4Addr() + klog.V(2).Infof("Got IP %v for Pod %s/%s", ipv4Addr, m.Pod.Namespace, m.Pod.Name) + ips.Insert(ipv4Addr.String()) } } return ips diff --git a/pkg/agent/controller/networkpolicy/reconciler_test.go b/pkg/agent/controller/networkpolicy/reconciler_test.go index c6977d47137..9e4237051e4 100644 --- a/pkg/agent/controller/networkpolicy/reconciler_test.go +++ b/pkg/agent/controller/networkpolicy/reconciler_test.go @@ -184,13 +184,13 @@ func TestReconcilerReconcile(t *testing.T) { ifaceStore := interfacestore.NewInterfaceStore() ifaceStore.AddInterface(&interfacestore.InterfaceConfig{ InterfaceName: util.GenerateContainerInterfaceName("pod1", "ns1", "container1"), - IP: net.ParseIP("2.2.2.2"), + IPs: []net.IP{net.ParseIP("2.2.2.2")}, ContainerInterfaceConfig: &interfacestore.ContainerInterfaceConfig{PodName: "pod1", PodNamespace: "ns1", ContainerID: "container1"}, OVSPortConfig: &interfacestore.OVSPortConfig{OFPort: 1}, }) ifaceStore.AddInterface(&interfacestore.InterfaceConfig{ InterfaceName: util.GenerateContainerInterfaceName("pod3", "ns1", "container3"), - IP: net.ParseIP("3.3.3.3"), + IPs: []net.IP{net.ParseIP("3.3.3.3")}, ContainerInterfaceConfig: &interfacestore.ContainerInterfaceConfig{PodName: "pod3", PodNamespace: "ns1", ContainerID: "container3"}, OVSPortConfig: &interfacestore.OVSPortConfig{OFPort: 3}, }) @@ -523,13 +523,13 @@ func TestReconcilerBatchReconcile(t *testing.T) { ifaceStore := interfacestore.NewInterfaceStore() ifaceStore.AddInterface(&interfacestore.InterfaceConfig{ InterfaceName: util.GenerateContainerInterfaceName("pod1", "ns1", "container1"), - IP: net.ParseIP("2.2.2.2"), + IPs: []net.IP{net.ParseIP("2.2.2.2")}, ContainerInterfaceConfig: &interfacestore.ContainerInterfaceConfig{PodName: "pod1", PodNamespace: "ns1", ContainerID: "container1"}, OVSPortConfig: &interfacestore.OVSPortConfig{OFPort: 1}, }) ifaceStore.AddInterface(&interfacestore.InterfaceConfig{ InterfaceName: util.GenerateContainerInterfaceName("pod3", "ns1", "container3"), - IP: net.ParseIP("3.3.3.3"), + IPs: []net.IP{net.ParseIP("3.3.3.3")}, ContainerInterfaceConfig: &interfacestore.ContainerInterfaceConfig{PodName: "pod3", PodNamespace: "ns1", ContainerID: "container3"}, OVSPortConfig: &interfacestore.OVSPortConfig{OFPort: 3}, }) @@ -648,13 +648,13 @@ func TestReconcilerUpdate(t *testing.T) { ifaceStore.AddInterface( &interfacestore.InterfaceConfig{ InterfaceName: util.GenerateContainerInterfaceName("pod1", "ns1", "container1"), - IP: net.ParseIP("2.2.2.2"), + IPs: []net.IP{net.ParseIP("2.2.2.2")}, ContainerInterfaceConfig: &interfacestore.ContainerInterfaceConfig{PodName: "pod1", PodNamespace: "ns1", ContainerID: "container1"}, OVSPortConfig: &interfacestore.OVSPortConfig{OFPort: 1}}) ifaceStore.AddInterface( &interfacestore.InterfaceConfig{ InterfaceName: util.GenerateContainerInterfaceName("pod2", "ns1", "container2"), - IP: net.ParseIP("3.3.3.3"), + IPs: []net.IP{net.ParseIP("3.3.3.3")}, ContainerInterfaceConfig: &interfacestore.ContainerInterfaceConfig{PodName: "pod2", PodNamespace: "ns1", ContainerID: "container2"}, OVSPortConfig: &interfacestore.OVSPortConfig{OFPort: 2}}) ifaceStore.AddInterface( diff --git a/pkg/agent/controller/traceflow/traceflow_controller.go b/pkg/agent/controller/traceflow/traceflow_controller.go index 644b0930f2a..33dcc177df6 100644 --- a/pkg/agent/controller/traceflow/traceflow_controller.go +++ b/pkg/agent/controller/traceflow/traceflow_controller.go @@ -334,7 +334,7 @@ func (c *Controller) injectPacket(tf *opsv1alpha1.Traceflow) error { dstPodInterfaces := c.interfaceStore.GetContainerInterfacesByPod(tf.Spec.Destination.Pod, tf.Spec.Destination.Namespace) if len(dstPodInterfaces) > 0 { dstMAC = dstPodInterfaces[0].MAC.String() - dstIP = dstPodInterfaces[0].IP.String() + dstIP = dstPodInterfaces[0].GetIPv4Addr().String() } else { dstPod, err := c.kubeClient.CoreV1().Pods(tf.Spec.Destination.Namespace).Get(context.TODO(), tf.Spec.Destination.Pod, metav1.GetOptions{}) if err != nil { @@ -388,7 +388,7 @@ func (c *Controller) injectPacket(tf *opsv1alpha1.Traceflow) error { tf.Status.DataplaneTag, podInterfaces[0].MAC.String(), dstMAC, - podInterfaces[0].IP.String(), + podInterfaces[0].GetIPv4Addr().String(), dstIP, uint8(tf.Spec.Packet.IPHeader.Protocol), uint8(tf.Spec.Packet.IPHeader.TTL), diff --git a/pkg/agent/flowexporter/connections/connections_test.go b/pkg/agent/flowexporter/connections/connections_test.go index 2c889f4ee79..5266b3b0fd0 100644 --- a/pkg/agent/flowexporter/connections/connections_test.go +++ b/pkg/agent/flowexporter/connections/connections_test.go @@ -123,7 +123,7 @@ func TestConnectionStore_addAndUpdateConn(t *testing.T) { } interfaceFlow2 := &interfacestore.InterfaceConfig{ InterfaceName: "interface2", - IP: net.IP{8, 7, 6, 5}, + IPs: []net.IP{{8, 7, 6, 5}}, ContainerInterfaceConfig: podConfigFlow2, } servicePortName := k8sproxy.ServicePortName{ diff --git a/pkg/agent/flowexporter/connections/conntrack.go b/pkg/agent/flowexporter/connections/conntrack.go index 33fc086e0e5..3af25915eae 100644 --- a/pkg/agent/flowexporter/connections/conntrack.go +++ b/pkg/agent/flowexporter/connections/conntrack.go @@ -37,6 +37,7 @@ func InitializeConnTrackDumper(nodeConfig *config.NodeConfig, serviceCIDR *net.I func filterAntreaConns(conns []*flowexporter.Connection, nodeConfig *config.NodeConfig, serviceCIDR *net.IPNet, zoneFilter uint16, isAntreaProxyEnabled bool) []*flowexporter.Connection { filteredConns := conns[:0] +conLoop: for _, conn := range conns { if conn.Zone != zoneFilter { continue @@ -44,10 +45,12 @@ func filterAntreaConns(conns []*flowexporter.Connection, nodeConfig *config.Node srcIP := conn.TupleOrig.SourceAddress dstIP := conn.TupleReply.SourceAddress - // Consider Pod-to-Pod, Pod-To-Service and Pod-To-External flows. - if srcIP.Equal(nodeConfig.GatewayConfig.IP) || dstIP.Equal(nodeConfig.GatewayConfig.IP) { - klog.V(4).Infof("Detected flow for which one of the endpoint is host gateway %s :%+v", nodeConfig.GatewayConfig.IP.String(), conn) - continue + // Only get Pod-to-Pod flows. + for _, ip := range nodeConfig.GatewayConfig.IPs { + if srcIP.Equal(ip) || dstIP.Equal(ip) { + klog.V(4).Infof("Detected flow for which one of the endpoint is host gateway %s :%+v", ip.String(), conn) + continue conLoop + } } if !isAntreaProxyEnabled { diff --git a/pkg/agent/flowexporter/connections/conntrack_linux_test.go b/pkg/agent/flowexporter/connections/conntrack_linux_test.go index 4efbccea3d8..d6cb8adf6f1 100644 --- a/pkg/agent/flowexporter/connections/conntrack_linux_test.go +++ b/pkg/agent/flowexporter/connections/conntrack_linux_test.go @@ -67,7 +67,7 @@ func TestConnTrackSystem_DumpFlows(t *testing.T) { // Create nodeConfig and gateWayConfig // Set antreaGWFlow.TupleOrig.IP.DestinationAddress as gateway IP gwConfig := &config.GatewayConfig{ - IP: net.IP{8, 7, 6, 5}, + IPs: []net.IP{{8, 7, 6, 5}}, } nodeConfig := &config.NodeConfig{ GatewayConfig: gwConfig, @@ -107,7 +107,7 @@ func TestConnTrackOvsAppCtl_DumpFlows(t *testing.T) { // Create nodeConfig and gateWayConfig // Set antreaGWFlow.TupleOrig.IP.DestinationAddress as gateway IP gwConfig := &config.GatewayConfig{ - IP: net.IP{8, 7, 6, 5}, + IPs: []net.IP{{8, 7, 6, 5}}, } nodeConfig := &config.NodeConfig{ GatewayConfig: gwConfig, diff --git a/pkg/agent/interfacestore/interface_cache.go b/pkg/agent/interfacestore/interface_cache.go index 23970c3c192..d60614bb0a1 100644 --- a/pkg/agent/interfacestore/interface_cache.go +++ b/pkg/agent/interfacestore/interface_cache.go @@ -245,11 +245,15 @@ func podIndexFunc(obj interface{}) ([]string, error) { func interfaceIPIndexFunc(obj interface{}) ([]string, error) { interfaceConfig := obj.(*InterfaceConfig) - if interfaceConfig.IP == nil { + if interfaceConfig.IPs == nil { // If interfaceConfig IP is not set, we return empty key. return []string{}, nil } - return []string{interfaceConfig.IP.String()}, nil + var intfIPs []string + for _, ip := range interfaceConfig.IPs { + intfIPs = append(intfIPs, ip.String()) + } + return intfIPs, nil } func NewInterfaceStore() InterfaceStore { diff --git a/pkg/agent/interfacestore/types.go b/pkg/agent/interfacestore/types.go index 889b995ffb6..c98a0c2f11c 100644 --- a/pkg/agent/interfacestore/types.go +++ b/pkg/agent/interfacestore/types.go @@ -18,6 +18,7 @@ import ( "net" "strconv" + "github.com/vmware-tanzu/antrea/pkg/agent/util" "github.com/vmware-tanzu/antrea/pkg/ovs/ovsconfig" ) @@ -67,7 +68,7 @@ type InterfaceConfig struct { Type InterfaceType // Unique name of the interface, also used for the OVS port name. InterfaceName string - IP net.IP + IPs []net.IP MAC net.HardwareAddr *OVSPortConfig *ContainerInterfaceConfig @@ -99,7 +100,7 @@ func NewContainerInterface( podName string, podNamespace string, mac net.HardwareAddr, - ip net.IP) *InterfaceConfig { + ips []net.IP) *InterfaceConfig { containerConfig := &ContainerInterfaceConfig{ ContainerID: containerID, PodName: podName, @@ -107,7 +108,7 @@ func NewContainerInterface( return &InterfaceConfig{ InterfaceName: interfaceName, Type: ContainerInterface, - IP: ip, + IPs: ips, MAC: mac, ContainerInterfaceConfig: containerConfig} } @@ -137,3 +138,8 @@ func NewUplinkInterface(uplinkName string) *InterfaceConfig { uplinkConfig := &InterfaceConfig{InterfaceName: uplinkName, Type: UplinkInterface} return uplinkConfig } + +// TODO: remove this method after IPv4/IPv6 dual-stack is supported completely. +func (c *InterfaceConfig) GetIPv4Addr() net.IP { + return util.GetIPv4Addr(c.IPs) +} diff --git a/pkg/agent/openflow/client.go b/pkg/agent/openflow/client.go index 2921fce1832..b13287e7181 100644 --- a/pkg/agent/openflow/client.go +++ b/pkg/agent/openflow/client.go @@ -25,6 +25,7 @@ import ( "github.com/vmware-tanzu/antrea/pkg/agent/config" "github.com/vmware-tanzu/antrea/pkg/agent/openflow/cookie" "github.com/vmware-tanzu/antrea/pkg/agent/types" + "github.com/vmware-tanzu/antrea/pkg/agent/util" "github.com/vmware-tanzu/antrea/pkg/apis/controlplane/v1beta2" binding "github.com/vmware-tanzu/antrea/pkg/ovs/openflow" "github.com/vmware-tanzu/antrea/third_party/proxy" @@ -42,7 +43,7 @@ type Client interface { Initialize(roundInfo types.RoundInfo, config *config.NodeConfig, encapMode config.TrafficEncapModeType, gatewayOFPort uint32) (<-chan struct{}, error) // InstallGatewayFlows sets up flows related to an OVS gateway port, the gateway must exist. - InstallGatewayFlows(gatewayAddr net.IP, gatewayMAC net.HardwareAddr, gatewayOFPort uint32) error + InstallGatewayFlows(gatewayAddrs []net.IP, gatewayMAC net.HardwareAddr, gatewayOFPort uint32) error // InstallBridgeUplinkFlows installs Openflow flows between bridge local port and uplink port to support // host networking. These flows are only needed on windows platform. @@ -85,7 +86,7 @@ type Client interface { // flows will be installed). Calls to InstallPodFlows are idempotent. Concurrent calls // to InstallPodFlows and / or UninstallPodFlows are supported as long as they are all // for different interfaceNames. - InstallPodFlows(interfaceName string, podInterfaceIP net.IP, podInterfaceMAC, gatewayMAC net.HardwareAddr, ofPort uint32) error + InstallPodFlows(interfaceName string, podInterfaceIPs []net.IP, podInterfaceMAC, gatewayMAC net.HardwareAddr, ofPort uint32) error // UninstallPodFlows removes the connection to the local Pod specified with the // interfaceName. UninstallPodFlows will do nothing if no connection to the Pod was established. @@ -318,21 +319,27 @@ func (c *client) UninstallNodeFlows(hostname string) error { return c.deleteFlows(c.nodeFlowCache, hostname) } -func (c *client) InstallPodFlows(interfaceName string, podInterfaceIP net.IP, podInterfaceMAC, gatewayMAC net.HardwareAddr, ofPort uint32) error { +func (c *client) InstallPodFlows(interfaceName string, podInterfaceIPs []net.IP, podInterfaceMAC, gatewayMAC net.HardwareAddr, ofPort uint32) error { c.replayMutex.RLock() defer c.replayMutex.RUnlock() flows := []binding.Flow{ c.podClassifierFlow(ofPort, cookie.Pod), - c.podIPSpoofGuardFlow(podInterfaceIP, podInterfaceMAC, ofPort, cookie.Pod), - c.arpSpoofGuardFlow(podInterfaceIP, podInterfaceMAC, ofPort, cookie.Pod), c.l2ForwardCalcFlow(podInterfaceMAC, ofPort, cookie.Pod), - c.l3FlowsToPod(gatewayMAC, podInterfaceIP, podInterfaceMAC, cookie.Pod), } + // Add support for IPv4 ARP responder. + podInterfaceIPv4 := util.GetIPv4Addr(podInterfaceIPs) + if podInterfaceIPv4 != nil { + flows = append(flows, c.arpSpoofGuardFlow(podInterfaceIPv4, podInterfaceMAC, ofPort, cookie.Pod)) + } + // Add IP SpoofGuard flows for all validate IPs. + flows = append(flows, c.podIPSpoofGuardFlow(podInterfaceIPs, podInterfaceMAC, ofPort, cookie.Pod)...) + // Add L3 Routing flows to rewrite Pod's dst MAC for all validate IPs. + flows = append(flows, c.l3FlowsToPod(gatewayMAC, podInterfaceIPs, podInterfaceMAC, cookie.Pod)...) if c.encapMode.IsNetworkPolicyOnly() { // In policy-only mode, traffic to local Pod is routed based on destination IP. flows = append(flows, - c.l3ToPodFlow(podInterfaceIP, podInterfaceMAC, cookie.Pod), + c.l3ToPodFlow(podInterfaceIPs, podInterfaceMAC, cookie.Pod)..., ) } return c.addFlows(c.podFlowCache, interfaceName, flows) @@ -469,19 +476,25 @@ func (c *client) InstallClusterServiceCIDRFlows(serviceNet *net.IPNet, gatewayOF return nil } -func (c *client) InstallGatewayFlows(gatewayAddr net.IP, gatewayMAC net.HardwareAddr, gatewayOFPort uint32) error { +func (c *client) InstallGatewayFlows(gatewayAddrs []net.IP, gatewayMAC net.HardwareAddr, gatewayOFPort uint32) error { flows := []binding.Flow{ c.gatewayClassifierFlow(gatewayOFPort, cookie.Default), c.gatewayIPSpoofGuardFlow(gatewayOFPort, cookie.Default), - c.gatewayARPSpoofGuardFlow(gatewayOFPort, gatewayAddr, gatewayMAC, cookie.Default), c.ctRewriteDstMACFlow(gatewayMAC, cookie.Default), c.l2ForwardCalcFlow(gatewayMAC, gatewayOFPort, cookie.Default), - c.localProbeFlow(gatewayAddr, cookie.Default), } + // Add ARP SpoofGuard flow for local gateway interface. + gwIPv4 := util.GetIPv4Addr(gatewayAddrs) + if gwIPv4 != nil { + flows = append(flows, c.gatewayARPSpoofGuardFlow(gatewayOFPort, gwIPv4, gatewayMAC, cookie.Default)) + } + // Add flow to ensure the liveness check packet could be forwarded correctly. + flows = append(flows, c.localProbeFlow(gatewayAddrs, cookie.Default)...) + // In NoEncap , no traffic from tunnel port if c.encapMode.SupportsEncap() { - flows = append(flows, c.l3ToGatewayFlow(gatewayAddr, gatewayMAC, cookie.Default)) + flows = append(flows, c.l3ToGatewayFlow(gatewayAddrs, gatewayMAC, cookie.Default)...) } if err := c.ofEntryOperations.AddAll(flows); err != nil { diff --git a/pkg/agent/openflow/client_test.go b/pkg/agent/openflow/client_test.go index 70a0c6ec366..1dc33b3c38c 100644 --- a/pkg/agent/openflow/client_test.go +++ b/pkg/agent/openflow/client_test.go @@ -59,7 +59,7 @@ func installPodFlows(ofClient Client, cacheKey string) (int, error) { podMAC, _ := net.ParseMAC("AA:BB:CC:DD:EE:EE") podIP := net.ParseIP("10.0.0.2") ofPort := uint32(10) - err := ofClient.InstallPodFlows(containerID, podIP, podMAC, gwMAC, ofPort) + err := ofClient.InstallPodFlows(containerID, []net.IP{podIP}, podMAC, gwMAC, ofPort) client := ofClient.(*client) fCacheI, ok := client.podFlowCache.Load(containerID) if ok { diff --git a/pkg/agent/openflow/pipeline.go b/pkg/agent/openflow/pipeline.go index 7a9d224d179..b0531b0655d 100644 --- a/pkg/agent/openflow/pipeline.go +++ b/pkg/agent/openflow/pipeline.go @@ -694,36 +694,45 @@ func (c *client) l2ForwardOutputServiceHairpinFlow() binding.Flow { } // l3FlowsToPod generates the flow to rewrite MAC if the packet is received from tunnel port and destined for local Pods. -func (c *client) l3FlowsToPod(localGatewayMAC net.HardwareAddr, podInterfaceIP net.IP, podInterfaceMAC net.HardwareAddr, category cookie.Category) binding.Flow { +func (c *client) l3FlowsToPod(localGatewayMAC net.HardwareAddr, podInterfaceIPs []net.IP, podInterfaceMAC net.HardwareAddr, category cookie.Category) []binding.Flow { l3FwdTable := c.pipeline[l3ForwardingTable] - flowBuilder := l3FwdTable.BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIP) - if c.enableProxy { - flowBuilder = flowBuilder.MatchRegRange(int(marksReg), macRewriteMark, macRewriteMarkRange) - } else { - flowBuilder = flowBuilder.MatchDstMAC(globalVirtualMAC) + var flows []binding.Flow + for _, ip := range podInterfaceIPs { + ipProtocol := parseIPProtocol(ip) + flowBuilder := l3FwdTable.BuildFlow(priorityNormal).MatchProtocol(ipProtocol) + if c.enableProxy { + flowBuilder = flowBuilder.MatchRegRange(int(marksReg), macRewriteMark, macRewriteMarkRange) + } else { + flowBuilder = flowBuilder.MatchDstMAC(globalVirtualMAC) + } + // Rewrite src MAC to local gateway MAC, and rewrite dst MAC to pod MAC + flows = append(flows, flowBuilder.MatchDstIP(ip). + Action().SetSrcMAC(localGatewayMAC). + Action().SetDstMAC(podInterfaceMAC). + Action().DecTTL(). + Action().GotoTable(l3FwdTable.GetNext()). + Cookie(c.cookieAllocator.Request(category).Raw()). + Done()) } - // Rewrite src MAC to local gateway MAC, and rewrite dst MAC to pod MAC - return flowBuilder. - MatchDstIP(podInterfaceIP). - Action().SetSrcMAC(localGatewayMAC). - Action().SetDstMAC(podInterfaceMAC). - Action().DecTTL(). - Action().GotoTable(l3FwdTable.GetNext()). - Cookie(c.cookieAllocator.Request(category).Raw()). - Done() + return flows } // l3ToPodFromGwFlow generates the flow to rewrite MAC if the packet IP matches an local IP. // This flow is used in policy only traffic mode. -func (c *client) l3ToPodFlow(podInterfaceIP net.IP, podInterfaceMAC net.HardwareAddr, category cookie.Category) binding.Flow { +func (c *client) l3ToPodFlow(podInterfaceIPs []net.IP, podInterfaceMAC net.HardwareAddr, category cookie.Category) []binding.Flow { l3FwdTable := c.pipeline[l3ForwardingTable] - return l3FwdTable.BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIP). - MatchDstIP(podInterfaceIP). - Action().SetDstMAC(podInterfaceMAC). - Action().DecTTL(). - Action().GotoTable(l3FwdTable.GetNext()). - Cookie(c.cookieAllocator.Request(category).Raw()). - Done() + var flows []binding.Flow + for _, ip := range podInterfaceIPs { + ipProtocol := parseIPProtocol(ip) + flows = append(flows, l3FwdTable.BuildFlow(priorityNormal).MatchProtocol(ipProtocol). + MatchDstIP(ip). + Action().SetDstMAC(podInterfaceMAC). + Action().DecTTL(). + Action().GotoTable(l3FwdTable.GetNext()). + Cookie(c.cookieAllocator.Request(category).Raw()). + Done()) + } + return flows } // l3ToGWFlow generates the flow to rewrite MAC to gw port if the packet received is unmatched by local Pod flows. @@ -739,15 +748,20 @@ func (c *client) l3ToGWFlow(gwMAC net.HardwareAddr, category cookie.Category) bi } // l3ToGatewayFlow generates flow that rewrites MAC of the packet received from tunnel port and destined to local gateway. -func (c *client) l3ToGatewayFlow(localGatewayIP net.IP, localGatewayMAC net.HardwareAddr, category cookie.Category) binding.Flow { +func (c *client) l3ToGatewayFlow(localGatewayIPs []net.IP, localGatewayMAC net.HardwareAddr, category cookie.Category) []binding.Flow { l3FwdTable := c.pipeline[l3ForwardingTable] - return l3FwdTable.BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIP). - MatchDstMAC(globalVirtualMAC). - MatchDstIP(localGatewayIP). - Action().SetDstMAC(localGatewayMAC). - Action().GotoTable(l3FwdTable.GetNext()). - Cookie(c.cookieAllocator.Request(category).Raw()). - Done() + var flows []binding.Flow + for _, ip := range localGatewayIPs { + ipProtocol := parseIPProtocol(ip) + flows = append(flows, l3FwdTable.BuildFlow(priorityNormal).MatchProtocol(ipProtocol). + MatchDstMAC(globalVirtualMAC). + MatchDstIP(ip). + Action().SetDstMAC(localGatewayMAC). + Action().GotoTable(l3FwdTable.GetNext()). + Cookie(c.cookieAllocator.Request(category).Raw()). + Done()) + } + return flows } // l3FwdFlowToRemote generates the L3 forward flow on source node to support traffic to remote pods/gateway. @@ -830,16 +844,32 @@ func (c *client) arpResponderStaticFlow(category cookie.Category) binding.Flow { // podIPSpoofGuardFlow generates the flow to check IP traffic sent out from local pod. Traffic from host gateway interface // will not be checked, since it might be pod to service traffic or host namespace traffic. -func (c *client) podIPSpoofGuardFlow(ifIP net.IP, ifMAC net.HardwareAddr, ifOFPort uint32, category cookie.Category) binding.Flow { +func (c *client) podIPSpoofGuardFlow(ifIPs []net.IP, ifMAC net.HardwareAddr, ifOFPort uint32, category cookie.Category) []binding.Flow { ipPipeline := c.pipeline ipSpoofGuardTable := ipPipeline[spoofGuardTable] - return ipSpoofGuardTable.BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIP). - MatchInPort(ifOFPort). - MatchSrcMAC(ifMAC). - MatchSrcIP(ifIP). - Action().GotoTable(ipSpoofGuardTable.GetNext()). - Cookie(c.cookieAllocator.Request(category).Raw()). - Done() + var flows []binding.Flow + for _, ifIP := range ifIPs { + ipProtocol := parseIPProtocol(ifIP) + flow := ipSpoofGuardTable.BuildFlow(priorityNormal).MatchProtocol(ipProtocol). + MatchInPort(ifOFPort). + MatchSrcMAC(ifMAC). + MatchSrcIP(ifIP). + Action().GotoTable(ipSpoofGuardTable.GetNext()). + Cookie(c.cookieAllocator.Request(category).Raw()). + Done() + flows = append(flows, flow) + } + return flows +} + +func parseIPProtocol(ip net.IP) binding.Protocol { + var ipProtocol binding.Protocol + if ip.To4() != nil { + ipProtocol = binding.ProtocolIP + } else { + ipProtocol = binding.ProtocolIPv6 + } + return ipProtocol } // serviceHairpinResponseDNATFlow generates the flow which transforms destination @@ -1178,13 +1208,18 @@ func (c *client) defaultDropFlow(tableID binding.TableIDType, matchKey int, matc } // localProbeFlow generates the flow to forward packets to conntrackCommitTable. The packets are sent from Node to probe the liveness/readiness of local Pods. -func (c *client) localProbeFlow(localGatewayIP net.IP, category cookie.Category) binding.Flow { - return c.pipeline[IngressRuleTable].BuildFlow(priorityHigh). - MatchProtocol(binding.ProtocolIP). - MatchSrcIP(localGatewayIP). - Action().GotoTable(conntrackCommitTable). - Cookie(c.cookieAllocator.Request(category).Raw()). - Done() +func (c *client) localProbeFlow(localGatewayIPs []net.IP, category cookie.Category) []binding.Flow { + var flows []binding.Flow + for _, ip := range localGatewayIPs { + ipProtocol := parseIPProtocol(ip) + flows = append(flows, c.pipeline[IngressRuleTable].BuildFlow(priorityHigh). + MatchProtocol(ipProtocol). + MatchSrcIP(ip). + Action().GotoTable(conntrackCommitTable). + Cookie(c.cookieAllocator.Request(category).Raw()). + Done()) + } + return flows } func (c *client) bridgeAndUplinkFlows(uplinkOfport uint32, bridgeLocalPort uint32, nodeIP net.IP, localSubnet net.IPNet, category cookie.Category) []binding.Flow { diff --git a/pkg/agent/openflow/testing/mock_openflow.go b/pkg/agent/openflow/testing/mock_openflow.go index a150a6a0b67..603130f3edc 100644 --- a/pkg/agent/openflow/testing/mock_openflow.go +++ b/pkg/agent/openflow/testing/mock_openflow.go @@ -323,7 +323,7 @@ func (mr *MockClientMockRecorder) InstallExternalFlows(arg0, arg1 interface{}) * } // InstallGatewayFlows mocks base method -func (m *MockClient) InstallGatewayFlows(arg0 net.IP, arg1 net.HardwareAddr, arg2 uint32) error { +func (m *MockClient) InstallGatewayFlows(arg0 []net.IP, arg1 net.HardwareAddr, arg2 uint32) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "InstallGatewayFlows", arg0, arg1, arg2) ret0, _ := ret[0].(error) @@ -365,7 +365,7 @@ func (mr *MockClientMockRecorder) InstallNodeFlows(arg0, arg1, arg2, arg3, arg4, } // InstallPodFlows mocks base method -func (m *MockClient) InstallPodFlows(arg0 string, arg1 net.IP, arg2, arg3 net.HardwareAddr, arg4 uint32) error { +func (m *MockClient) InstallPodFlows(arg0 string, arg1 []net.IP, arg2, arg3 net.HardwareAddr, arg4 uint32) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "InstallPodFlows", arg0, arg1, arg2, arg3, arg4) ret0, _ := ret[0].(error) diff --git a/pkg/agent/querier/querier.go b/pkg/agent/querier/querier.go index 34a24795f5a..101c25cb070 100644 --- a/pkg/agent/querier/querier.go +++ b/pkg/agent/querier/querier.go @@ -193,8 +193,8 @@ func (aq agentQuerier) GetAgentInfo(agentInfo *v1beta1.AntreaAgentInfo, partial agentInfo.Version = querier.GetVersion() agentInfo.PodRef = querier.GetSelfPod() agentInfo.NodeRef = querier.GetSelfNode(true, aq.nodeConfig.Name) - if aq.nodeConfig.PodCIDR != nil { - agentInfo.NodeSubnet = []string{aq.nodeConfig.PodCIDR.String()} + if aq.nodeConfig.PodIPv4CIDR != nil { + agentInfo.NodeSubnet = append(agentInfo.NodeSubnet, aq.nodeConfig.PodIPv4CIDR.String()) } agentInfo.OVSInfo.BridgeName = aq.nodeConfig.OVSBridge agentInfo.APIPort = aq.apiPort diff --git a/pkg/agent/querier/querier_test.go b/pkg/agent/querier/querier_test.go index 6f6d57f9610..5460de4b52f 100644 --- a/pkg/agent/querier/querier_test.go +++ b/pkg/agent/querier/querier_test.go @@ -120,10 +120,10 @@ func TestAgentQuerierGetAgentInfo(t *testing.T) { { name: "encap-mode non-partial", nodeConfig: &config.NodeConfig{ - Name: "foo", - OVSBridge: "br-int", - NodeIPAddr: getIPNet("10.10.0.10"), - PodCIDR: getIPNet("20.20.20.0/24"), + Name: "foo", + OVSBridge: "br-int", + NodeIPAddr: getIPNet("10.10.0.10"), + PodIPv4CIDR: getIPNet("20.20.20.0/24"), }, apiPort: 10350, partial: false, diff --git a/pkg/agent/route/route_linux.go b/pkg/agent/route/route_linux.go index f9a13ee38df..ba475da1fb6 100644 --- a/pkg/agent/route/route_linux.go +++ b/pkg/agent/route/route_linux.go @@ -115,7 +115,7 @@ func (c *Client) initIPSet() error { return err } // Ensure its own PodCIDR is in it. - if err := ipset.AddEntry(antreaPodIPSet, c.nodeConfig.PodCIDR.String()); err != nil { + if err := ipset.AddEntry(antreaPodIPSet, c.nodeConfig.PodIPv4CIDR.String()); err != nil { return err } @@ -244,7 +244,7 @@ func (c *Client) initIPTables() error { writeLine(iptablesData, []string{ "-A", antreaPostRoutingChain, "-m", "comment", "--comment", `"Antrea: masquerade pod to external packets"`, - "-s", c.nodeConfig.PodCIDR.String(), "-m", "set", "!", "--match-set", antreaPodIPSet, "dst", + "-s", c.nodeConfig.PodIPv4CIDR.String(), "-m", "set", "!", "--match-set", antreaPodIPSet, "dst", "-j", iptables.MasqueradeTarget, }...) } @@ -318,7 +318,7 @@ func (c *Client) Reconcile(podCIDRs []string, remoteNodeIPs []string) error { return fmt.Errorf("error listing ip routes: %v", err) } for _, route := range routes { - if reflect.DeepEqual(route.Dst, c.nodeConfig.PodCIDR) { + if reflect.DeepEqual(route.Dst, c.nodeConfig.PodIPv4CIDR) || reflect.DeepEqual(route.Dst, c.nodeConfig.PodIPv6CIDR) { continue } if desiredPodCIDRs.Has(route.Dst.String()) { diff --git a/pkg/agent/route/route_windows.go b/pkg/agent/route/route_windows.go index 1c1b8aaf502..bfe8a73567e 100644 --- a/pkg/agent/route/route_windows.go +++ b/pkg/agent/route/route_windows.go @@ -189,11 +189,14 @@ func (c *Client) listRoutes() (map[string]*netroute.Route, error) { // initFwRules adds Windows Firewall rules to accept the traffic that is sent to or from local Pods. func (c *Client) initFwRules() error { - err := c.fwClient.AddRuleAllowIP(inboundFirewallRuleName, winfirewall.FWRuleIn, c.nodeConfig.PodCIDR) + if c.nodeConfig.PodIPv4CIDR == nil { + return errors.New("no valid IPv4 PodCIDR") + } + err := c.fwClient.AddRuleAllowIP(inboundFirewallRuleName, winfirewall.FWRuleIn, c.nodeConfig.PodIPv4CIDR) if err != nil { return err } - err = c.fwClient.AddRuleAllowIP(outboundFirewallRuleName, winfirewall.FWRuleOut, c.nodeConfig.PodCIDR) + err = c.fwClient.AddRuleAllowIP(outboundFirewallRuleName, winfirewall.FWRuleOut, c.nodeConfig.PodIPv4CIDR) if err != nil { return err } diff --git a/pkg/agent/util/net.go b/pkg/agent/util/net.go index bd0cb9ea3c2..d1bf14dc0a2 100644 --- a/pkg/agent/util/net.go +++ b/pkg/agent/util/net.go @@ -118,3 +118,12 @@ func GetIPNetDeviceFromIP(localIP net.IP) (*net.IPNet, *net.Interface, error) { } return nil, nil, fmt.Errorf("unable to find local IP and device") } + +func GetIPv4Addr(ips []net.IP) net.IP { + for _, ip := range ips { + if ip.To4() != nil { + return ip + } + } + return nil +} diff --git a/pkg/ovs/openflow/interfaces.go b/pkg/ovs/openflow/interfaces.go index 9e8acca1678..562eb119612 100644 --- a/pkg/ovs/openflow/interfaces.go +++ b/pkg/ovs/openflow/interfaces.go @@ -35,12 +35,17 @@ const ( ) const ( - ProtocolIP Protocol = "ip" - ProtocolARP Protocol = "arp" - ProtocolTCP Protocol = "tcp" - ProtocolUDP Protocol = "udp" - ProtocolSCTP Protocol = "sctp" - ProtocolICMP Protocol = "icmp" + ProtocolIP Protocol = "ip" + ProtocolIPv6 Protocol = "ipv6" + ProtocolARP Protocol = "arp" + ProtocolTCP Protocol = "tcp" + ProtocolTCPv6 Protocol = "tcpv6" + ProtocolUDP Protocol = "udp" + ProtocolUDPv6 Protocol = "udpv6" + ProtocolSCTP Protocol = "sctp" + ProtocolSCTPv6 Protocol = "sctpv6" + ProtocolICMP Protocol = "icmp" + ProtocolICMPv6 Protocol = "icmpv6" ) const ( diff --git a/pkg/ovs/openflow/ofctrl_builder.go b/pkg/ovs/openflow/ofctrl_builder.go index 2d4a128e138..204bdf353bd 100644 --- a/pkg/ovs/openflow/ofctrl_builder.go +++ b/pkg/ovs/openflow/ofctrl_builder.go @@ -352,20 +352,34 @@ func (b *ofFlowBuilder) MatchProtocol(protocol Protocol) FlowBuilder { switch protocol { case ProtocolIP: b.Match.Ethertype = 0x0800 + case ProtocolIPv6: + b.Match.Ethertype = 0x86dd case ProtocolARP: b.Match.Ethertype = 0x0806 case ProtocolTCP: b.Match.Ethertype = 0x0800 b.Match.IpProto = 6 + case ProtocolTCPv6: + b.Match.Ethertype = 0x86dd + b.Match.IpProto = 6 case ProtocolUDP: b.Match.Ethertype = 0x0800 b.Match.IpProto = 17 + case ProtocolUDPv6: + b.Match.Ethertype = 0x86dd + b.Match.IpProto = 17 case ProtocolSCTP: b.Match.Ethertype = 0x0800 b.Match.IpProto = 132 + case ProtocolSCTPv6: + b.Match.Ethertype = 0x86dd + b.Match.IpProto = 132 case ProtocolICMP: b.Match.Ethertype = 0x0800 b.Match.IpProto = 1 + case ProtocolICMPv6: + b.Match.Ethertype = 0x86dd + b.Match.IpProto = 58 } b.protocol = protocol return b diff --git a/test/integration/agent/cniserver_test.go b/test/integration/agent/cniserver_test.go index fdfbb8f66d6..bb79f569c37 100644 --- a/test/integration/agent/cniserver_test.go +++ b/test/integration/agent/cniserver_test.go @@ -606,7 +606,7 @@ func cmdAddDelCheckTest(testNS ns.NetNS, tc testCase, dataDir string) { ovsServiceMock.EXPECT().GetOFPort(ovsPortname).Return(int32(10), nil).AnyTimes() ofServiceMock.EXPECT().InstallPodFlows(ovsPortname, mock.Any(), mock.Any(), mock.Any(), mock.Any()).Return(nil) - // Test ip allocation + // Test ips allocation prevResult, err := tester.cmdAddTest(tc, dataDir) testRequire.Nil(err) @@ -814,7 +814,7 @@ func TestCNIServerChaining(t *testing.T) { routeMock.EXPECT().MigrateRoutesToGw(hostVeth.Name), ovsServiceMock.EXPECT().CreatePort(ovsPortname, ovsPortname, mock.Any()).Return(ovsPortUUID, nil), ovsServiceMock.EXPECT().GetOFPort(ovsPortname).Return(testContainerOFPort, nil), - ofServiceMock.EXPECT().InstallPodFlows(ovsPortname, podIP, containerIntf.HardwareAddr, gwMAC, mock.Any()), + ofServiceMock.EXPECT().InstallPodFlows(ovsPortname, []net.IP{podIP}, containerIntf.HardwareAddr, gwMAC, mock.Any()), ) mock.InOrder(orderedCalls...) cniResp, err := server.CmdAdd(ctx, cniReq) @@ -845,9 +845,9 @@ func init() { nodeName := "node1" gwIP := net.ParseIP("192.168.1.1") gwMAC, _ = net.ParseMAC("11:11:11:11:11:11") - nodeGateway := &config.GatewayConfig{IP: gwIP, MAC: gwMAC, Name: ""} + nodeGateway := &config.GatewayConfig{IPs: []net.IP{gwIP}, MAC: gwMAC, Name: ""} _, nodePodCIDR, _ := net.ParseCIDR("192.168.1.0/24") nodeMTU := 1500 - testNodeConfig = &config.NodeConfig{Name: nodeName, PodCIDR: nodePodCIDR, NodeMTU: nodeMTU, GatewayConfig: nodeGateway} + testNodeConfig = &config.NodeConfig{Name: nodeName, PodIPv4CIDR: nodePodCIDR, NodeMTU: nodeMTU, GatewayConfig: nodeGateway} } diff --git a/test/integration/agent/flowexporter_test.go b/test/integration/agent/flowexporter_test.go index f2b5d563cc7..2129da4a0cf 100644 --- a/test/integration/agent/flowexporter_test.go +++ b/test/integration/agent/flowexporter_test.go @@ -90,7 +90,7 @@ func prepareInterfaceConfigs(contID, podName, podNS, ifName string, ip *net.IP) } iface := &interfacestore.InterfaceConfig{ InterfaceName: ifName, - IP: *ip, + IPs: []net.IP{*ip}, ContainerInterfaceConfig: podConfig, } return iface diff --git a/test/integration/agent/openflow_test.go b/test/integration/agent/openflow_test.go index 990bc2c1403..7c2d94ff3f4 100644 --- a/test/integration/agent/openflow_test.go +++ b/test/integration/agent/openflow_test.go @@ -63,7 +63,7 @@ type expectTableFlows struct { } type testPortConfig struct { - ip net.IP + ips []net.IP mac net.HardwareAddr ofPort uint32 } @@ -291,11 +291,11 @@ func testUninstallNodeFlows(t *testing.T, config *testConfig) { func testInstallPodFlows(t *testing.T, config *testConfig) { for _, pod := range config.localPods { - err := c.InstallPodFlows(pod.name, pod.ip, pod.mac, config.localGateway.mac, pod.ofPort) + err := c.InstallPodFlows(pod.name, pod.ips, pod.mac, config.localGateway.mac, pod.ofPort) if err != nil { t.Fatalf("Failed to install Openflow entries for pod: %v", err) } - for _, tableFlow := range preparePodFlows(pod.ip, pod.mac, pod.ofPort, config.localGateway.mac, config.globalMAC) { + for _, tableFlow := range preparePodFlows(pod.ips, pod.mac, pod.ofPort, config.localGateway.mac, config.globalMAC) { ofTestUtils.CheckFlowExists(t, ovsCtlClient, tableFlow.tableID, true, tableFlow.flows) } } @@ -307,7 +307,7 @@ func testUninstallPodFlows(t *testing.T, config *testConfig) { if err != nil { t.Fatalf("Failed to uninstall Openflow entries for pod: %v", err) } - for _, tableFlow := range preparePodFlows(pod.ip, pod.mac, pod.ofPort, config.localGateway.mac, config.globalMAC) { + for _, tableFlow := range preparePodFlows(pod.ips, pod.mac, pod.ofPort, config.localGateway.mac, config.globalMAC) { ofTestUtils.CheckFlowExists(t, ovsCtlClient, tableFlow.tableID, false, tableFlow.flows) } } @@ -750,11 +750,11 @@ func checkOVSFlowMetrics(t *testing.T, client ofClient.Client) { } func testInstallGatewayFlows(t *testing.T, config *testConfig) { - err := c.InstallGatewayFlows(config.localGateway.ip, config.localGateway.mac, config.localGateway.ofPort) + err := c.InstallGatewayFlows(config.localGateway.ips, config.localGateway.mac, config.localGateway.ofPort) if err != nil { t.Fatalf("Failed to install Openflow entries for gateway: %v", err) } - for _, tableFlow := range prepareGatewayFlows(config.localGateway.ip, config.localGateway.mac, config.localGateway.ofPort, config.globalMAC) { + for _, tableFlow := range prepareGatewayFlows(config.localGateway.ips, config.localGateway.mac, config.localGateway.ofPort, config.globalMAC) { ofTestUtils.CheckFlowExists(t, ovsCtlClient, tableFlow.tableID, true, tableFlow.flows) } } @@ -765,13 +765,13 @@ func prepareConfiguration() *testConfig { podCfg := &testLocalPodConfig{ name: "container-1", testPortConfig: &testPortConfig{ - ip: net.ParseIP("192.168.1.3"), + ips: []net.IP{net.ParseIP("192.168.1.3")}, mac: podMAC, ofPort: uint32(3), }, } gwCfg := &testPortConfig{ - ip: net.ParseIP("192.168.1.1"), + ips: []net.IP{net.ParseIP("192.168.1.1")}, mac: gwMAC, ofPort: uint32(1), } @@ -795,8 +795,8 @@ func prepareConfiguration() *testConfig { } } -func preparePodFlows(podIP net.IP, podMAC net.HardwareAddr, podOFPort uint32, gwMAC, vMAC net.HardwareAddr) []expectTableFlows { - return []expectTableFlows{ +func preparePodFlows(podIPs []net.IP, podMAC net.HardwareAddr, podOFPort uint32, gwMAC, vMAC net.HardwareAddr) []expectTableFlows { + flows := []expectTableFlows{ { uint8(0), []*ofTestUtils.ExpectFlow{ @@ -807,41 +807,59 @@ func preparePodFlows(podIP net.IP, podMAC net.HardwareAddr, podOFPort uint32, gw }, }, { - uint8(10), + uint8(80), []*ofTestUtils.ExpectFlow{ { - MatchStr: fmt.Sprintf("priority=200,ip,in_port=%d,dl_src=%s,nw_src=%s", podOFPort, podMAC.String(), podIP.String()), - ActStr: "goto_table:29", - }, - { - MatchStr: fmt.Sprintf("priority=200,arp,in_port=%d,arp_spa=%s,arp_sha=%s", podOFPort, podIP.String(), podMAC.String()), - ActStr: "goto_table:20", + MatchStr: fmt.Sprintf("priority=200,dl_dst=%s", podMAC.String()), + ActStr: fmt.Sprintf("load:0x%x->NXM_NX_REG1[],load:0x1->NXM_NX_REG0[16],goto_table:90", podOFPort), }, }, }, - { - uint8(70), - []*ofTestUtils.ExpectFlow{ - { - MatchStr: fmt.Sprintf("priority=200,ip,reg0=0x80000/0x80000,nw_dst=%s", podIP.String()), - ActStr: fmt.Sprintf("set_field:%s->eth_src,set_field:%s->eth_dst,dec_ttl,goto_table:80", gwMAC.String(), podMAC.String()), + } + + for _, podIP := range podIPs { + var ipProto string + if podIP.To4() != nil { + ipProto = "ip" + flows = append(flows, + expectTableFlows{ + uint8(10), + []*ofTestUtils.ExpectFlow{ + { + MatchStr: fmt.Sprintf("priority=200,arp,in_port=%d,arp_spa=%s,arp_sha=%s", podOFPort, podIP.String(), podMAC.String()), + ActStr: "goto_table:20", + }, + }, + }) + } else { + ipProto = "ipv6" + } + flows = append(flows, + expectTableFlows{ + uint8(10), + []*ofTestUtils.ExpectFlow{ + { + MatchStr: fmt.Sprintf("priority=200,%s,in_port=%d,dl_src=%s,nw_src=%s", ipProto, podOFPort, podMAC.String(), podIP.String()), + ActStr: "goto_table:29", + }, }, }, - }, - { - uint8(80), - []*ofTestUtils.ExpectFlow{ - { - MatchStr: fmt.Sprintf("priority=200,dl_dst=%s", podMAC.String()), - ActStr: fmt.Sprintf("load:0x%x->NXM_NX_REG1[],load:0x1->NXM_NX_REG0[16],goto_table:90", podOFPort), + expectTableFlows{ + uint8(70), + []*ofTestUtils.ExpectFlow{ + { + MatchStr: fmt.Sprintf("priority=200,ip,reg0=0x80000/0x80000,nw_dst=%s", podIP.String()), + ActStr: fmt.Sprintf("set_field:%s->eth_src,set_field:%s->eth_dst,dec_ttl,goto_table:80", gwMAC.String(), podMAC.String())}, }, }, - }, + ) } + + return flows } -func prepareGatewayFlows(gwIP net.IP, gwMAC net.HardwareAddr, gwOFPort uint32, vMAC net.HardwareAddr) []expectTableFlows { - return []expectTableFlows{ +func prepareGatewayFlows(gwIPs []net.IP, gwMAC net.HardwareAddr, gwOFPort uint32, vMAC net.HardwareAddr) []expectTableFlows { + flows := []expectTableFlows{ { uint8(0), []*ofTestUtils.ExpectFlow{ @@ -860,28 +878,6 @@ func prepareGatewayFlows(gwIP net.IP, gwMAC net.HardwareAddr, gwOFPort uint32, v }, }, }, - { - uint8(10), - []*ofTestUtils.ExpectFlow{ - { - MatchStr: fmt.Sprintf("priority=200,arp,in_port=%d,arp_spa=%s,arp_sha=%s", gwOFPort, gwIP, gwMAC), - ActStr: "goto_table:20", - }, - { - MatchStr: fmt.Sprintf("priority=200,ip,in_port=%d", gwOFPort), - ActStr: "goto_table:29", - }, - }, - }, - { - uint8(70), - []*ofTestUtils.ExpectFlow{ - { - MatchStr: fmt.Sprintf("priority=200,ip,dl_dst=%s,nw_dst=%s", vMAC.String(), gwIP.String()), - ActStr: fmt.Sprintf("set_field:%s->eth_dst,goto_table:80", gwMAC.String()), - }, - }, - }, { uint8(80), []*ofTestUtils.ExpectFlow{ @@ -891,16 +887,52 @@ func prepareGatewayFlows(gwIP net.IP, gwMAC net.HardwareAddr, gwOFPort uint32, v }, }, }, - { - uint8(90), - []*ofTestUtils.ExpectFlow{ - { - MatchStr: fmt.Sprintf("priority=210,ip,nw_src=%s", gwIP.String()), - ActStr: "goto_table:105", + } + + for _, gwIP := range gwIPs { + var ipProtoStr string + if gwIP.To4() != nil { + ipProtoStr = "ip" + flows = append(flows, + expectTableFlows{ + uint8(10), + []*ofTestUtils.ExpectFlow{ + { + MatchStr: fmt.Sprintf("priority=200,arp,in_port=%d,arp_spa=%s,arp_sha=%s", gwOFPort, gwIP, gwMAC), + ActStr: "goto_table:20", + }, + { + MatchStr: fmt.Sprintf("priority=200,ip,in_port=%d", gwOFPort), + ActStr: "goto_table:29", + }, + }, + }) + } else { + ipProtoStr = "ipv6" + } + flows = append(flows, + expectTableFlows{ + uint8(70), + []*ofTestUtils.ExpectFlow{ + { + MatchStr: fmt.Sprintf("priority=200,%s,dl_dst=%s,nw_dst=%s", ipProtoStr, vMAC.String(), gwIP.String()), + ActStr: fmt.Sprintf("set_field:%s->eth_dst,goto_table:80", gwMAC.String()), + }, }, }, - }, + expectTableFlows{ + tableID: uint8(90), + flows: []*ofTestUtils.ExpectFlow{ + { + MatchStr: fmt.Sprintf("priority=210,%s,nw_src=%s", ipProtoStr, gwIP.String()), + ActStr: "goto_table:105", + }, + }, + }, + ) } + + return flows } func prepareTunnelFlows(tunnelPort uint32, vMAC net.HardwareAddr) []expectTableFlows { diff --git a/test/integration/agent/route_test.go b/test/integration/agent/route_test.go index d2ee50845c5..00c25aaa767 100644 --- a/test/integration/agent/route_test.go +++ b/test/integration/agent/route_test.go @@ -57,10 +57,10 @@ var ( gwIP = net.ParseIP("10.10.10.1") gwMAC, _ = net.ParseMAC("12:34:56:78:bb:cc") gwName = "antrea-gw0" - gwConfig = &config.GatewayConfig{IP: gwIP, MAC: gwMAC, Name: gwName} + gwConfig = &config.GatewayConfig{IPs: []net.IP{gwIP}, MAC: gwMAC, Name: gwName} nodeConfig = &config.NodeConfig{ Name: "test", - PodCIDR: podCIDR, + PodIPv4CIDR: podCIDR, NodeIPAddr: nodeIP, GatewayConfig: gwConfig, }