From 98735cfeba06746dea62dcc6d5cf6d9bbc574c33 Mon Sep 17 00:00:00 2001 From: wenyingd Date: Tue, 21 Jul 2020 02:26:28 +0000 Subject: [PATCH] [IPv6] Consume Node.Spec.CIDRs to support dual-stack configuration 1. Consume Node.Spec.CIDRs to support IPv4/IPv6 dual-stack Pod Subnets 2. Change NodeConfig.PodCIDR as a slice 3. Change GatewayConfig.IP as a slice to support multiple addresses for antrea-gw0 4. Change InterfaceConfig.IP as a slice to support multiple address for a Pod --- pkg/agent/agent.go | 78 +++++++--- pkg/agent/agent_test.go | 6 +- pkg/agent/agent_windows.go | 12 +- .../apiserver/handlers/ovstracing/handler.go | 4 +- .../handlers/ovstracing/handler_test.go | 8 +- .../handlers/podinterface/handler.go | 2 +- .../handlers/podinterface/handler_test.go | 6 +- pkg/agent/cniserver/pod_configuration.go | 43 ++++-- pkg/agent/cniserver/server.go | 12 +- pkg/agent/cniserver/server_test.go | 49 +++++-- pkg/agent/config/node_config.go | 17 ++- .../controller/networkpolicy/reconciler.go | 7 +- .../networkpolicy/reconciler_test.go | 8 +- .../traceflow/traceflow_controller.go | 4 +- .../connections/connections_test.go | 2 +- .../connections/conntrack_linux.go | 7 +- .../connections/conntrack_test.go | 2 +- pkg/agent/interfacestore/interface_cache.go | 8 +- pkg/agent/interfacestore/types.go | 11 +- pkg/agent/openflow/client.go | 35 +++-- pkg/agent/openflow/client_test.go | 2 +- pkg/agent/openflow/pipeline.go | 118 +++++++++------ pkg/agent/openflow/testing/mock_openflow.go | 4 +- pkg/agent/querier/querier.go | 4 +- pkg/agent/querier/querier_test.go | 8 +- pkg/agent/route/route_linux.go | 8 +- pkg/agent/route/route_windows.go | 7 +- pkg/agent/util/net.go | 9 ++ pkg/ovs/openflow/interfaces.go | 17 ++- pkg/ovs/openflow/ofctrl_builder.go | 14 ++ test/integration/agent/cniserver_test.go | 8 +- test/integration/agent/openflow_test.go | 137 +++++++++++------- test/integration/agent/route_test.go | 4 +- 33 files changed, 443 insertions(+), 218 deletions(-) diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index 0f4f28471e3..59f533dc627 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -285,7 +285,7 @@ func (i *Initializer) initOpenFlowPipeline() error { // Set up flow entries for gateway interface, including classifier, skip spoof guard check, // L3 forwarding and L2 forwarding - if err := i.ofClient.InstallGatewayFlows(gateway.IP, gateway.MAC, gatewayOFPort); err != nil { + if err := i.ofClient.InstallGatewayFlows(gateway.IPs, gateway.MAC, gatewayOFPort); err != nil { klog.Errorf("Failed to setup openflow entries for gateway: %v", err) return err } @@ -451,28 +451,20 @@ func (i *Initializer) configureGatewayInterface(gatewayIface *interfacestore.Int gatewayIface.MAC = gwMAC if i.networkConfig.TrafficEncapMode.IsNetworkPolicyOnly() { // In policy-only mode, Node IP is also assigned to local gateway for masquerade. - i.nodeConfig.GatewayConfig.IP = i.nodeConfig.NodeIPAddr.IP - gatewayIface.IP = i.nodeConfig.NodeIPAddr.IP + i.nodeConfig.GatewayConfig.IPs = []net.IP{i.nodeConfig.NodeIPAddr.IP} + gatewayIface.IPs = []net.IP{i.nodeConfig.NodeIPAddr.IP} return nil } - // Configure host gateway IP using the first address of node localSubnet. - localSubnet := i.nodeConfig.PodCIDR - subnetID := localSubnet.IP.Mask(localSubnet.Mask) - gwIP := &net.IPNet{IP: ip.NextIP(subnetID), Mask: localSubnet.Mask} - - // Check IP address configuration on existing interface first, return if the interface has the desired address. - // We perform this check unconditionally, even if the OVS port does not exist when this function is called - // (i.e. portExists is false). Indeed, it may be possible for the interface to exist even if the OVS bridge does - // not exist. - // Configure the IP address on the interface if it does not exist. - if err := util.ConfigureLinkAddress(gwLinkIdx, gwIP); err != nil { - return err + i.nodeConfig.GatewayConfig.LinkIndex = gwLinkIdx + // Allocate the gateway IP address from the Pod CIDRs if it exists. The gateway IP should be the first address + // in the Subnet and configure on the host gateway. + for _, podCIDR := range []*net.IPNet{i.nodeConfig.PodIPv4CIDR, i.nodeConfig.PodIPv6CIDR} { + if err := i.allocateGatewayAddress(podCIDR, gatewayIface); err != nil { + return err + } } - i.nodeConfig.GatewayConfig.LinkIndex = gwLinkIdx - i.nodeConfig.GatewayConfig.IP = gwIP.IP - gatewayIface.IP = gwIP.IP return nil } @@ -526,7 +518,7 @@ func (i *Initializer) setupDefaultTunnelInterface() error { return nil } -// initNodeLocalConfig retrieves node's subnet CIDR from node.spec.PodCIDR, which is used for IPAM and setup +// initNodeLocalConfig retrieves node's subnet CIDR from node.spec.PodCIDRs, which is used for IPAM and setup // host gateway interface. func (i *Initializer) initNodeLocalConfig() error { nodeName, err := env.GetNodeName() @@ -566,7 +558,30 @@ func (i *Initializer) initNodeLocalConfig() error { return nil } - // Spec.PodCIDR can be empty due to misconfiguration + // Parse all PodCIDRs first, so that we could support IPv4/IPv6 dual-stack configurations. + if node.Spec.PodCIDRs != nil { + for _, podCIDR := range node.Spec.PodCIDRs { + _, localSubnet, err := net.ParseCIDR(podCIDR) + if err != nil { + klog.Errorf("Failed to parse subnet from CIDR string %s: %v", node.Spec.PodCIDR, err) + return err + } + if localSubnet.IP.To4() != nil { + if i.nodeConfig.PodIPv4CIDR != nil { + return fmt.Errorf("one IPv4 PodCIDR is allowed at most") + } + i.nodeConfig.PodIPv4CIDR = localSubnet + continue + } + if i.nodeConfig.PodIPv6CIDR != nil { + return fmt.Errorf("one IPv6 PodCIDR is allowed at most") + } + i.nodeConfig.PodIPv6CIDR = localSubnet + } + return nil + } + + // Spec.PodCIDR can be empty due to misconfiguration. if node.Spec.PodCIDR == "" { klog.Errorf("Spec.PodCIDR is empty for Node %s. Please make sure --allocate-node-cidrs is enabled "+ "for kube-controller-manager and --cluster-cidr specifies a sufficient CIDR range", nodeName) @@ -577,7 +592,7 @@ func (i *Initializer) initNodeLocalConfig() error { klog.Errorf("Failed to parse subnet from CIDR string %s: %v", node.Spec.PodCIDR, err) return err } - i.nodeConfig.PodCIDR = localSubnet + i.nodeConfig.PodIPv4CIDR = localSubnet return nil } @@ -672,3 +687,24 @@ func (i *Initializer) getNodeMTU(localIntf *net.Interface) (int, error) { } return mtu, nil } + +func (i *Initializer) allocateGatewayAddress(localSubnet *net.IPNet, gatewayIface *interfacestore.InterfaceConfig) error { + if localSubnet == nil { + return nil + } + subnetID := localSubnet.IP.Mask(localSubnet.Mask) + gwIP := &net.IPNet{IP: ip.NextIP(subnetID), Mask: localSubnet.Mask} + + // Check IP address configuration on existing interface first, return if the interface has the desired address. + // We perform this check unconditionally, even if the OVS port does not exist when this function is called + // (i.e. portExists is false). Indeed, it may be possible for the interface to exist even if the OVS bridge does + // not exist. + // Configure the IP address on the interface if it does not exist. + if err := util.ConfigureLinkAddress(i.nodeConfig.GatewayConfig.LinkIndex, gwIP); err != nil { + return err + } + + i.nodeConfig.GatewayConfig.IPs = append(i.nodeConfig.GatewayConfig.IPs, gwIP.IP) + gatewayIface.IPs = append(gatewayIface.IPs, gwIP.IP) + return nil +} diff --git a/pkg/agent/agent_test.go b/pkg/agent/agent_test.go index 9ef3b78533c..a70f58a95b6 100644 --- a/pkg/agent/agent_test.go +++ b/pkg/agent/agent_test.go @@ -72,10 +72,10 @@ func TestInitstore(t *testing.T) { ovsPort1 := ovsconfig.OVSPortData{UUID: uuid1, Name: "p1", IFName: "p1", OFPort: 11, ExternalIDs: convertExternalIDMap(cniserver.BuildOVSPortExternalIDs( - interfacestore.NewContainerInterface("p1", uuid1, "pod1", "ns1", p1NetMAC, p1NetIP)))} + interfacestore.NewContainerInterface("p1", uuid1, "pod1", "ns1", p1NetMAC, []net.IP{p1NetIP})))} ovsPort2 := ovsconfig.OVSPortData{UUID: uuid2, Name: "p2", IFName: "p2", OFPort: 12, ExternalIDs: convertExternalIDMap(cniserver.BuildOVSPortExternalIDs( - interfacestore.NewContainerInterface("p2", uuid2, "pod2", "ns2", p2NetMAC, p2NetIP)))} + interfacestore.NewContainerInterface("p2", uuid2, "pod2", "ns2", p2NetMAC, []net.IP{p2NetIP})))} initOVSPorts := []ovsconfig.OVSPortData{ovsPort1, ovsPort2} mockOVSBridgeClient.EXPECT().GetPortList().Return(initOVSPorts, ovsconfig.NewTransactionError(fmt.Errorf("Failed to list OVS ports"), true)) @@ -92,7 +92,7 @@ func TestInitstore(t *testing.T) { container1, found1 := store.GetContainerInterface(uuid1) if !found1 { t.Errorf("Failed to load OVS port into local store") - } else if container1.OFPort != 11 || container1.IP.String() != p1IP || container1.MAC.String() != p1MAC || container1.InterfaceName != "p1" { + } else if container1.OFPort != 11 || container1.GetIPv4Addr() == nil || container1.GetIPv4Addr().String() != p1IP || container1.MAC.String() != p1MAC || container1.InterfaceName != "p1" { t.Errorf("Failed to load OVS port configuration into local store") } _, found2 := store.GetContainerInterface(uuid2) diff --git a/pkg/agent/agent_windows.go b/pkg/agent/agent_windows.go index d2c5a777786..2042305e736 100644 --- a/pkg/agent/agent_windows.go +++ b/pkg/agent/agent_windows.go @@ -17,6 +17,7 @@ package agent import ( + "fmt" "net" "strings" @@ -32,7 +33,10 @@ import ( // setupExternalConnectivity installs OpenFlow entries to SNAT Pod traffic using Node IP, and then Pod could communicate // to the external IP address. func (i *Initializer) setupExternalConnectivity() error { - subnetCIDR := i.nodeConfig.PodCIDR + subnetCIDR := i.nodeConfig.PodIPv4CIDR + if subnetCIDR == nil { + return fmt.Errorf("Failed to find valid IPv4 PodCIDR") + } nodeIP := i.nodeConfig.NodeIPAddr.IP // Install OpenFlow entries on the OVS to enable Pod traffic to communicate to external IP addresses. if err := i.ofClient.InstallExternalFlows(nodeIP, *subnetCIDR); err != nil { @@ -74,7 +78,11 @@ func (i *Initializer) prepareHostNetwork() error { } i.nodeConfig.UplinkNetConfig.DNSServers = dnsServers // Create HNS network. - return util.PrepareHNSNetwork(i.nodeConfig.PodCIDR, i.nodeConfig.NodeIPAddr, adapter) + subnetCIDR := i.nodeConfig.PodIPv4CIDR + if subnetCIDR == nil { + return fmt.Errorf("Failed to find valid IPv4 PodCIDR") + } + return util.PrepareHNSNetwork(subnetCIDR, i.nodeConfig.NodeIPAddr, adapter) } // prepareOVSBridge adds local port and uplink to ovs bridge. diff --git a/pkg/agent/apiserver/handlers/ovstracing/handler.go b/pkg/agent/apiserver/handlers/ovstracing/handler.go index fbbfc1bf0bc..25c23ba20e0 100644 --- a/pkg/agent/apiserver/handlers/ovstracing/handler.go +++ b/pkg/agent/apiserver/handlers/ovstracing/handler.go @@ -83,13 +83,13 @@ func getPeerAddress(aq querier.AgentQuerier, peer *tracingPeer) (net.IP, *interf err := handlers.NewHandlerError(fmt.Errorf("OVS port %s not found", peer.ovsPort), http.StatusNotFound) return nil, nil, err } - return intf.IP, intf, nil + return intf.GetIPv4Addr(), intf, nil } interfaces := aq.GetInterfaceStore().GetContainerInterfacesByPod(peer.name, peer.namespace) if len(interfaces) > 0 { // Local Pod. - return interfaces[0].IP, interfaces[0], nil + return interfaces[0].GetIPv4Addr(), interfaces[0], nil } // Try getting the Pod from K8s API. diff --git a/pkg/agent/apiserver/handlers/ovstracing/handler_test.go b/pkg/agent/apiserver/handlers/ovstracing/handler_test.go index 61fc21ca924..b370bf823e0 100644 --- a/pkg/agent/apiserver/handlers/ovstracing/handler_test.go +++ b/pkg/agent/apiserver/handlers/ovstracing/handler_test.go @@ -46,7 +46,7 @@ var ( testNodeConfig = &config.NodeConfig{ GatewayConfig: &config.GatewayConfig{ Name: "antrea-gw0", - IP: net.ParseIP("10.1.1.1"), + IPs: []net.IP{net.ParseIP("10.1.1.1")}, MAC: gatewayMAC}, } @@ -55,19 +55,19 @@ var ( inPodInterface = &interfacestore.InterfaceConfig{ Type: interfacestore.ContainerInterface, InterfaceName: "inPod", - IP: net.ParseIP("10.1.1.11"), + IPs: []net.IP{net.ParseIP("10.1.1.11")}, MAC: podMAC, } srcPodInterface = &interfacestore.InterfaceConfig{ Type: interfacestore.ContainerInterface, InterfaceName: "srcPod", - IP: net.ParseIP("10.1.1.12"), + IPs: []net.IP{net.ParseIP("10.1.1.12")}, MAC: podMAC, } dstPodInterface = &interfacestore.InterfaceConfig{ Type: interfacestore.ContainerInterface, InterfaceName: "dstPod", - IP: net.ParseIP("10.1.1.13"), + IPs: []net.IP{net.ParseIP("10.1.1.13")}, MAC: podMAC, } ) diff --git a/pkg/agent/apiserver/handlers/podinterface/handler.go b/pkg/agent/apiserver/handlers/podinterface/handler.go index 712f21260d0..97a3c248e5d 100644 --- a/pkg/agent/apiserver/handlers/podinterface/handler.go +++ b/pkg/agent/apiserver/handlers/podinterface/handler.go @@ -40,7 +40,7 @@ func generateResponse(i *interfacestore.InterfaceConfig) Response { PodName: i.ContainerInterfaceConfig.PodName, PodNamespace: i.ContainerInterfaceConfig.PodNamespace, InterfaceName: i.InterfaceName, - IP: i.IP.String(), + IP: i.GetIPv4Addr().String(), MAC: i.MAC.String(), PortUUID: i.OVSPortConfig.PortUUID, OFPort: i.OVSPortConfig.OFPort, diff --git a/pkg/agent/apiserver/handlers/podinterface/handler_test.go b/pkg/agent/apiserver/handlers/podinterface/handler_test.go index ea41161ccec..5c127d93a2e 100644 --- a/pkg/agent/apiserver/handlers/podinterface/handler_test.go +++ b/pkg/agent/apiserver/handlers/podinterface/handler_test.go @@ -91,7 +91,7 @@ var responses = []Response{ var testInterfaceConfigs = []*interfacestore.InterfaceConfig{ { InterfaceName: "interface0", - IP: net.ParseIP(ipStrs[0]), + IPs: []net.IP{net.ParseIP(ipStrs[0])}, MAC: macs[0], OVSPortConfig: &interfacestore.OVSPortConfig{ PortUUID: "portuuid0", @@ -105,7 +105,7 @@ var testInterfaceConfigs = []*interfacestore.InterfaceConfig{ }, { InterfaceName: "interface1", - IP: net.ParseIP(ipStrs[1]), + IPs: []net.IP{net.ParseIP(ipStrs[1])}, MAC: macs[1], OVSPortConfig: &interfacestore.OVSPortConfig{ PortUUID: "portuuid1", @@ -119,7 +119,7 @@ var testInterfaceConfigs = []*interfacestore.InterfaceConfig{ }, { InterfaceName: "interface2", - IP: net.ParseIP(ipStrs[2]), + IPs: []net.IP{net.ParseIP(ipStrs[2])}, MAC: macs[2], OVSPortConfig: &interfacestore.OVSPortConfig{ PortUUID: "portuuid2", diff --git a/pkg/agent/cniserver/pod_configuration.go b/pkg/agent/cniserver/pod_configuration.go index 2ab1783c896..cd486a96bb0 100644 --- a/pkg/agent/cniserver/pod_configuration.go +++ b/pkg/agent/cniserver/pod_configuration.go @@ -18,6 +18,7 @@ import ( "encoding/json" "fmt" "net" + "strings" cnitypes "github.com/containernetworking/cni/pkg/types" "github.com/containernetworking/cni/pkg/types/current" @@ -134,7 +135,7 @@ func buildContainerConfig( podName, podNamespace, containerMAC, - containerIP) + []net.IP{containerIP}) } // BuildOVSPortExternalIDs parses OVS port external_ids from InterfaceConfig. @@ -143,12 +144,20 @@ func BuildOVSPortExternalIDs(containerConfig *interfacestore.InterfaceConfig) ma externalIDs := make(map[string]interface{}) externalIDs[ovsExternalIDMAC] = containerConfig.MAC.String() externalIDs[ovsExternalIDContainerID] = containerConfig.ContainerID - externalIDs[ovsExternalIDIP] = containerConfig.IP.String() + externalIDs[ovsExternalIDIP] = getContainerIPsString(containerConfig.IPs) externalIDs[ovsExternalIDPodName] = containerConfig.PodName externalIDs[ovsExternalIDPodNamespace] = containerConfig.PodNamespace return externalIDs } +func getContainerIPsString(ips []net.IP) string { + var containerIPs []string + for _, ip := range ips { + containerIPs = append(containerIPs, ip.String()) + } + return strings.Join(containerIPs, ",") +} + // ParseOVSPortInterfaceConfig reads the Pod properties saved in the OVS port // external_ids, initializes and returns an InterfaceConfig struct. // nill will be returned, if the OVS port does not have external IDs or it is @@ -164,7 +173,12 @@ func ParseOVSPortInterfaceConfig(portData *ovsconfig.OVSPortData, portConfig *in klog.V(2).Infof("OVS port %s has no %s in external_ids", portData.Name, ovsExternalIDContainerID) return nil } - containerIP := net.ParseIP(portData.ExternalIDs[ovsExternalIDIP]) + containerIPStrs := strings.Split(portData.ExternalIDs[ovsExternalIDIP], ",") + var containerIPs []net.IP + for _, ipStr := range containerIPStrs { + containerIPs = append(containerIPs, net.ParseIP(ipStr)) + } + containerMAC, err := net.ParseMAC(portData.ExternalIDs[ovsExternalIDMAC]) if err != nil { klog.Errorf("Failed to parse MAC address from OVS external config %s: %v", @@ -179,7 +193,7 @@ func ParseOVSPortInterfaceConfig(portData *ovsconfig.OVSPortData, portConfig *in podName, podNamespace, containerMAC, - containerIP) + containerIPs) interfaceConfig.OVSPortConfig = portConfig return interfaceConfig } @@ -342,13 +356,14 @@ func (pc *podConfigurator) validateOVSInterfaceConfig(containerID string, contai for _, ipc := range ips { if ipc.Version == "4" { - if containerConfig.IP.Equal(ipc.Address.IP) { + ipv4Addr := util.GetIPv4Addr(containerConfig.IPs) + if ipv4Addr != nil && ipv4Addr.Equal(ipc.Address.IP) { return nil } } } return fmt.Errorf("interface IP %s does not match container %s IP", - containerConfig.IP.String(), containerID) + getContainerIPsString(containerConfig.IPs), containerID) } else { return fmt.Errorf("container %s interface not found from local cache", containerID) } @@ -400,7 +415,7 @@ func (pc *podConfigurator) reconcile(pods []corev1.Pod) error { klog.V(4).Infof("Syncing interface %s for Pod %s", containerConfig.InterfaceName, namespacedName) if err := pc.ofClient.InstallPodFlows( containerConfig.InterfaceName, - containerConfig.IP, + containerConfig.IPs, containerConfig.MAC, pc.gatewayMAC, uint32(containerConfig.OFPort), @@ -466,7 +481,7 @@ func (pc *podConfigurator) connectInterfaceToOVS( } klog.V(2).Infof("Setting up Openflow entries for container %s", containerID) - err = pc.ofClient.InstallPodFlows(ovsPortName, containerConfig.IP, containerConfig.MAC, pc.gatewayMAC, uint32(ofPort)) + err = pc.ofClient.InstallPodFlows(ovsPortName, containerConfig.IPs, containerConfig.MAC, pc.gatewayMAC, uint32(ofPort)) if err != nil { return nil, fmt.Errorf("failed to add Openflow entries for container %s: %v", containerID, err) } @@ -531,11 +546,13 @@ func (pc *podConfigurator) disconnectInterceptedInterface(podName, podNamespace, klog.V(2).Infof("Did not find the port for container %s in local cache", containerID) return nil } - if err := pc.routeClient.UnMigrateRoutesFromGw(&net.IPNet{ - IP: containerConfig.IP, - Mask: net.CIDRMask(32, 32), - }, ""); err != nil { - return fmt.Errorf("connectInterceptedInterface failed to migrate: %w", err) + for _, ip := range containerConfig.IPs { + if err := pc.routeClient.UnMigrateRoutesFromGw(&net.IPNet{ + IP: ip, + Mask: net.CIDRMask(32, 32), + }, ""); err != nil { + return fmt.Errorf("connectInterceptedInterface failed to migrate: %w", err) + } } return pc.disconnectInterfaceFromOVS(containerConfig) // TODO recover pre-connect state? repatch vethpair to original bridge etc ?? to make first CNI happy?? diff --git a/pkg/agent/cniserver/server.go b/pkg/agent/cniserver/server.go index 96c63a43f25..2b52b0c0968 100644 --- a/pkg/agent/cniserver/server.go +++ b/pkg/agent/cniserver/server.go @@ -218,9 +218,12 @@ func (s *CNIServer) checkRequestMessage(request *cnipb.CniCmdRequest) (*CNIConfi } func (s *CNIServer) updateLocalIPAMSubnet(cniConfig *CNIConfig) { - cniConfig.NetworkConfig.IPAM.Gateway = s.nodeConfig.GatewayConfig.IP.String() - cniConfig.NetworkConfig.IPAM.Subnet = s.nodeConfig.PodCIDR.String() - cniConfig.NetworkConfiguration, _ = json.Marshal(cniConfig.NetworkConfig) + gwIPv4 := util.GetIPv4Addr(s.nodeConfig.GatewayConfig.IPs) + cniConfig.NetworkConfig.IPAM.Gateway = gwIPv4.String() + if s.nodeConfig.PodIPv4CIDR != nil { + cniConfig.NetworkConfig.IPAM.Subnet = s.nodeConfig.PodIPv4CIDR.String() + cniConfig.NetworkConfiguration, _ = json.Marshal(cniConfig.NetworkConfig) + } } func (s *CNIServer) generateCNIErrorResponse(cniErrorCode cnipb.ErrorCode, cniErrorMsg string) *cnipb.CniCmdResponse { @@ -399,7 +402,8 @@ func (s *CNIServer) CmdAdd(ctx context.Context, request *cnipb.CniCmdRequest) (* result.IPs = ipamResult.IPs result.Routes = ipamResult.Routes // Ensure interface gateway setting and mapping relations between result.Interfaces and result.IPs - updateResultIfaceConfig(result, s.nodeConfig.GatewayConfig.IP) + gwIPv4 := util.GetIPv4Addr(s.nodeConfig.GatewayConfig.IPs) + updateResultIfaceConfig(result, gwIPv4) // Setup pod interfaces and connect to ovs bridge podName := string(cniConfig.K8S_POD_NAME) podNamespace := string(cniConfig.K8S_POD_NAMESPACE) diff --git a/pkg/agent/cniserver/server_test.go b/pkg/agent/cniserver/server_test.go index 07dafc71f17..6e77eff9d0c 100644 --- a/pkg/agent/cniserver/server_test.go +++ b/pkg/agent/cniserver/server_test.go @@ -19,6 +19,7 @@ import ( "encoding/json" "fmt" "net" + "strings" "testing" cnitypes "github.com/containernetworking/cni/pkg/types" @@ -79,11 +80,11 @@ func TestLoadNetConfig(t *testing.T) { assert.Equal(networkCfg.Name, netCfg.Name) assert.Equal(networkCfg.IPAM.Type, netCfg.IPAM.Type) assert.Equal( - netCfg.IPAM.Subnet, testNodeConfig.PodCIDR.String(), - "Network configuration (PodCIDR) was not updated", + netCfg.IPAM.Subnet, testNodeConfig.PodIPv4CIDR.String(), + "Network configuration (PodCIDRs) was not updated", ) assert.Equal( - netCfg.IPAM.Gateway, testNodeConfig.GatewayConfig.IP.String(), + netCfg.IPAM.Gateway, testNodeConfig.GatewayConfig.IPs[0].String(), "Network configuration (Gateway IP) was not updated", ) } @@ -311,7 +312,7 @@ func TestUpdateResultIfaceConfig(t *testing.T) { // return a Result with 2 v4 addresses. testIps := []string{"10.1.2.100/24, ,4", "192.168.1.100/24, 192.168.2.253, 4"} - require.Equal(gwIP, testNodeConfig.GatewayConfig.IP) + require.Equal(gwIP, testNodeConfig.GatewayConfig.IPs[0]) t.Run("Gateways updated", func(t *testing.T) { assert := assert.New(t) @@ -409,7 +410,7 @@ func TestRemoveInterface(t *testing.T) { podName, testPodNamespace, containerMAC, - containerIP) + []net.IP{containerIP}) containerConfig.OVSPortConfig = &interfacestore.OVSPortConfig{fakePortUUID, 0} } @@ -455,11 +456,14 @@ func TestRemoveInterface(t *testing.T) { func TestBuildOVSPortExternalIDs(t *testing.T) { containerID := uuid.New().String() containerMAC, _ := net.ParseMAC("aa:bb:cc:dd:ee:ff") - containerIP := net.ParseIP("10.1.2.100") - containerConfig := interfacestore.NewContainerInterface("pod1-abcd", containerID, "test-1", "t1", containerMAC, containerIP) + containerIP1 := net.ParseIP("10.1.2.100") + containerIP2 := net.ParseIP("2001:fd1a::2") + containerIPs := []net.IP{containerIP1, containerIP2} + containerConfig := interfacestore.NewContainerInterface("pod1-abcd", containerID, "test-1", "t1", containerMAC, containerIPs) externalIds := BuildOVSPortExternalIDs(containerConfig) parsedIP, existed := externalIds[ovsExternalIDIP] - if !existed || parsedIP != "10.1.2.100" { + parsedIPStr := parsedIP.(string) + if !existed || !strings.Contains(parsedIPStr, "10.1.2.100") || !strings.Contains(parsedIPStr, "2001:fd1a::2") { t.Errorf("Failed to parse container configuration") } parsedMac, existed := externalIds[ovsExternalIDMAC] @@ -470,6 +474,31 @@ func TestBuildOVSPortExternalIDs(t *testing.T) { if !existed || parsedID != containerID { t.Errorf("Failed to parse container configuration") } + portExternalIDs := make(map[string]string) + for k, v := range externalIds { + val := v.(string) + portExternalIDs[k] = val + } + mockPort := &ovsconfig.OVSPortData{ + Name: "testPort", + ExternalIDs: portExternalIDs, + } + portConfig := &interfacestore.OVSPortConfig{ + PortUUID: "12345678", + OFPort: int32(1), + } + ifaceConfig := ParseOVSPortInterfaceConfig(mockPort, portConfig) + assert.Equal(t, len(containerIPs), len(ifaceConfig.IPs)) + for _, ip1 := range containerIPs { + existed := false + for _, ip2 := range ifaceConfig.IPs { + if ip2.Equal(ip1) { + existed = true + break + } + } + assert.True(t, existed, fmt.Sprintf("IP %s should exist in the restored InterfaceConfig", ip1.String())) + } } func translateRawPrevResult(prevResult *current.Result, cniVersion string) (map[string]interface{}, error) { @@ -543,6 +572,6 @@ func init() { gwIP = net.ParseIP("192.168.1.1") _, nodePodCIDR, _ := net.ParseCIDR("192.168.1.0/24") gwMAC, _ := net.ParseMAC("00:00:00:00:00:01") - gateway := &config.GatewayConfig{Name: "", IP: gwIP, MAC: gwMAC} - testNodeConfig = &config.NodeConfig{Name: nodeName, PodCIDR: nodePodCIDR, GatewayConfig: gateway} + gateway := &config.GatewayConfig{Name: "", IPs: []net.IP{gwIP}, MAC: gwMAC} + testNodeConfig = &config.NodeConfig{Name: nodeName, PodIPv4CIDR: nodePodCIDR, GatewayConfig: gateway} } diff --git a/pkg/agent/config/node_config.go b/pkg/agent/config/node_config.go index ecf7c82c2a4..cada373e54a 100644 --- a/pkg/agent/config/node_config.go +++ b/pkg/agent/config/node_config.go @@ -45,14 +45,14 @@ const ( type GatewayConfig struct { // Name is the name of host gateway, e.g. antrea-gw0. Name string - IP net.IP + IPs []net.IP MAC net.HardwareAddr // LinkIndex is the link index of host gateway. LinkIndex int } func (g *GatewayConfig) String() string { - return fmt.Sprintf("Name %s: IP %s, MAC %s", g.Name, g.IP, g.MAC) + return fmt.Sprintf("Name %s: IP %s, MAC %s", g.Name, g.IPs, g.MAC) } type AdapterNetConfig struct { @@ -73,9 +73,12 @@ type NodeConfig struct { // The name of the default tunnel interface. Defaults to "antrea-tun0", but can // be overridden by the discovered tunnel interface name from the OVS bridge. DefaultTunName string - // The CIDR block to allocate Pod IPs out of. - // It's nil for the networkPolicyOnly trafficEncapMode which doesn't do IPAM. - PodCIDR *net.IPNet + // The CIDR block from where to allocate IPv4 address to Pod. + // It's nil for the net workPolicyOnly trafficEncapMode which doesn't do IPAM. + PodIPv4CIDR *net.IPNet + // The CIDR block from where to allocate IPv6 address to Pod. + // It's nil for the net workPolicyOnly trafficEncapMode which doesn't do IPAM. + PodIPv6CIDR *net.IPNet // The Node's IP used in Kubernetes. It has the network mask information. NodeIPAddr *net.IPNet // Set either via defaultMTU config in antrea.yaml or auto discovered. @@ -89,8 +92,8 @@ type NodeConfig struct { } func (n *NodeConfig) String() string { - return fmt.Sprintf("NodeName: %s, OVSBridge: %s, PodCIDR: %s, NodeIP: %s, Gateway: %s", - n.Name, n.OVSBridge, n.PodCIDR, n.NodeIPAddr, n.GatewayConfig) + return fmt.Sprintf("NodeName: %s, OVSBridge: %s, PodIPv4CIDR %s, PodIPv6CIDR: %s, NodeIP: %s, Gateway: %s", + n.Name, n.OVSBridge, n.PodIPv4CIDR, n.PodIPv6CIDR, n.NodeIPAddr, n.GatewayConfig) } // User provided network configuration parameters. diff --git a/pkg/agent/controller/networkpolicy/reconciler.go b/pkg/agent/controller/networkpolicy/reconciler.go index b2d61d18206..372cb3bd3ba 100644 --- a/pkg/agent/controller/networkpolicy/reconciler.go +++ b/pkg/agent/controller/networkpolicy/reconciler.go @@ -543,8 +543,11 @@ func (r *reconciler) getPodIPs(pods v1beta1.GroupMemberPodSet) sets.String { continue } for _, iface := range ifaces { - klog.V(2).Infof("Got IP %v for Pod %s/%s", iface.IP, pod.Pod.Namespace, pod.Pod.Name) - ips.Insert(iface.IP.String()) + ipv4Addr := iface.GetIPv4Addr() + if ipv4Addr != nil { + klog.V(2).Infof("Got IP %v for Pod %s/%s", ipv4Addr, pod.Pod.Namespace, pod.Pod.Name) + ips.Insert(ipv4Addr.String()) + } } } return ips diff --git a/pkg/agent/controller/networkpolicy/reconciler_test.go b/pkg/agent/controller/networkpolicy/reconciler_test.go index e1228031038..677d69402a8 100644 --- a/pkg/agent/controller/networkpolicy/reconciler_test.go +++ b/pkg/agent/controller/networkpolicy/reconciler_test.go @@ -124,13 +124,13 @@ func TestReconcilerReconcile(t *testing.T) { ifaceStore := interfacestore.NewInterfaceStore() ifaceStore.AddInterface(&interfacestore.InterfaceConfig{ InterfaceName: util.GenerateContainerInterfaceName("pod1", "ns1", "container1"), - IP: net.ParseIP("2.2.2.2"), + IPs: []net.IP{net.ParseIP("2.2.2.2")}, ContainerInterfaceConfig: &interfacestore.ContainerInterfaceConfig{PodName: "pod1", PodNamespace: "ns1", ContainerID: "container1"}, OVSPortConfig: &interfacestore.OVSPortConfig{OFPort: 1}, }) ifaceStore.AddInterface(&interfacestore.InterfaceConfig{ InterfaceName: util.GenerateContainerInterfaceName("pod3", "ns1", "container3"), - IP: net.ParseIP("3.3.3.3"), + IPs: []net.IP{net.ParseIP("3.3.3.3")}, ContainerInterfaceConfig: &interfacestore.ContainerInterfaceConfig{PodName: "pod3", PodNamespace: "ns1", ContainerID: "container3"}, OVSPortConfig: &interfacestore.OVSPortConfig{OFPort: 3}, }) @@ -444,13 +444,13 @@ func TestReconcilerUpdate(t *testing.T) { ifaceStore.AddInterface( &interfacestore.InterfaceConfig{ InterfaceName: util.GenerateContainerInterfaceName("pod1", "ns1", "container1"), - IP: net.ParseIP("2.2.2.2"), + IPs: []net.IP{net.ParseIP("2.2.2.2")}, ContainerInterfaceConfig: &interfacestore.ContainerInterfaceConfig{PodName: "pod1", PodNamespace: "ns1", ContainerID: "container1"}, OVSPortConfig: &interfacestore.OVSPortConfig{OFPort: 1}}) ifaceStore.AddInterface( &interfacestore.InterfaceConfig{ InterfaceName: util.GenerateContainerInterfaceName("pod2", "ns1", "container2"), - IP: net.ParseIP("3.3.3.3"), + IPs: []net.IP{net.ParseIP("3.3.3.3")}, ContainerInterfaceConfig: &interfacestore.ContainerInterfaceConfig{PodName: "pod2", PodNamespace: "ns1", ContainerID: "container2"}, OVSPortConfig: &interfacestore.OVSPortConfig{OFPort: 2}}) tests := []struct { diff --git a/pkg/agent/controller/traceflow/traceflow_controller.go b/pkg/agent/controller/traceflow/traceflow_controller.go index 30faebba4c0..865caadaa95 100644 --- a/pkg/agent/controller/traceflow/traceflow_controller.go +++ b/pkg/agent/controller/traceflow/traceflow_controller.go @@ -279,7 +279,7 @@ func (c *Controller) injectPacket(tf *opsv1alpha1.Traceflow) error { dstPodInterfaces := c.interfaceStore.GetContainerInterfacesByPod(tf.Spec.Destination.Pod, tf.Spec.Destination.Namespace) if len(dstPodInterfaces) > 0 { dstMAC = dstPodInterfaces[0].MAC.String() - dstIP = dstPodInterfaces[0].IP.String() + dstIP = dstPodInterfaces[0].GetIPv4Addr().String() } else { dstPod, err := c.kubeClient.CoreV1().Pods(tf.Spec.Destination.Namespace).Get(context.TODO(), tf.Spec.Destination.Pod, v1.GetOptions{}) if err != nil { @@ -331,7 +331,7 @@ func (c *Controller) injectPacket(tf *opsv1alpha1.Traceflow) error { tf.Status.DataplaneTag, podInterfaces[0].MAC.String(), dstMAC, - podInterfaces[0].IP.String(), + podInterfaces[0].GetIPv4Addr().String(), dstIP, uint8(tf.Spec.Packet.IPHeader.Protocol), uint8(tf.Spec.Packet.IPHeader.TTL), diff --git a/pkg/agent/flowexporter/connections/connections_test.go b/pkg/agent/flowexporter/connections/connections_test.go index 0b481ea1d61..12c1ef84eab 100644 --- a/pkg/agent/flowexporter/connections/connections_test.go +++ b/pkg/agent/flowexporter/connections/connections_test.go @@ -98,7 +98,7 @@ func TestConnectionStore_addAndUpdateConn(t *testing.T) { } interfaceFlow2 := &interfacestore.InterfaceConfig{ InterfaceName: "interface2", - IP: net.IP{8, 7, 6, 5}, + IPs: []net.IP{{8, 7, 6, 5}}, ContainerInterfaceConfig: podConfigFlow2, } // Mock interface store with one of the couple of IPs correspond to Pods diff --git a/pkg/agent/flowexporter/connections/conntrack_linux.go b/pkg/agent/flowexporter/connections/conntrack_linux.go index bc1581f372d..06672485b7c 100644 --- a/pkg/agent/flowexporter/connections/conntrack_linux.go +++ b/pkg/agent/flowexporter/connections/conntrack_linux.go @@ -64,6 +64,7 @@ func (ctdump *connTrackDumper) DumpFlows(zoneFilter uint16) ([]*flowexporter.Con } filteredConns := make([]*flowexporter.Connection, 0, len(conns)) +connLoop: for _, conn := range conns { if conn.Zone != openflow.CtZone { continue @@ -71,8 +72,10 @@ func (ctdump *connTrackDumper) DumpFlows(zoneFilter uint16) ([]*flowexporter.Con srcIP := conn.TupleOrig.IP.SourceAddress dstIP := conn.TupleReply.IP.SourceAddress // Only get Pod-to-Pod flows. Pod-to-ExternalService flows are ignored for now. - if srcIP.Equal(ctdump.nodeConfig.GatewayConfig.IP) || dstIP.Equal(ctdump.nodeConfig.GatewayConfig.IP) { - continue + for _, ip := range ctdump.nodeConfig.GatewayConfig.IPs { + if srcIP.Equal(ip) || dstIP.Equal(ip) { + continue connLoop + } } // Pod-to-Service flows w/ kube-proxy: There are two conntrack flows for every Pod-to-Service flow. diff --git a/pkg/agent/flowexporter/connections/conntrack_test.go b/pkg/agent/flowexporter/connections/conntrack_test.go index e88a99e46d7..1d335824d13 100644 --- a/pkg/agent/flowexporter/connections/conntrack_test.go +++ b/pkg/agent/flowexporter/connections/conntrack_test.go @@ -131,7 +131,7 @@ func TestConnTrack_DumpFilter(t *testing.T) { // Create nodeConfig and gateWayConfig // Set antreaGWFlow.TupleOrig.IP.DestinationAddress as gateway IP gwConfig := &config.GatewayConfig{ - IP: net.IP{8, 7, 6, 5}, + IPs: []net.IP{{8, 7, 6, 5}}, } nodeConfig := &config.NodeConfig{ GatewayConfig: gwConfig, diff --git a/pkg/agent/interfacestore/interface_cache.go b/pkg/agent/interfacestore/interface_cache.go index 23970c3c192..d60614bb0a1 100644 --- a/pkg/agent/interfacestore/interface_cache.go +++ b/pkg/agent/interfacestore/interface_cache.go @@ -245,11 +245,15 @@ func podIndexFunc(obj interface{}) ([]string, error) { func interfaceIPIndexFunc(obj interface{}) ([]string, error) { interfaceConfig := obj.(*InterfaceConfig) - if interfaceConfig.IP == nil { + if interfaceConfig.IPs == nil { // If interfaceConfig IP is not set, we return empty key. return []string{}, nil } - return []string{interfaceConfig.IP.String()}, nil + var intfIPs []string + for _, ip := range interfaceConfig.IPs { + intfIPs = append(intfIPs, ip.String()) + } + return intfIPs, nil } func NewInterfaceStore() InterfaceStore { diff --git a/pkg/agent/interfacestore/types.go b/pkg/agent/interfacestore/types.go index 95927cf3e30..27053e82262 100644 --- a/pkg/agent/interfacestore/types.go +++ b/pkg/agent/interfacestore/types.go @@ -18,6 +18,7 @@ import ( "net" "strconv" + "github.com/vmware-tanzu/antrea/pkg/agent/util" "github.com/vmware-tanzu/antrea/pkg/ovs/ovsconfig" ) @@ -64,7 +65,7 @@ type InterfaceConfig struct { Type InterfaceType // Unique name of the interface, also used for the OVS port name. InterfaceName string - IP net.IP + IPs []net.IP MAC net.HardwareAddr *OVSPortConfig *ContainerInterfaceConfig @@ -96,7 +97,7 @@ func NewContainerInterface( podName string, podNamespace string, mac net.HardwareAddr, - ip net.IP) *InterfaceConfig { + ips []net.IP) *InterfaceConfig { containerConfig := &ContainerInterfaceConfig{ ContainerID: containerID, PodName: podName, @@ -104,7 +105,7 @@ func NewContainerInterface( return &InterfaceConfig{ InterfaceName: interfaceName, Type: ContainerInterface, - IP: ip, + IPs: ips, MAC: mac, ContainerInterfaceConfig: containerConfig} } @@ -134,3 +135,7 @@ func NewUplinkInterface(uplinkName string) *InterfaceConfig { uplinkConfig := &InterfaceConfig{InterfaceName: uplinkName, Type: UplinkInterface} return uplinkConfig } + +func (c *InterfaceConfig) GetIPv4Addr() net.IP { + return util.GetIPv4Addr(c.IPs) +} diff --git a/pkg/agent/openflow/client.go b/pkg/agent/openflow/client.go index adae1122f31..5bd3bb07a00 100644 --- a/pkg/agent/openflow/client.go +++ b/pkg/agent/openflow/client.go @@ -25,6 +25,7 @@ import ( "github.com/vmware-tanzu/antrea/pkg/agent/config" "github.com/vmware-tanzu/antrea/pkg/agent/openflow/cookie" "github.com/vmware-tanzu/antrea/pkg/agent/types" + "github.com/vmware-tanzu/antrea/pkg/agent/util" binding "github.com/vmware-tanzu/antrea/pkg/ovs/openflow" "github.com/vmware-tanzu/antrea/third_party/proxy" ) @@ -41,7 +42,7 @@ type Client interface { Initialize(roundInfo types.RoundInfo, config *config.NodeConfig, encapMode config.TrafficEncapModeType, gatewayOFPort uint32) (<-chan struct{}, error) // InstallGatewayFlows sets up flows related to an OVS gateway port, the gateway must exist. - InstallGatewayFlows(gatewayAddr net.IP, gatewayMAC net.HardwareAddr, gatewayOFPort uint32) error + InstallGatewayFlows(gatewayAddrs []net.IP, gatewayMAC net.HardwareAddr, gatewayOFPort uint32) error // InstallBridgeUplinkFlows installs Openflow flows between bridge local port and uplink port to support // host networking. These flows are only needed on windows platform. @@ -84,7 +85,7 @@ type Client interface { // flows will be installed). Calls to InstallPodFlows are idempotent. Concurrent calls // to InstallPodFlows and / or UninstallPodFlows are supported as long as they are all // for different interfaceNames. - InstallPodFlows(interfaceName string, podInterfaceIP net.IP, podInterfaceMAC, gatewayMAC net.HardwareAddr, ofPort uint32) error + InstallPodFlows(interfaceName string, podInterfaceIPs []net.IP, podInterfaceMAC, gatewayMAC net.HardwareAddr, ofPort uint32) error // UninstallPodFlows removes the connection to the local Pod specified with the // interfaceName. UninstallPodFlows will do nothing if no connection to the Pod was established. @@ -303,21 +304,27 @@ func (c *client) UninstallNodeFlows(hostname string) error { return c.deleteFlows(c.nodeFlowCache, hostname) } -func (c *client) InstallPodFlows(interfaceName string, podInterfaceIP net.IP, podInterfaceMAC, gatewayMAC net.HardwareAddr, ofPort uint32) error { +func (c *client) InstallPodFlows(interfaceName string, podInterfaceIPs []net.IP, podInterfaceMAC, gatewayMAC net.HardwareAddr, ofPort uint32) error { c.replayMutex.RLock() defer c.replayMutex.RUnlock() flows := []binding.Flow{ c.podClassifierFlow(ofPort, cookie.Pod), - c.podIPSpoofGuardFlow(podInterfaceIP, podInterfaceMAC, ofPort, cookie.Pod), - c.arpSpoofGuardFlow(podInterfaceIP, podInterfaceMAC, ofPort, cookie.Pod), c.l2ForwardCalcFlow(podInterfaceMAC, ofPort, cookie.Pod), - c.l3FlowsToPod(gatewayMAC, podInterfaceIP, podInterfaceMAC, cookie.Pod), } + // Add support for IPv4 ARP responder. + podInterfaceIPv4 := util.GetIPv4Addr(podInterfaceIPs) + if podInterfaceIPv4 != nil { + flows = append(flows, c.arpSpoofGuardFlow(podInterfaceIPv4, podInterfaceMAC, ofPort, cookie.Pod)) + } + // Add IP SpoofGuard flows for all validate IPs. + flows = append(flows, c.podIPSpoofGuardFlow(podInterfaceIPs, podInterfaceMAC, ofPort, cookie.Pod)...) + // Add L3 Routing flows to rewrite Pod's dst MAC for all validate IPs. + flows = append(flows, c.l3FlowsToPod(gatewayMAC, podInterfaceIPs, podInterfaceMAC, cookie.Pod)...) if c.encapMode.IsNetworkPolicyOnly() { // In policy-only mode, traffic to local Pod is routed based on destination IP. flows = append(flows, - c.l3ToPodFlow(podInterfaceIP, podInterfaceMAC, cookie.Pod), + c.l3ToPodFlow(podInterfaceIPs, podInterfaceMAC, cookie.Pod)..., ) } return c.addFlows(c.podFlowCache, interfaceName, flows) @@ -445,19 +452,25 @@ func (c *client) InstallClusterServiceCIDRFlows(serviceNet *net.IPNet, gatewayMA return nil } -func (c *client) InstallGatewayFlows(gatewayAddr net.IP, gatewayMAC net.HardwareAddr, gatewayOFPort uint32) error { +func (c *client) InstallGatewayFlows(gatewayAddrs []net.IP, gatewayMAC net.HardwareAddr, gatewayOFPort uint32) error { flows := []binding.Flow{ c.gatewayClassifierFlow(gatewayOFPort, cookie.Default), c.gatewayIPSpoofGuardFlow(gatewayOFPort, cookie.Default), - c.gatewayARPSpoofGuardFlow(gatewayOFPort, gatewayAddr, gatewayMAC, cookie.Default), c.ctRewriteDstMACFlow(gatewayMAC, cookie.Default), c.l2ForwardCalcFlow(gatewayMAC, gatewayOFPort, cookie.Default), - c.localProbeFlow(gatewayAddr, cookie.Default), } + // Add ARP SpoofGuard flow for local gateway interface. + gwIPv4 := util.GetIPv4Addr(gatewayAddrs) + if gwIPv4 != nil { + flows = append(flows, c.gatewayARPSpoofGuardFlow(gatewayOFPort, gwIPv4, gatewayMAC, cookie.Default)) + } + // Add flow to ensure the liveness check packet could be forwarded correctly. + flows = append(flows, c.localProbeFlow(gatewayAddrs, cookie.Default)...) + // In NoEncap , no traffic from tunnel port if c.encapMode.SupportsEncap() { - flows = append(flows, c.l3ToGatewayFlow(gatewayAddr, gatewayMAC, cookie.Default)) + flows = append(flows, c.l3ToGatewayFlow(gatewayAddrs, gatewayMAC, cookie.Default)...) } if c.encapMode.SupportsNoEncap() { diff --git a/pkg/agent/openflow/client_test.go b/pkg/agent/openflow/client_test.go index af1acfb6e49..a78be13d6ed 100644 --- a/pkg/agent/openflow/client_test.go +++ b/pkg/agent/openflow/client_test.go @@ -59,7 +59,7 @@ func installPodFlows(ofClient Client, cacheKey string) (int, error) { podMAC, _ := net.ParseMAC("AA:BB:CC:DD:EE:EE") podIP := net.ParseIP("10.0.0.2") ofPort := uint32(10) - err := ofClient.InstallPodFlows(containerID, podIP, podMAC, gwMAC, ofPort) + err := ofClient.InstallPodFlows(containerID, []net.IP{podIP}, podMAC, gwMAC, ofPort) client := ofClient.(*client) fCacheI, ok := client.podFlowCache.Load(containerID) if ok { diff --git a/pkg/agent/openflow/pipeline.go b/pkg/agent/openflow/pipeline.go index f7887ef5ede..4f62bee65d9 100644 --- a/pkg/agent/openflow/pipeline.go +++ b/pkg/agent/openflow/pipeline.go @@ -556,7 +556,7 @@ func (c *client) l3BypassMACRewriteFlow(gatewayMAC net.HardwareAddr, category co } // l3FlowsToPod generates the flow to rewrite MAC if the packet is received from tunnel port and destined for local Pods. -func (c *client) l3FlowsToPod(localGatewayMAC net.HardwareAddr, podInterfaceIP net.IP, podInterfaceMAC net.HardwareAddr, category cookie.Category) binding.Flow { +func (c *client) l3FlowsToPod(localGatewayMAC net.HardwareAddr, podInterfaceIPs []net.IP, podInterfaceMAC net.HardwareAddr, category cookie.Category) []binding.Flow { l3FwdTable := c.pipeline[l3ForwardingTable] flowBuilder := l3FwdTable.BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIP) if c.enableProxy { @@ -564,28 +564,38 @@ func (c *client) l3FlowsToPod(localGatewayMAC net.HardwareAddr, podInterfaceIP n } else { flowBuilder = flowBuilder.MatchDstMAC(globalVirtualMAC) } - // Rewrite src MAC to local gateway MAC, and rewrite dst MAC to pod MAC - return flowBuilder. - MatchDstIP(podInterfaceIP). - Action().SetSrcMAC(localGatewayMAC). - Action().SetDstMAC(podInterfaceMAC). - Action().DecTTL(). - Action().GotoTable(l3FwdTable.GetNext()). - Cookie(c.cookieAllocator.Request(category).Raw()). - Done() + var flows []binding.Flow + for _, ip := range podInterfaceIPs { + ipProtocol := parseIPProtocol(ip) + // Rewrite src MAC to local gateway MAC, and rewrite dst MAC to pod MAC + flows = append(flows, flowBuilder.MatchProtocol(ipProtocol). + MatchDstIP(ip). + Action().SetSrcMAC(localGatewayMAC). + Action().SetDstMAC(podInterfaceMAC). + Action().DecTTL(). + Action().GotoTable(l3FwdTable.GetNext()). + Cookie(c.cookieAllocator.Request(category).Raw()). + Done()) + } + return flows } // l3ToPodFromGwFlow generates the flow to rewrite MAC if the packet IP matches an local IP. // This flow is used in policy only traffic mode. -func (c *client) l3ToPodFlow(podInterfaceIP net.IP, podInterfaceMAC net.HardwareAddr, category cookie.Category) binding.Flow { +func (c *client) l3ToPodFlow(podInterfaceIPs []net.IP, podInterfaceMAC net.HardwareAddr, category cookie.Category) []binding.Flow { l3FwdTable := c.pipeline[l3ForwardingTable] - return l3FwdTable.BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIP). - MatchDstIP(podInterfaceIP). - Action().SetDstMAC(podInterfaceMAC). - Action().DecTTL(). - Action().GotoTable(l3FwdTable.GetNext()). - Cookie(c.cookieAllocator.Request(category).Raw()). - Done() + var flows []binding.Flow + for _, ip := range podInterfaceIPs { + ipProtocol := parseIPProtocol(ip) + flows = append(flows, l3FwdTable.BuildFlow(priorityNormal).MatchProtocol(ipProtocol). + MatchDstIP(ip). + Action().SetDstMAC(podInterfaceMAC). + Action().DecTTL(). + Action().GotoTable(l3FwdTable.GetNext()). + Cookie(c.cookieAllocator.Request(category).Raw()). + Done()) + } + return flows } // l3ToGWFlow generates the flow to rewrite MAC to gw port if the packet received is unmatched by local Pod flows. @@ -601,15 +611,20 @@ func (c *client) l3ToGWFlow(gwMAC net.HardwareAddr, category cookie.Category) bi } // l3ToGatewayFlow generates flow that rewrites MAC of the packet received from tunnel port and destined to local gateway. -func (c *client) l3ToGatewayFlow(localGatewayIP net.IP, localGatewayMAC net.HardwareAddr, category cookie.Category) binding.Flow { +func (c *client) l3ToGatewayFlow(localGatewayIPs []net.IP, localGatewayMAC net.HardwareAddr, category cookie.Category) []binding.Flow { l3FwdTable := c.pipeline[l3ForwardingTable] - return l3FwdTable.BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIP). - MatchDstMAC(globalVirtualMAC). - MatchDstIP(localGatewayIP). - Action().SetDstMAC(localGatewayMAC). - Action().GotoTable(l3FwdTable.GetNext()). - Cookie(c.cookieAllocator.Request(category).Raw()). - Done() + var flows []binding.Flow + for _, ip := range localGatewayIPs { + ipProtocol := parseIPProtocol(ip) + flows = append(flows, l3FwdTable.BuildFlow(priorityNormal).MatchProtocol(ipProtocol). + MatchDstMAC(globalVirtualMAC). + MatchDstIP(ip). + Action().SetDstMAC(localGatewayMAC). + Action().GotoTable(l3FwdTable.GetNext()). + Cookie(c.cookieAllocator.Request(category).Raw()). + Done()) + } + return flows } // l3FwdFlowToRemote generates the L3 forward flow on source node to support traffic to remote pods/gateway. @@ -692,16 +707,32 @@ func (c *client) arpResponderStaticFlow(category cookie.Category) binding.Flow { // podIPSpoofGuardFlow generates the flow to check IP traffic sent out from local pod. Traffic from host gateway interface // will not be checked, since it might be pod to service traffic or host namespace traffic. -func (c *client) podIPSpoofGuardFlow(ifIP net.IP, ifMAC net.HardwareAddr, ifOFPort uint32, category cookie.Category) binding.Flow { +func (c *client) podIPSpoofGuardFlow(ifIPs []net.IP, ifMAC net.HardwareAddr, ifOFPort uint32, category cookie.Category) []binding.Flow { ipPipeline := c.pipeline ipSpoofGuardTable := ipPipeline[spoofGuardTable] - return ipSpoofGuardTable.BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIP). - MatchInPort(ifOFPort). - MatchSrcMAC(ifMAC). - MatchSrcIP(ifIP). - Action().GotoTable(ipSpoofGuardTable.GetNext()). - Cookie(c.cookieAllocator.Request(category).Raw()). - Done() + var flows []binding.Flow + for _, ifIP := range ifIPs { + ipProtocol := parseIPProtocol(ifIP) + flow := ipSpoofGuardTable.BuildFlow(priorityNormal).MatchProtocol(ipProtocol). + MatchInPort(ifOFPort). + MatchSrcMAC(ifMAC). + MatchSrcIP(ifIP). + Action().GotoTable(ipSpoofGuardTable.GetNext()). + Cookie(c.cookieAllocator.Request(category).Raw()). + Done() + flows = append(flows, flow) + } + return flows +} + +func parseIPProtocol(ip net.IP) binding.Protocol { + var ipProtocol binding.Protocol + if ip.To4() != nil { + ipProtocol = binding.ProtocolIP + } else { + ipProtocol = binding.ProtocolIPv6 + } + return ipProtocol } // serviceHairpinResponseDNATFlow generates the flow which transforms destination @@ -942,13 +973,18 @@ func (c *client) defaultDropFlow(tableID binding.TableIDType, matchKey int, matc } // localProbeFlow generates the flow to forward packets to conntrackCommitTable. The packets are sent from Node to probe the liveness/readiness of local Pods. -func (c *client) localProbeFlow(localGatewayIP net.IP, category cookie.Category) binding.Flow { - return c.pipeline[IngressRuleTable].BuildFlow(priorityHigh). - MatchProtocol(binding.ProtocolIP). - MatchSrcIP(localGatewayIP). - Action().GotoTable(conntrackCommitTable). - Cookie(c.cookieAllocator.Request(category).Raw()). - Done() +func (c *client) localProbeFlow(localGatewayIPs []net.IP, category cookie.Category) []binding.Flow { + var flows []binding.Flow + for _, ip := range localGatewayIPs { + ipProtocol := parseIPProtocol(ip) + flows = append(flows, c.pipeline[IngressRuleTable].BuildFlow(priorityHigh). + MatchProtocol(ipProtocol). + MatchSrcIP(ip). + Action().GotoTable(conntrackCommitTable). + Cookie(c.cookieAllocator.Request(category).Raw()). + Done()) + } + return flows } func (c *client) bridgeAndUplinkFlows(uplinkOfport uint32, bridgeLocalPort uint32, nodeIP net.IP, localSubnet net.IPNet, category cookie.Category) []binding.Flow { diff --git a/pkg/agent/openflow/testing/mock_openflow.go b/pkg/agent/openflow/testing/mock_openflow.go index 6e1d1925a19..c0bfc0a658a 100644 --- a/pkg/agent/openflow/testing/mock_openflow.go +++ b/pkg/agent/openflow/testing/mock_openflow.go @@ -294,7 +294,7 @@ func (mr *MockClientMockRecorder) InstallExternalFlows(arg0, arg1 interface{}) * } // InstallGatewayFlows mocks base method -func (m *MockClient) InstallGatewayFlows(arg0 net.IP, arg1 net.HardwareAddr, arg2 uint32) error { +func (m *MockClient) InstallGatewayFlows(arg0 []net.IP, arg1 net.HardwareAddr, arg2 uint32) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "InstallGatewayFlows", arg0, arg1, arg2) ret0, _ := ret[0].(error) @@ -322,7 +322,7 @@ func (mr *MockClientMockRecorder) InstallNodeFlows(arg0, arg1, arg2, arg3, arg4, } // InstallPodFlows mocks base method -func (m *MockClient) InstallPodFlows(arg0 string, arg1 net.IP, arg2, arg3 net.HardwareAddr, arg4 uint32) error { +func (m *MockClient) InstallPodFlows(arg0 string, arg1 []net.IP, arg2, arg3 net.HardwareAddr, arg4 uint32) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "InstallPodFlows", arg0, arg1, arg2, arg3, arg4) ret0, _ := ret[0].(error) diff --git a/pkg/agent/querier/querier.go b/pkg/agent/querier/querier.go index 34a24795f5a..101c25cb070 100644 --- a/pkg/agent/querier/querier.go +++ b/pkg/agent/querier/querier.go @@ -193,8 +193,8 @@ func (aq agentQuerier) GetAgentInfo(agentInfo *v1beta1.AntreaAgentInfo, partial agentInfo.Version = querier.GetVersion() agentInfo.PodRef = querier.GetSelfPod() agentInfo.NodeRef = querier.GetSelfNode(true, aq.nodeConfig.Name) - if aq.nodeConfig.PodCIDR != nil { - agentInfo.NodeSubnet = []string{aq.nodeConfig.PodCIDR.String()} + if aq.nodeConfig.PodIPv4CIDR != nil { + agentInfo.NodeSubnet = append(agentInfo.NodeSubnet, aq.nodeConfig.PodIPv4CIDR.String()) } agentInfo.OVSInfo.BridgeName = aq.nodeConfig.OVSBridge agentInfo.APIPort = aq.apiPort diff --git a/pkg/agent/querier/querier_test.go b/pkg/agent/querier/querier_test.go index 6f6d57f9610..5460de4b52f 100644 --- a/pkg/agent/querier/querier_test.go +++ b/pkg/agent/querier/querier_test.go @@ -120,10 +120,10 @@ func TestAgentQuerierGetAgentInfo(t *testing.T) { { name: "encap-mode non-partial", nodeConfig: &config.NodeConfig{ - Name: "foo", - OVSBridge: "br-int", - NodeIPAddr: getIPNet("10.10.0.10"), - PodCIDR: getIPNet("20.20.20.0/24"), + Name: "foo", + OVSBridge: "br-int", + NodeIPAddr: getIPNet("10.10.0.10"), + PodIPv4CIDR: getIPNet("20.20.20.0/24"), }, apiPort: 10350, partial: false, diff --git a/pkg/agent/route/route_linux.go b/pkg/agent/route/route_linux.go index e1dbd956a8d..4c622f771c8 100644 --- a/pkg/agent/route/route_linux.go +++ b/pkg/agent/route/route_linux.go @@ -165,7 +165,7 @@ func (c *Client) initIPSet() error { return err } // Ensure its own PodCIDR is in it. - if err := ipset.AddEntry(antreaPodIPSet, c.nodeConfig.PodCIDR.String()); err != nil { + if err := ipset.AddEntry(antreaPodIPSet, c.nodeConfig.PodIPv4CIDR.String()); err != nil { return err } return nil @@ -269,7 +269,7 @@ func (c *Client) initIPTables() error { writeLine(iptablesData, []string{ "-A", antreaPostRoutingChain, "-m", "comment", "--comment", `"Antrea: masquerade pod to external packets"`, - "-s", c.nodeConfig.PodCIDR.String(), "-m", "set", "!", "--match-set", antreaPodIPSet, "dst", + "-s", c.nodeConfig.PodIPv4CIDR.String(), "-m", "set", "!", "--match-set", antreaPodIPSet, "dst", "-j", iptables.MasqueradeTarget, }...) } @@ -407,7 +407,7 @@ func (c *Client) AddRoutes(podCIDR *net.IPNet, nodeIP, nodeGwIP net.IP) error { return nil } -// DeleteRoutes deletes routes to a PodCIDR. It does nothing if the routes doesn't exist. +// DeleteRoutes deletes routes to a PodCIDRs. It does nothing if the routes doesn't exist. func (c *Client) DeleteRoutes(podCIDR *net.IPNet) error { podCIDRStr := podCIDR.String() // Delete this podCIDR from antreaPodIPSet as the CIDR is no longer for Pods. @@ -509,7 +509,7 @@ func (c *Client) addServiceRouting() error { route := &netlink.Route{ LinkIndex: gwConfig.LinkIndex, Scope: netlink.SCOPE_LINK, - Dst: c.nodeConfig.PodCIDR, + Dst: c.nodeConfig.PodIPv4CIDR, Table: c.serviceRtTable.Idx, } if err := netlink.RouteReplace(route); err != nil { diff --git a/pkg/agent/route/route_windows.go b/pkg/agent/route/route_windows.go index 267f32e9fac..9d4f5077c40 100644 --- a/pkg/agent/route/route_windows.go +++ b/pkg/agent/route/route_windows.go @@ -189,11 +189,14 @@ func (c *Client) listRoutes() (map[string]*netroute.Route, error) { // initFwRules adds Windows Firewall rules to accept the traffic that is sent to or from local Pods. func (c *Client) initFwRules() error { - err := c.fwClient.AddRuleAllowIP(inboundFirewallRuleName, winfirewall.FWRuleIn, c.nodeConfig.PodCIDR) + if c.nodeConfig.PodIPv4CIDR == nil { + return errors.New("no valid IPv4 PodCIDR") + } + err := c.fwClient.AddRuleAllowIP(inboundFirewallRuleName, winfirewall.FWRuleIn, c.nodeConfig.PodIPv4CIDR) if err != nil { return err } - err = c.fwClient.AddRuleAllowIP(outboundFirewallRuleName, winfirewall.FWRuleOut, c.nodeConfig.PodCIDR) + err = c.fwClient.AddRuleAllowIP(outboundFirewallRuleName, winfirewall.FWRuleOut, c.nodeConfig.PodIPv4CIDR) if err != nil { return err } diff --git a/pkg/agent/util/net.go b/pkg/agent/util/net.go index 195ced6294c..a70e170a0ef 100644 --- a/pkg/agent/util/net.go +++ b/pkg/agent/util/net.go @@ -118,3 +118,12 @@ func GetIPNetDeviceFromIP(localIP net.IP) (*net.IPNet, *net.Interface, error) { } return nil, nil, fmt.Errorf("unable to find local IP and device") } + +func GetIPv4Addr(ips []net.IP) net.IP { + for _, ip := range ips { + if ip.To4() != nil { + return ip + } + } + return nil +} diff --git a/pkg/ovs/openflow/interfaces.go b/pkg/ovs/openflow/interfaces.go index 7588b9463c9..f707917f6e9 100644 --- a/pkg/ovs/openflow/interfaces.go +++ b/pkg/ovs/openflow/interfaces.go @@ -35,12 +35,17 @@ const ( ) const ( - ProtocolIP Protocol = "ip" - ProtocolARP Protocol = "arp" - ProtocolTCP Protocol = "tcp" - ProtocolUDP Protocol = "udp" - ProtocolSCTP Protocol = "sctp" - ProtocolICMP Protocol = "icmp" + ProtocolIP Protocol = "ip" + ProtocolIPv6 Protocol = "ipv6" + ProtocolARP Protocol = "arp" + ProtocolTCP Protocol = "tcp" + ProtocolTCPv6 Protocol = "tcpv6" + ProtocolUDP Protocol = "udp" + ProtocolUDPv6 Protocol = "udpv6" + ProtocolSCTP Protocol = "sctp" + ProtocolSCTPv6 Protocol = "sctpv6" + ProtocolICMP Protocol = "icmp" + ProtocolICMPv6 Protocol = "icmpv6" ) const ( diff --git a/pkg/ovs/openflow/ofctrl_builder.go b/pkg/ovs/openflow/ofctrl_builder.go index 1fc167e228a..4e862cff662 100644 --- a/pkg/ovs/openflow/ofctrl_builder.go +++ b/pkg/ovs/openflow/ofctrl_builder.go @@ -292,20 +292,34 @@ func (b *ofFlowBuilder) MatchProtocol(protocol Protocol) FlowBuilder { switch protocol { case ProtocolIP: b.Match.Ethertype = 0x0800 + case ProtocolIPv6: + b.Match.Ethertype = 0x86dd case ProtocolARP: b.Match.Ethertype = 0x0806 case ProtocolTCP: b.Match.Ethertype = 0x0800 b.Match.IpProto = 6 + case ProtocolTCPv6: + b.Match.Ethertype = 0x86dd + b.Match.IpProto = 6 case ProtocolUDP: b.Match.Ethertype = 0x0800 b.Match.IpProto = 17 + case ProtocolUDPv6: + b.Match.Ethertype = 0x86dd + b.Match.IpProto = 17 case ProtocolSCTP: b.Match.Ethertype = 0x0800 b.Match.IpProto = 132 + case ProtocolSCTPv6: + b.Match.Ethertype = 0x86dd + b.Match.IpProto = 132 case ProtocolICMP: b.Match.Ethertype = 0x0800 b.Match.IpProto = 1 + case ProtocolICMPv6: + b.Match.Ethertype = 0x86dd + b.Match.IpProto = 58 } b.protocol = protocol return b diff --git a/test/integration/agent/cniserver_test.go b/test/integration/agent/cniserver_test.go index ef2c096a6e2..40681c652ec 100644 --- a/test/integration/agent/cniserver_test.go +++ b/test/integration/agent/cniserver_test.go @@ -606,7 +606,7 @@ func cmdAddDelCheckTest(testNS ns.NetNS, tc testCase, dataDir string) { ovsServiceMock.EXPECT().GetOFPort(ovsPortname).Return(int32(10), nil).AnyTimes() ofServiceMock.EXPECT().InstallPodFlows(ovsPortname, mock.Any(), mock.Any(), mock.Any(), mock.Any()).Return(nil) - // Test ip allocation + // Test ips allocation prevResult, err := tester.cmdAddTest(tc, dataDir) testRequire.Nil(err) @@ -812,7 +812,7 @@ func TestCNIServerChaining(t *testing.T) { routeMock.EXPECT().MigrateRoutesToGw(hostVeth.Name), ovsServiceMock.EXPECT().CreatePort(ovsPortname, ovsPortname, mock.Any()).Return(ovsPortUUID, nil), ovsServiceMock.EXPECT().GetOFPort(ovsPortname).Return(testContainerOFPort, nil), - ofServiceMock.EXPECT().InstallPodFlows(ovsPortname, podIP, containerIntf.HardwareAddr, gwMAC, mock.Any()), + ofServiceMock.EXPECT().InstallPodFlows(ovsPortname, []net.IP{podIP}, containerIntf.HardwareAddr, gwMAC, mock.Any()), ) mock.InOrder(orderedCalls...) cniResp, err := server.CmdAdd(ctx, cniReq) @@ -843,9 +843,9 @@ func init() { nodeName := "node1" gwIP := net.ParseIP("192.168.1.1") gwMAC, _ = net.ParseMAC("11:11:11:11:11:11") - nodeGateway := &config.GatewayConfig{IP: gwIP, MAC: gwMAC, Name: ""} + nodeGateway := &config.GatewayConfig{IPs: []net.IP{gwIP}, MAC: gwMAC, Name: ""} _, nodePodCIDR, _ := net.ParseCIDR("192.168.1.0/24") nodeMTU := 1500 - testNodeConfig = &config.NodeConfig{Name: nodeName, PodCIDR: nodePodCIDR, NodeMTU: nodeMTU, GatewayConfig: nodeGateway} + testNodeConfig = &config.NodeConfig{Name: nodeName, PodIPv4CIDR: nodePodCIDR, NodeMTU: nodeMTU, GatewayConfig: nodeGateway} } diff --git a/test/integration/agent/openflow_test.go b/test/integration/agent/openflow_test.go index 1caca3e2548..93242ae0d20 100644 --- a/test/integration/agent/openflow_test.go +++ b/test/integration/agent/openflow_test.go @@ -59,7 +59,7 @@ type expectTableFlows struct { } type testPortConfig struct { - ip net.IP + ips []net.IP mac net.HardwareAddr ofPort uint32 } @@ -279,11 +279,11 @@ func testUninstallNodeFlows(t *testing.T, config *testConfig) { func testInstallPodFlows(t *testing.T, config *testConfig) { for _, pod := range config.localPods { - err := c.InstallPodFlows(pod.name, pod.ip, pod.mac, config.localGateway.mac, pod.ofPort) + err := c.InstallPodFlows(pod.name, pod.ips, pod.mac, config.localGateway.mac, pod.ofPort) if err != nil { t.Fatalf("Failed to install Openflow entries for pod: %v", err) } - for _, tableFlow := range preparePodFlows(pod.ip, pod.mac, pod.ofPort, config.localGateway.mac, config.globalMAC) { + for _, tableFlow := range preparePodFlows(pod.ips, pod.mac, pod.ofPort, config.localGateway.mac, config.globalMAC) { ofTestUtils.CheckFlowExists(t, ovsCtlClient, tableFlow.tableID, true, tableFlow.flows) } } @@ -295,7 +295,7 @@ func testUninstallPodFlows(t *testing.T, config *testConfig) { if err != nil { t.Fatalf("Failed to uninstall Openflow entries for pod: %v", err) } - for _, tableFlow := range preparePodFlows(pod.ip, pod.mac, pod.ofPort, config.localGateway.mac, config.globalMAC) { + for _, tableFlow := range preparePodFlows(pod.ips, pod.mac, pod.ofPort, config.localGateway.mac, config.globalMAC) { ofTestUtils.CheckFlowExists(t, ovsCtlClient, tableFlow.tableID, false, tableFlow.flows) } } @@ -546,11 +546,11 @@ func checkOVSFlowMetrics(t *testing.T, client ofClient.Client) { } func testInstallGatewayFlows(t *testing.T, config *testConfig) { - err := c.InstallGatewayFlows(config.localGateway.ip, config.localGateway.mac, config.localGateway.ofPort) + err := c.InstallGatewayFlows(config.localGateway.ips, config.localGateway.mac, config.localGateway.ofPort) if err != nil { t.Fatalf("Failed to install Openflow entries for gateway: %v", err) } - for _, tableFlow := range prepareGatewayFlows(config.localGateway.ip, config.localGateway.mac, config.localGateway.ofPort, config.globalMAC) { + for _, tableFlow := range prepareGatewayFlows(config.localGateway.ips, config.localGateway.mac, config.localGateway.ofPort, config.globalMAC) { ofTestUtils.CheckFlowExists(t, ovsCtlClient, tableFlow.tableID, true, tableFlow.flows) } } @@ -561,13 +561,13 @@ func prepareConfiguration() *testConfig { podCfg := &testLocalPodConfig{ name: "container-1", testPortConfig: &testPortConfig{ - ip: net.ParseIP("192.168.1.3"), + ips: []net.IP{net.ParseIP("192.168.1.3")}, mac: podMAC, ofPort: uint32(3), }, } gwCfg := &testPortConfig{ - ip: net.ParseIP("192.168.1.1"), + ips: []net.IP{net.ParseIP("192.168.1.1")}, mac: gwMAC, ofPort: uint32(1), } @@ -591,32 +591,14 @@ func prepareConfiguration() *testConfig { } } -func preparePodFlows(podIP net.IP, podMAC net.HardwareAddr, podOFPort uint32, gwMAC, vMAC net.HardwareAddr) []expectTableFlows { - return []expectTableFlows{ +func preparePodFlows(podIPs []net.IP, podMAC net.HardwareAddr, podOFPort uint32, gwMAC, vMAC net.HardwareAddr) []expectTableFlows { + flows := []expectTableFlows{ { uint8(0), []*ofTestUtils.ExpectFlow{ {fmt.Sprintf("priority=190,in_port=%d", podOFPort), "load:0x2->NXM_NX_REG0[0..15],goto_table:10"}, }, }, - { - uint8(10), - []*ofTestUtils.ExpectFlow{ - {fmt.Sprintf("priority=200,ip,in_port=%d,dl_src=%s,nw_src=%s", podOFPort, podMAC.String(), podIP.String()), - "goto_table:29"}, - { - fmt.Sprintf("priority=200,arp,in_port=%d,arp_spa=%s,arp_sha=%s", podOFPort, podIP.String(), podMAC.String()), - "goto_table:20"}, - }, - }, - { - uint8(70), - []*ofTestUtils.ExpectFlow{ - { - fmt.Sprintf("priority=200,ip,reg0=0x80000/0x80000,nw_dst=%s", podIP.String()), - fmt.Sprintf("set_field:%s->eth_src,set_field:%s->eth_dst,dec_ttl,goto_table:80", gwMAC.String(), podMAC.String())}, - }, - }, { uint8(80), []*ofTestUtils.ExpectFlow{ @@ -626,10 +608,47 @@ func preparePodFlows(podIP net.IP, podMAC net.HardwareAddr, podOFPort uint32, gw }, }, } + + for _, podIP := range podIPs { + var ipProto string + if podIP.To4() != nil { + ipProto = "ip" + flows = append(flows, + expectTableFlows{ + uint8(10), + []*ofTestUtils.ExpectFlow{ + { + fmt.Sprintf("priority=200,arp,in_port=%d,arp_spa=%s,arp_sha=%s", podOFPort, podIP.String(), podMAC.String()), + "goto_table:20"}, + }, + }) + } else { + ipProto = "ipv6" + } + flows = append(flows, + expectTableFlows{ + uint8(10), + []*ofTestUtils.ExpectFlow{ + {fmt.Sprintf("priority=200,%s,in_port=%d,dl_src=%s,nw_src=%s", ipProto, podOFPort, podMAC.String(), podIP.String()), + "goto_table:29"}, + }, + }, + expectTableFlows{ + uint8(70), + []*ofTestUtils.ExpectFlow{ + { + fmt.Sprintf("priority=200,ip,reg0=0x80000/0x80000,nw_dst=%s", podIP.String()), + fmt.Sprintf("set_field:%s->eth_src,set_field:%s->eth_dst,dec_ttl,goto_table:80", gwMAC.String(), podMAC.String())}, + }, + }, + ) + } + + return flows } -func prepareGatewayFlows(gwIP net.IP, gwMAC net.HardwareAddr, gwOFPort uint32, vMAC net.HardwareAddr) []expectTableFlows { - return []expectTableFlows{ +func prepareGatewayFlows(gwIPs []net.IP, gwMAC net.HardwareAddr, gwOFPort uint32, vMAC net.HardwareAddr) []expectTableFlows { + flows := []expectTableFlows{ { uint8(0), []*ofTestUtils.ExpectFlow{ @@ -644,21 +663,6 @@ func prepareGatewayFlows(gwIP net.IP, gwMAC net.HardwareAddr, gwOFPort uint32, v fmt.Sprintf("load:0x%s->NXM_OF_ETH_DST[],goto_table:42", strings.Replace(gwMAC.String(), ":", "", -1))}, }, }, - { - uint8(10), - []*ofTestUtils.ExpectFlow{ - {fmt.Sprintf("priority=200,arp,in_port=%d,arp_spa=%s,arp_sha=%s", gwOFPort, gwIP, gwMAC), "goto_table:20"}, - {fmt.Sprintf("priority=200,ip,in_port=%d", gwOFPort), "goto_table:29"}, - }, - }, - { - uint8(70), - []*ofTestUtils.ExpectFlow{ - { - fmt.Sprintf("priority=200,ip,dl_dst=%s,nw_dst=%s", vMAC.String(), gwIP.String()), - fmt.Sprintf("set_field:%s->eth_dst,goto_table:80", gwMAC.String())}, - }, - }, { uint8(80), []*ofTestUtils.ExpectFlow{ @@ -667,15 +671,44 @@ func prepareGatewayFlows(gwIP net.IP, gwMAC net.HardwareAddr, gwOFPort uint32, v fmt.Sprintf("load:0x%x->NXM_NX_REG1[],load:0x1->NXM_NX_REG0[16],goto_table:85", gwOFPort)}, }, }, - { - uint8(90), - []*ofTestUtils.ExpectFlow{ - { - fmt.Sprintf("priority=210,ip,nw_src=%s", gwIP.String()), - "goto_table:105"}, + } + + for _, gwIP := range gwIPs { + var ipProtoStr string + if gwIP.To4() != nil { + ipProtoStr = "ip" + flows = append(flows, + expectTableFlows{ + uint8(10), + []*ofTestUtils.ExpectFlow{ + {fmt.Sprintf("priority=200,arp,in_port=%d,arp_spa=%s,arp_sha=%s", gwOFPort, gwIP, gwMAC), "goto_table:20"}, + {fmt.Sprintf("priority=200,ip,in_port=%d", gwOFPort), "goto_table:29"}, + }, + }) + } else { + ipProtoStr = "ipv6" + } + flows = append(flows, + expectTableFlows{ + uint8(70), + []*ofTestUtils.ExpectFlow{ + { + fmt.Sprintf("priority=200,%s,dl_dst=%s,nw_dst=%s", ipProtoStr, vMAC.String(), gwIP.String()), + fmt.Sprintf("set_field:%s->eth_dst,goto_table:80", gwMAC.String())}, + }, }, - }, + expectTableFlows{ + tableID: uint8(90), + flows: []*ofTestUtils.ExpectFlow{ + { + fmt.Sprintf("priority=210,%s,nw_src=%s", ipProtoStr, gwIP.String()), + "goto_table:105"}, + }, + }, + ) } + + return flows } func prepareTunnelFlows(tunnelPort uint32, vMAC net.HardwareAddr) []expectTableFlows { diff --git a/test/integration/agent/route_test.go b/test/integration/agent/route_test.go index b6c2c8b4f5f..344aecea150 100644 --- a/test/integration/agent/route_test.go +++ b/test/integration/agent/route_test.go @@ -61,10 +61,10 @@ var ( svcTblIdx = route.AntreaServiceTableIdx svcTblName = route.AntreaServiceTable mainTblIdx = 254 - gwConfig = &config.GatewayConfig{IP: gwIP, MAC: gwMAC, Name: gwName} + gwConfig = &config.GatewayConfig{IPs: []net.IP{gwIP}, MAC: gwMAC, Name: gwName} nodeConfig = &config.NodeConfig{ Name: "test", - PodCIDR: podCIDR, + PodIPv4CIDR: podCIDR, NodeIPAddr: nodeIP, GatewayConfig: gwConfig, }