diff --git a/ci/jenkins/test.sh b/ci/jenkins/test.sh index c6593299af6..c4def3d3a5e 100755 --- a/ci/jenkins/test.sh +++ b/ci/jenkins/test.sh @@ -373,8 +373,9 @@ function deliver_antrea_windows { sleep 5 # Some tests need us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.13 image but it is not for windows/amd64 10.0.17763 # Use e2eteam/agnhost:2.13 instead - harbor_images=("sigwindowstools-kube-proxy:v1.18.0" "agnhost:2.13" "agnhost:2.13" "agnhost:2.29" "e2eteam-jessie-dnsutils:1.0" "e2eteam-pause:3.2") - antrea_images=("sigwindowstools/kube-proxy:v1.18.0" "e2eteam/agnhost:2.13" "us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.13" "k8s.gcr.io/e2e-test-images/agnhost:2.29" "e2eteam/jessie-dnsutils:1.0" "e2eteam/pause:3.2") + harbor_images=("sigwindowstools-kube-proxy:v1.18.0" "agnhost:2.13" "agnhost:2.13" "agnhost:2.13" "agnhost:2.29" "e2eteam-jessie-dnsutils:1.0" "e2eteam-jessie-dnsutils:1.0" "e2eteam-pause:3.2" "e2eteam-pause:3.2" "e2eteam-busybox:1.29-windows-amd64-1809") + antrea_images=("sigwindowstools/kube-proxy:v1.18.0" "e2eteam/agnhost:2.13" "us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.13" "k8sprow.azurecr.io/kubernetes-e2e-test-images/agnhost:2.13" "k8s.gcr.io/e2e-test-images/agnhost:2.29" "e2eteam/jessie-dnsutils:1.0" "gcr.io/kubernetes-e2e-test-images/jessie-dnsutils:1.0" "e2eteam/pause:3.2" "k8s.gcr.io/pause:3.2" "docker.io/library/busybox:1.29") + common_images=("mcr.microsoft.com/windows/servercore/iis:latest") # Pull necessary images in advance to avoid transient error for i in "${!harbor_images[@]}"; do ssh -o StrictHostKeyChecking=no -n Administrator@${IP} "docker pull -q ${DOCKER_REGISTRY}/antrea/${harbor_images[i]} && docker tag ${DOCKER_REGISTRY}/antrea/${harbor_images[i]} ${antrea_images[i]}" || true @@ -410,7 +411,7 @@ function deliver_antrea_windows { done echo "=== Build Windows on Windows Node===" ssh -o StrictHostKeyChecking=no -n Administrator@${IP} "docker pull ${DOCKER_REGISTRY}/antrea/golang:${GO_VERSION}-nanoserver && docker tag ${DOCKER_REGISTRY}/antrea/golang:${GO_VERSION}-nanoserver golang:${GO_VERSION}-nanoserver" - ssh -o StrictHostKeyChecking=no -n Administrator@${IP} "rm -rf antrea && mkdir antrea && cd antrea && tar -xzf ../antrea_repo.tar.gz > /dev/null && sed -i \"s|build/images/base-windows/Dockerfile|build/images/base-windows/Dockerfile --network host|g\" Makefile && sed -i \"s|build/images/Dockerfile.build.windows|build/images/Dockerfile.build.windows --network host|g\" Makefile && NO_PULL=${NO_PULL} make build-windows && docker save -o antrea-windows.tar ${DOCKER_REGISTRY}/antrea/antrea-windows:latest && gzip -f antrea-windows.tar" || true + ssh -o StrictHostKeyChecking=no -n Administrator@${IP} "rm -rf antrea && mkdir antrea && cd antrea && tar -xzf ../antrea_repo.tar.gz > /dev/null && NO_PULL=${NO_PULL}; DOCKER_NETWORK=host make build-windows && docker save -o antrea-windows.tar antrea/antrea-windows:latest && gzip -f antrea-windows.tar" || true for i in `seq 2`; do timeout 2m scp -o StrictHostKeyChecking=no -T Administrator@${IP}:antrea/antrea-windows.tar.gz . && break done diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go index a2439be3cd9..970d57ba445 100644 --- a/cmd/antrea-agent/agent.go +++ b/cmd/antrea-agent/agent.go @@ -232,6 +232,7 @@ func run(o *Options) error { k8sClient, crdClient, ovsBridgeClient, + ovsctl.NewClient(o.config.OVSBridge), ofClient, routeClient, ifaceStore, @@ -268,6 +269,7 @@ func run(o *Options) error { k8sClient, informerFactory, ofClient, + ovsctl.NewClient(o.config.OVSBridge), ovsBridgeClient, routeClient, ifaceStore, diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index d23059cdc8c..c7be57f31b6 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -95,6 +95,7 @@ type Initializer struct { client clientset.Interface crdClient versioned.Interface ovsBridgeClient ovsconfig.OVSBridgeClient + ovsCtlClient ovsctl.OVSCtlClient ofClient openflow.Client routeClient route.Interface wireGuardClient wireguard.Interface @@ -122,6 +123,7 @@ func NewInitializer( k8sClient clientset.Interface, crdClient versioned.Interface, ovsBridgeClient ovsconfig.OVSBridgeClient, + ovsCtlClient ovsctl.OVSCtlClient, ofClient openflow.Client, routeClient route.Interface, ifaceStore interfacestore.InterfaceStore, @@ -142,6 +144,7 @@ func NewInitializer( ) *Initializer { return &Initializer{ ovsBridgeClient: ovsBridgeClient, + ovsCtlClient: ovsCtlClient, client: k8sClient, crdClient: crdClient, ifaceStore: ifaceStore, @@ -279,7 +282,6 @@ func (i *Initializer) initInterfaceStore() error { return intf } ifaceList := make([]*interfacestore.InterfaceConfig, 0, len(ovsPorts)) - ovsCtlClient := ovsctl.NewClient(i.ovsBridge) for index := range ovsPorts { port := &ovsPorts[index] ovsPort := &interfacestore.OVSPortConfig{ @@ -297,6 +299,8 @@ func (i *Initializer) initInterfaceStore() error { case interfacestore.AntreaUplink: intf = parseUplinkInterfaceFunc(port, ovsPort) case interfacestore.AntreaTunnel: + fallthrough + case interfacestore.AntreaIPsecTunnel: intf = parseTunnelInterfaceFunc(port, ovsPort) case interfacestore.AntreaHost: if port.Name == i.ovsBridge { @@ -314,9 +318,6 @@ func (i *Initializer) initInterfaceStore() error { intf = cniserver.ParseOVSPortInterfaceConfig(port, ovsPort, true) case interfacestore.AntreaTrafficControl: intf = trafficcontrol.ParseTrafficControlInterfaceConfig(port, ovsPort) - if err := ovsCtlClient.SetPortNoFlood(int(ovsPort.OFPort)); err != nil { - klog.ErrorS(err, "Failed to set port with no-flood config", "PortName", port.Name) - } default: klog.InfoS("Unknown Antrea interface type", "type", interfaceType) } @@ -340,7 +341,11 @@ func (i *Initializer) initInterfaceStore() error { fallthrough case port.IFType == ovsconfig.STTTunnel: intf = parseTunnelInterfaceFunc(port, ovsPort) - antreaIFType = interfacestore.AntreaTunnel + if intf.Type == interfacestore.IPSecTunnelInterface { + antreaIFType = interfacestore.AntreaIPsecTunnel + } else { + antreaIFType = interfacestore.AntreaTunnel + } case port.Name == i.ovsBridge: intf = nil antreaIFType = interfacestore.AntreaHost @@ -368,6 +373,26 @@ func (i *Initializer) initInterfaceStore() error { return nil } +func (i *Initializer) restorePortConfigs() error { + interfaces := i.ifaceStore.ListInterfaces() + for _, intf := range interfaces { + switch intf.Type { + case interfacestore.IPSecTunnelInterface: + fallthrough + case interfacestore.TrafficControlInterface: + if intf.OFPort < 0 { + klog.InfoS("Skipped setting no-flood for port due to invalid ofPort", "port", intf.InterfaceName, "ofport", intf.OFPort) + continue + } + if err := i.ovsCtlClient.SetPortNoFlood(int(intf.OFPort)); err != nil { + return fmt.Errorf("failed to set no-flood for port %s: %w", intf.InterfaceName, err) + } + klog.InfoS("Set no-flood for port", "port", intf.InterfaceName) + } + } + return nil +} + // Initialize sets up agent initial configurations. func (i *Initializer) Initialize() error { klog.Info("Setting up node network") @@ -386,6 +411,10 @@ func (i *Initializer) Initialize() error { return err } + if err := i.restorePortConfigs(); err != nil { + return err + } + // initializeWireGuard must be executed after setupOVSBridge as it requires gateway addresses on the OVS bridge. if i.networkConfig.TrafficEncryptionMode == config.TrafficEncryptionModeWireGuard { if err := i.initializeWireGuard(); err != nil { @@ -553,11 +582,17 @@ func (i *Initializer) initOpenFlowPipeline() error { i.ofClient.ReplayFlows() klog.Info("Flow replay completed") + klog.InfoS("Restoring OF port configs to OVS bridge") + if err := i.restorePortConfigs(); err != nil { + klog.ErrorS(err, "Failed to restore OF port configs") + } else { + klog.InfoS("Port configs restoration completed") + } // ofClient and ovsBridgeClient have their own mechanisms to restore connections with OVS, and it could // happen that ovsBridgeClient's connection is not ready when ofClient completes flow replay. We retry it // with a timeout that is longer time than ovsBridgeClient's maximum connecting retry interval (8 seconds) // to ensure the flag can be removed successfully. - err := wait.PollImmediate(200*time.Millisecond, 10*time.Second, func() (done bool, err error) { + err = wait.PollImmediate(200*time.Millisecond, 10*time.Second, func() (done bool, err error) { if err := i.FlowRestoreComplete(); err != nil { return false, nil } diff --git a/pkg/agent/agent_test.go b/pkg/agent/agent_test.go index dd179481ac0..3309a6c897f 100644 --- a/pkg/agent/agent_test.go +++ b/pkg/agent/agent_test.go @@ -37,6 +37,7 @@ import ( "antrea.io/antrea/pkg/agent/types" "antrea.io/antrea/pkg/ovs/ovsconfig" ovsconfigtest "antrea.io/antrea/pkg/ovs/ovsconfig/testing" + ovsctltest "antrea.io/antrea/pkg/ovs/ovsctl/testing" "antrea.io/antrea/pkg/util/env" "antrea.io/antrea/pkg/util/ip" "antrea.io/antrea/pkg/util/runtime" @@ -647,3 +648,70 @@ func mockConfigureLinkAddress(returnedErr error) func() { configureLinkAddresses = originalConfigureLinkAddresses } } + +func TestRestorePortConfigs(t *testing.T) { + tests := []struct { + name string + existingInterfaces []*interfacestore.InterfaceConfig + expectedOVSCtlCalls func(client *ovsctltest.MockOVSCtlClientMockRecorder) + expectedErr string + }{ + { + name: "success", + existingInterfaces: []*interfacestore.InterfaceConfig{ + interfacestore.NewIPSecTunnelInterface("antrea-ipsec1", + ovsconfig.GeneveTunnel, + "node1", + net.ParseIP("1.1.1.1"), + "abcdefg", + "node1", + &interfacestore.OVSPortConfig{OFPort: 11, PortUUID: "uuid1"}), + interfacestore.NewTunnelInterface(defaultTunInterfaceName, + ovsconfig.GeneveTunnel, + 0, + net.ParseIP("1.1.1.10"), + true, + &interfacestore.OVSPortConfig{OFPort: 12}), + interfacestore.NewTrafficControlInterface("antrea-tap1", + &interfacestore.OVSPortConfig{OFPort: 13, PortUUID: "uuid3"}), + interfacestore.NewTrafficControlInterface("antrea-tap2", + &interfacestore.OVSPortConfig{OFPort: -1, PortUUID: "uuid3"}), + }, + expectedOVSCtlCalls: func(client *ovsctltest.MockOVSCtlClientMockRecorder) { + client.SetPortNoFlood(11).Return(nil) + client.SetPortNoFlood(13).Return(nil) + }, + }, + { + name: "fail", + existingInterfaces: []*interfacestore.InterfaceConfig{ + interfacestore.NewTrafficControlInterface("antrea-tap1", + &interfacestore.OVSPortConfig{OFPort: 10, PortUUID: "uuid3"}), + }, + expectedOVSCtlCalls: func(client *ovsctltest.MockOVSCtlClientMockRecorder) { + client.SetPortNoFlood(10).Return(fmt.Errorf("server unavailable")) + }, + expectedErr: "failed to set no-flood for port antrea-tap1: server unavailable", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + controller := mock.NewController(t) + defer controller.Finish() + mockOVSCtlClient := ovsctltest.NewMockOVSCtlClient(controller) + ifaceStore := interfacestore.NewInterfaceStore() + initializer := &Initializer{ + ifaceStore: ifaceStore, + ovsCtlClient: mockOVSCtlClient, + } + ifaceStore.Initialize(tt.existingInterfaces) + tt.expectedOVSCtlCalls(mockOVSCtlClient.EXPECT()) + err := initializer.restorePortConfigs() + if tt.expectedErr == "" { + assert.NoError(t, err) + } else { + assert.ErrorContains(t, err, tt.expectedErr) + } + }) + } +} diff --git a/pkg/agent/controller/noderoute/node_route_controller.go b/pkg/agent/controller/noderoute/node_route_controller.go index 7c40d933613..0bfcd801bcd 100644 --- a/pkg/agent/controller/noderoute/node_route_controller.go +++ b/pkg/agent/controller/noderoute/node_route_controller.go @@ -40,9 +40,9 @@ import ( "antrea.io/antrea/pkg/agent/util" "antrea.io/antrea/pkg/agent/wireguard" "antrea.io/antrea/pkg/ovs/ovsconfig" + "antrea.io/antrea/pkg/ovs/ovsctl" utilip "antrea.io/antrea/pkg/util/ip" "antrea.io/antrea/pkg/util/k8s" - "antrea.io/antrea/pkg/util/runtime" ) const ( @@ -65,6 +65,7 @@ type Controller struct { kubeClient clientset.Interface ovsBridgeClient ovsconfig.OVSBridgeClient ofClient openflow.Client + ovsCtlClient ovsctl.OVSCtlClient routeClient route.Interface interfaceStore interfacestore.InterfaceStore networkConfig *config.NetworkConfig @@ -72,7 +73,6 @@ type Controller struct { nodeInformer coreinformers.NodeInformer nodeLister corelisters.NodeLister nodeListerSynced cache.InformerSynced - svcLister corelisters.ServiceLister queue workqueue.RateLimitingInterface // installedNodes records routes and flows installation states of Nodes. // The key is the host name of the Node, the value is the nodeRouteInfo of the Node. @@ -92,6 +92,7 @@ func NewNodeRouteController( kubeClient clientset.Interface, informerFactory informers.SharedInformerFactory, client openflow.Client, + ovsCtlClient ovsctl.OVSCtlClient, ovsBridgeClient ovsconfig.OVSBridgeClient, routeClient route.Interface, interfaceStore interfacestore.InterfaceStore, @@ -102,11 +103,11 @@ func NewNodeRouteController( ipsecCertificateManager ipseccertificate.Manager, ) *Controller { nodeInformer := informerFactory.Core().V1().Nodes() - svcLister := informerFactory.Core().V1().Services() controller := &Controller{ kubeClient: kubeClient, ovsBridgeClient: ovsBridgeClient, ofClient: client, + ovsCtlClient: ovsCtlClient, routeClient: routeClient, interfaceStore: interfaceStore, networkConfig: networkConfig, @@ -114,7 +115,6 @@ func NewNodeRouteController( nodeInformer: nodeInformer, nodeLister: nodeInformer.Lister(), nodeListerSynced: nodeInformer.Informer().HasSynced, - svcLister: svcLister.Lister(), queue: workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(minRetryDelay, maxRetryDelay), "noderoute"), installedNodes: cache.NewIndexer(nodeRouteInfoKeyFunc, cache.Indexers{nodeRouteInfoPodCIDRIndexName: nodeRouteInfoPodCIDRIndexFunc}), wireGuardClient: wireguardClient, @@ -203,27 +203,10 @@ func (c *Controller) removeStaleGatewayRoutes() error { desiredPodCIDRs = append(desiredPodCIDRs, podCIDRs...) } - // TODO: This is not the best place to keep the ClusterIP Service routes. - desiredClusterIPSvcIPs := map[string]bool{} - if c.proxyAll && runtime.IsWindowsPlatform() { - // The route for virtual IP -> antrea-gw0 should be always kept. - desiredClusterIPSvcIPs[config.VirtualServiceIPv4.String()] = true - - svcs, err := c.svcLister.List(labels.Everything()) - for _, svc := range svcs { - for _, ip := range svc.Spec.ClusterIPs { - desiredClusterIPSvcIPs[ip] = true - } - } - if err != nil { - return fmt.Errorf("error when listing ClusterIP Service IPs: %v", err) - } - } - // routeClient will remove orphaned routes whose destinations are not in desiredPodCIDRs. // If proxyAll enabled, it will also remove routes that are for Windows ClusterIP Services // which no longer exist. - if err := c.routeClient.Reconcile(desiredPodCIDRs, desiredClusterIPSvcIPs); err != nil { + if err := c.routeClient.Reconcile(desiredPodCIDRs); err != nil { return err } return nil @@ -244,7 +227,7 @@ func (c *Controller) removeStaleTunnelPorts() error { // will not include it in the set. desiredInterfaces := make(map[string]bool) // knownInterfaces is the list of interfaces currently in the local cache. - knownInterfaces := c.interfaceStore.GetInterfaceKeysByType(interfacestore.TunnelInterface) + knownInterfaces := c.interfaceStore.GetInterfaceKeysByType(interfacestore.IPSecTunnelInterface) if c.networkConfig.TrafficEncryptionMode == config.TrafficEncryptionModeIPSec { for _, node := range nodes { @@ -671,15 +654,14 @@ func (c *Controller) createIPSecTunnelPort(nodeName string, nodeIP net.IP) (int3 } c.interfaceStore.DeleteInterface(interfaceConfig) exists = false - } else { - if interfaceConfig.OFPort != 0 { - klog.V(2).InfoS("Found cached IPsec tunnel interface", "node", nodeName, "interface", interfaceConfig.InterfaceName, "port", interfaceConfig.OFPort) - return interfaceConfig.OFPort, nil - } } } + if !exists { - ovsExternalIDs := map[string]interface{}{ovsExternalIDNodeName: nodeName} + ovsExternalIDs := map[string]interface{}{ + ovsExternalIDNodeName: nodeName, + interfacestore.AntreaInterfaceTypeKey: interfacestore.AntreaIPsecTunnel, + } portUUID, err := c.ovsBridgeClient.CreateTunnelPortExt( portName, c.networkConfig.TunnelType, @@ -715,6 +697,12 @@ func (c *Controller) createIPSecTunnelPort(nodeName string, nodeIP net.IP) (int3 // Let NodeRouteController retry at errors. return 0, fmt.Errorf("failed to get of_port of IPsec tunnel port for Node %s", nodeName) } + + // Set the port with no-flood to reject ARP flood packets. + if err := c.ovsCtlClient.SetPortNoFlood(int(ofPort)); err != nil { + return 0, fmt.Errorf("failed to set port %s with no-flood config: %w", portName, err) + } + interfaceConfig.OFPort = ofPort return ofPort, nil } diff --git a/pkg/agent/controller/noderoute/node_route_controller_test.go b/pkg/agent/controller/noderoute/node_route_controller_test.go index fd70d204270..a324dd43695 100644 --- a/pkg/agent/controller/noderoute/node_route_controller_test.go +++ b/pkg/agent/controller/noderoute/node_route_controller_test.go @@ -35,6 +35,7 @@ import ( "antrea.io/antrea/pkg/agent/util" "antrea.io/antrea/pkg/ovs/ovsconfig" ovsconfigtest "antrea.io/antrea/pkg/ovs/ovsconfig/testing" + ovsctltest "antrea.io/antrea/pkg/ovs/ovsctl/testing" utilip "antrea.io/antrea/pkg/util/ip" ) @@ -58,6 +59,7 @@ type fakeController struct { ovsClient *ovsconfigtest.MockOVSBridgeClient routeClient *routetest.MockInterface interfaceStore interfacestore.InterfaceStore + ovsCtlClient *ovsctltest.MockOVSCtlClient } type fakeIPsecCertificateManager struct{} @@ -75,7 +77,9 @@ func newController(t *testing.T, networkConfig *config.NetworkConfig) (*fakeCont routeClient := routetest.NewMockInterface(ctrl) interfaceStore := interfacestore.NewInterfaceStore() ipsecCertificateManager := &fakeIPsecCertificateManager{} - c := NewNodeRouteController(clientset, informerFactory, ofClient, ovsClient, routeClient, interfaceStore, networkConfig, &config.NodeConfig{GatewayConfig: &config.GatewayConfig{ + ovsCtlClient := ovsctltest.NewMockOVSCtlClient(ctrl) + + c := NewNodeRouteController(clientset, informerFactory, ofClient, ovsCtlClient, ovsClient, routeClient, interfaceStore, networkConfig, &config.NodeConfig{GatewayConfig: &config.GatewayConfig{ IPv4: nil, MAC: gatewayMAC, }}, nil, false, ipsecCertificateManager) @@ -86,6 +90,7 @@ func newController(t *testing.T, networkConfig *config.NetworkConfig) (*fakeCont ofClient: ofClient, ovsClient: ovsClient, routeClient: routeClient, + ovsCtlClient: ovsCtlClient, interfaceStore: interfaceStore, }, ctrl.Finish } @@ -254,7 +259,7 @@ func setup(t *testing.T, ifaces []*interfacestore.InterfaceConfig, authenticatio func TestRemoveStaleTunnelPorts(t *testing.T) { c, closeFn := setup(t, []*interfacestore.InterfaceConfig{ { - Type: interfacestore.TunnelInterface, + Type: interfacestore.IPSecTunnelInterface, InterfaceName: util.GenerateNodeTunnelInterfaceName("xyz-k8s-0-1"), TunnelInterfaceConfig: &interfacestore.TunnelInterfaceConfig{ NodeName: "xyz-k8s-0-1", @@ -302,7 +307,7 @@ func TestRemoveStaleTunnelPorts(t *testing.T) { func TestCreateIPSecTunnelPortPSK(t *testing.T) { c, closeFn := setup(t, []*interfacestore.InterfaceConfig{ { - Type: interfacestore.TunnelInterface, + Type: interfacestore.IPSecTunnelInterface, InterfaceName: "mismatchedname", TunnelInterfaceConfig: &interfacestore.TunnelInterfaceConfig{ NodeName: "xyz-k8s-0-2", @@ -315,7 +320,7 @@ func TestCreateIPSecTunnelPortPSK(t *testing.T) { }, }, { - Type: interfacestore.TunnelInterface, + Type: interfacestore.IPSecTunnelInterface, InterfaceName: util.GenerateNodeTunnelInterfaceName("xyz-k8s-0-3"), TunnelInterfaceConfig: &interfacestore.TunnelInterfaceConfig{ NodeName: "xyz-k8s-0-3", @@ -339,16 +344,25 @@ func TestCreateIPSecTunnelPortPSK(t *testing.T) { node1PortName := util.GenerateNodeTunnelInterfaceName("xyz-k8s-0-1") node2PortName := util.GenerateNodeTunnelInterfaceName("xyz-k8s-0-2") + node3PortName := util.GenerateNodeTunnelInterfaceName("xyz-k8s-0-3") c.ovsClient.EXPECT().CreateTunnelPortExt( node1PortName, ovsconfig.TunnelType("vxlan"), int32(0), false, "", nodeIP1.String(), "", "changeme", nil, - map[string]interface{}{ovsExternalIDNodeName: "xyz-k8s-0-1"}).Times(1) + map[string]interface{}{ovsExternalIDNodeName: "xyz-k8s-0-1", + interfacestore.AntreaInterfaceTypeKey: interfacestore.AntreaIPsecTunnel, + }).Times(1) c.ovsClient.EXPECT().CreateTunnelPortExt( node2PortName, ovsconfig.TunnelType("vxlan"), int32(0), false, "", nodeIP2.String(), "", "changeme", nil, - map[string]interface{}{ovsExternalIDNodeName: "xyz-k8s-0-2"}).Times(1) + map[string]interface{}{ovsExternalIDNodeName: "xyz-k8s-0-2", + interfacestore.AntreaInterfaceTypeKey: interfacestore.AntreaIPsecTunnel, + }).Times(1) c.ovsClient.EXPECT().GetOFPort(node1PortName, false).Return(int32(1), nil) + c.ovsCtlClient.EXPECT().SetPortNoFlood(1) c.ovsClient.EXPECT().GetOFPort(node2PortName, false).Return(int32(2), nil) + c.ovsCtlClient.EXPECT().SetPortNoFlood(2) + c.ovsClient.EXPECT().GetOFPort(node3PortName, false).Return(int32(5), nil) + c.ovsCtlClient.EXPECT().SetPortNoFlood(5) c.ovsClient.EXPECT().DeletePort("123").Times(1) tests := []struct { @@ -405,8 +419,11 @@ func TestCreateIPSecTunnelPortCert(t *testing.T) { c.ovsClient.EXPECT().CreateTunnelPortExt( node1PortName, ovsconfig.TunnelType("vxlan"), int32(0), false, "", nodeIP1.String(), "xyz-k8s-0-1", "", nil, - map[string]interface{}{ovsExternalIDNodeName: "xyz-k8s-0-1"}).Times(1) + map[string]interface{}{ovsExternalIDNodeName: "xyz-k8s-0-1", + interfacestore.AntreaInterfaceTypeKey: interfacestore.AntreaIPsecTunnel, + }).Times(1) c.ovsClient.EXPECT().GetOFPort(node1PortName, false).Return(int32(1), nil) + c.ovsCtlClient.EXPECT().SetPortNoFlood(1) tests := []struct { name string diff --git a/pkg/agent/controller/trafficcontrol/controller.go b/pkg/agent/controller/trafficcontrol/controller.go index 3ed0675184e..3c37909fb23 100644 --- a/pkg/agent/controller/trafficcontrol/controller.go +++ b/pkg/agent/controller/trafficcontrol/controller.go @@ -762,8 +762,7 @@ func (c *Controller) getOrCreateTrafficControlPort(port *v1alpha2.TrafficControl return 0, err } } - itf := interfacestore.NewTrafficControlInterface(portName) - itf.OVSPortConfig = &interfacestore.OVSPortConfig{PortUUID: portUUID, OFPort: ofPort} + itf := interfacestore.NewTrafficControlInterface(portName, &interfacestore.OVSPortConfig{PortUUID: portUUID, OFPort: ofPort}) c.interfaceStore.AddInterface(itf) // Create binding for the newly created port. c.portToTCBindings[portName] = &portToTCBinding{ diff --git a/pkg/agent/interfacestore/interface_cache.go b/pkg/agent/interfacestore/interface_cache.go index 1d35378ae1c..479593452ab 100644 --- a/pkg/agent/interfacestore/interface_cache.go +++ b/pkg/agent/interfacestore/interface_cache.go @@ -86,8 +86,8 @@ func getInterfaceKey(obj interface{}) (string, error) { var key string if interfaceConfig.Type == ContainerInterface { key = util.GenerateContainerInterfaceKey(interfaceConfig.ContainerID) - } else if interfaceConfig.Type == TunnelInterface && interfaceConfig.NodeName != "" { - // Tunnel interface for a Node. + } else if interfaceConfig.Type == IPSecTunnelInterface { + // IPsec tunnel interface for a Node. key = util.GenerateNodeTunnelInterfaceKey(interfaceConfig.NodeName) } else { // Use the interface name as the key by default. @@ -123,6 +123,15 @@ func (c *interfaceCache) GetInterface(interfaceKey string) (*InterfaceConfig, bo return iface.(*InterfaceConfig), found } +// ListInterfacesByType lists all interfaces from local cache. +func (c *interfaceCache) ListInterfaces() []*InterfaceConfig { + interfaceConfigs := make([]*InterfaceConfig, 0) + for _, iface := range c.cache.List() { + interfaceConfigs = append(interfaceConfigs, iface.(*InterfaceConfig)) + } + return interfaceConfigs +} + // GetInterfaceByName retrieves interface from local cache given the interface // name. func (c *interfaceCache) GetInterfaceByName(interfaceName string) (*InterfaceConfig, bool) { diff --git a/pkg/agent/interfacestore/interface_cache_test.go b/pkg/agent/interfacestore/interface_cache_test.go index de41c9a597e..c338fb2fe09 100644 --- a/pkg/agent/interfacestore/interface_cache_test.go +++ b/pkg/agent/interfacestore/interface_cache_test.go @@ -40,7 +40,6 @@ func TestNewInterfaceStore(t *testing.T) { t.Run("testGatewayInterface", testGatewayInterface) t.Run("testTunnelInterface", testTunnelInterface) t.Run("testUplinkInterface", testUplinkInterface) - t.Run("testTrafficControlInterface", testTrafficControlInterface) t.Run("testExternalEntityInterface", testEntityInterface) } @@ -140,14 +139,16 @@ func testTunnelInterface(t *testing.T) { assert.False(t, exists) ifaceNames := store.GetInterfaceKeysByType(TunnelInterface) - assert.Equal(t, 2, len(ifaceNames)) + assert.Equal(t, 1, len(ifaceNames)) + ipsecIfaceNames := store.GetInterfaceKeysByType(IPSecTunnelInterface) + assert.Equal(t, 1, len(ipsecIfaceNames)) store.DeleteInterface(ipsecTunnelInterface) - assert.Equal(t, 1, len(store.GetInterfaceKeysByType(TunnelInterface))) + assert.Equal(t, 0, len(store.GetInterfaceKeysByType(IPSecTunnelInterface))) _, exists = store.GetInterfaceByName(ipsecTunnelInterface.InterfaceName) assert.False(t, exists) store.AddInterface(ipsecTunnelInterface) - ifaceNames = store.GetInterfaceKeysByType(TunnelInterface) - assert.Equal(t, 2, len(ifaceNames)) + ifaceNames = store.GetInterfaceKeysByType(IPSecTunnelInterface) + assert.Equal(t, 1, len(ifaceNames)) _, exists = store.GetInterfaceByName(ipsecTunnelInterface.InterfaceName) assert.True(t, exists) } @@ -162,16 +163,6 @@ func testUplinkInterface(t *testing.T) { testGeneralInterface(t, uplinkInterface, UplinkInterface) } -func testTrafficControlInterface(t *testing.T) { - tcInterface := NewTrafficControlInterface("tc0") - tcInterface.IPs = []net.IP{hostIP} - tcInterface.OVSPortConfig = &OVSPortConfig{ - OFPort: 17, - PortUUID: "1234567890", - } - testGeneralInterface(t, tcInterface, TrafficControlInterface) -} - func testEntityInterface(t *testing.T) { store := NewInterfaceStore() portConfig := &OVSPortConfig{OFPort: 18, PortUUID: "123456789"} diff --git a/pkg/agent/interfacestore/testing/mock_interfacestore.go b/pkg/agent/interfacestore/testing/mock_interfacestore.go index 3e8aa50fbc5..70519cf5256 100644 --- a/pkg/agent/interfacestore/testing/mock_interfacestore.go +++ b/pkg/agent/interfacestore/testing/mock_interfacestore.go @@ -257,3 +257,17 @@ func (mr *MockInterfaceStoreMockRecorder) Len() *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Len", reflect.TypeOf((*MockInterfaceStore)(nil).Len)) } + +// ListInterfaces mocks base method +func (m *MockInterfaceStore) ListInterfaces() []*interfacestore.InterfaceConfig { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListInterfaces") + ret0, _ := ret[0].([]*interfacestore.InterfaceConfig) + return ret0 +} + +// ListInterfaces indicates an expected call of ListInterfaces +func (mr *MockInterfaceStoreMockRecorder) ListInterfaces() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListInterfaces", reflect.TypeOf((*MockInterfaceStore)(nil).ListInterfaces)) +} diff --git a/pkg/agent/interfacestore/types.go b/pkg/agent/interfacestore/types.go index b0e03a31b35..90715c1a2bc 100644 --- a/pkg/agent/interfacestore/types.go +++ b/pkg/agent/interfacestore/types.go @@ -35,6 +35,8 @@ const ( TrafficControlInterface // ExternalEntityInterface is used to mark current interface is for ExternalEntity Endpoint ExternalEntityInterface + // IPSecTunnelInterface is used to mark current interface is for IPSec tunnel port + IPSecTunnelInterface AntreaInterfaceTypeKey = "antrea-type" AntreaGateway = "gateway" @@ -43,6 +45,7 @@ const ( AntreaUplink = "uplink" AntreaHost = "host" AntreaTrafficControl = "traffic-control" + AntreaIPsecTunnel = "ipsec-tunnel" AntreaUnset = "" ) @@ -108,6 +111,7 @@ type InterfaceConfig struct { type InterfaceStore interface { Initialize(interfaces []*InterfaceConfig) AddInterface(interfaceConfig *InterfaceConfig) + ListInterfaces() []*InterfaceConfig DeleteInterface(interfaceConfig *InterfaceConfig) GetInterface(interfaceKey string) (*InterfaceConfig, bool) GetInterfaceByName(interfaceName string) (*InterfaceConfig, bool) @@ -162,7 +166,7 @@ func NewTunnelInterface(tunnelName string, tunnelType ovsconfig.TunnelType, dest // Node. func NewIPSecTunnelInterface(interfaceName string, tunnelType ovsconfig.TunnelType, nodeName string, nodeIP net.IP, psk, remoteName string, ovsPortConfig *OVSPortConfig) *InterfaceConfig { tunnelConfig := &TunnelInterfaceConfig{Type: tunnelType, NodeName: nodeName, RemoteIP: nodeIP, PSK: psk, RemoteName: remoteName} - return &InterfaceConfig{InterfaceName: interfaceName, Type: TunnelInterface, TunnelInterfaceConfig: tunnelConfig, OVSPortConfig: ovsPortConfig} + return &InterfaceConfig{InterfaceName: interfaceName, Type: IPSecTunnelInterface, TunnelInterfaceConfig: tunnelConfig, OVSPortConfig: ovsPortConfig} } // NewUplinkInterface creates InterfaceConfig for the uplink interface. @@ -171,8 +175,8 @@ func NewUplinkInterface(uplinkName string) *InterfaceConfig { return uplinkConfig } -func NewTrafficControlInterface(interfaceName string) *InterfaceConfig { - trafficControlConfig := &InterfaceConfig{InterfaceName: interfaceName, Type: TrafficControlInterface} +func NewTrafficControlInterface(interfaceName string, ovsPortConfig *OVSPortConfig) *InterfaceConfig { + trafficControlConfig := &InterfaceConfig{InterfaceName: interfaceName, Type: TrafficControlInterface, OVSPortConfig: ovsPortConfig} return trafficControlConfig } diff --git a/pkg/agent/proxy/proxier.go b/pkg/agent/proxy/proxier.go index 6be85805658..80edc3eb3ab 100644 --- a/pkg/agent/proxy/proxier.go +++ b/pkg/agent/proxy/proxier.go @@ -27,6 +27,7 @@ import ( discovery "k8s.io/api/discovery/v1" "k8s.io/apimachinery/pkg/runtime" apimachinerytypes "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/informers" "k8s.io/client-go/tools/record" "k8s.io/klog/v2" @@ -104,6 +105,13 @@ type proxier struct { serviceHealthServer healthcheck.ServiceHealthServer numLocalEndpoints map[apimachinerytypes.NamespacedName]int + // serviceIPRouteReferences tracks the references of Service IP routes. The key is the Service IP and the value is + // the set of ServiceInfo strings. Because a Service could have multiple ports and each port will generate a + // ServicePort (which is the unit of the processing), a Service IP route may be required by several ServicePorts. + // With the references, we install a route exactly once as long as it's used by any ServicePorts and uninstall it + // exactly once when it's no longer used by any ServicePorts. + // It applies to ClusterIP and LoadBalancerIP. + serviceIPRouteReferences map[string]sets.String // syncedOnce returns true if the proxier has synced rules at least once. syncedOnce bool syncedOnceMutex sync.RWMutex @@ -113,7 +121,6 @@ type proxier struct { ofClient openflow.Client routeClient route.Interface nodePortAddresses []net.IP - hostGateWay string hostname string isIPv6 bool proxyAll bool @@ -146,25 +153,31 @@ func (p *proxier) removeStaleServices() { continue } svcInfo := svcPort.(*types.ServiceInfo) + svcInfoStr := svcInfo.String() klog.V(2).Infof("Removing stale Service: %s %s", svcPortName.Name, svcInfo.String()) if err := p.ofClient.UninstallServiceFlows(svcInfo.ClusterIP(), uint16(svcInfo.Port()), svcInfo.OFProtocol); err != nil { klog.ErrorS(err, "Failed to remove flows of Service", "Service", svcPortName) continue } - + // Remove NodePort and ClusterIP flows and configurations. if p.proxyAll { - // Remove NodePort flows and configurations. if svcInfo.NodePort() > 0 { if err := p.uninstallNodePortService(uint16(svcInfo.NodePort()), svcInfo.OFProtocol); err != nil { - klog.ErrorS(err, "Failed to remove flows and configurations of Service", "Service", svcPortName) + klog.ErrorS(err, "Error when uninstalling NodePort flows and configurations for Service", "ServicePortName", svcPortName, "ServiceInfo", svcInfoStr) + continue + } + } + if svcInfo.ClusterIP() != nil { + if err := p.deleteRouteForServiceIP(svcInfoStr, svcInfo.ClusterIP(), p.routeClient.DeleteClusterIPRoute); err != nil { + klog.ErrorS(err, "Failed to remove ClusterIP Service routes", "Service", svcPortName) continue } } } // Remove LoadBalancer flows and configurations. if p.proxyLoadBalancerIPs && len(svcInfo.LoadBalancerIPStrings()) > 0 { - if err := p.uninstallLoadBalancerService(svcInfo.LoadBalancerIPStrings(), uint16(svcInfo.Port()), svcInfo.OFProtocol); err != nil { - klog.ErrorS(err, "Failed to remove flows and configurations of Service", "Service", svcPortName) + if err := p.uninstallLoadBalancerService(svcInfoStr, svcInfo.LoadBalancerIPStrings(), uint16(svcInfo.Port()), svcInfo.OFProtocol); err != nil { + klog.ErrorS(err, "Error when uninstalling LoadBalancer flows and configurations for Service", "ServicePortName", svcPortName, "ServiceInfo", svcInfoStr) continue } } @@ -312,43 +325,80 @@ func (p *proxier) uninstallNodePortService(svcPort uint16, protocol binding.Prot return nil } -func (p *proxier) installLoadBalancerService(groupID binding.GroupIDType, loadBalancerIPStrings []string, svcPort uint16, protocol binding.Protocol, affinityTimeout uint16, nodeLocalExternal bool) error { +func (p *proxier) installLoadBalancerService(svcInfoStr string, groupID binding.GroupIDType, loadBalancerIPStrings []string, svcPort uint16, protocol binding.Protocol, affinityTimeout uint16, nodeLocalExternal bool) error { for _, ingress := range loadBalancerIPStrings { if ingress != "" { - if err := p.ofClient.InstallServiceFlows(groupID, net.ParseIP(ingress), svcPort, protocol, affinityTimeout, nodeLocalExternal, corev1.ServiceTypeLoadBalancer); err != nil { - return fmt.Errorf("failed to install Service LoadBalancer load balancing flows: %w", err) + ip := net.ParseIP(ingress) + if err := p.ofClient.InstallServiceFlows(groupID, ip, svcPort, protocol, affinityTimeout, nodeLocalExternal, corev1.ServiceTypeLoadBalancer); err != nil { + return fmt.Errorf("failed to install LoadBalancer load balancing flows: %w", err) + } + if p.proxyAll { + if err := p.addRouteForServiceIP(svcInfoStr, ip, p.routeClient.AddLoadBalancer); err != nil { + return fmt.Errorf("failed to install LoadBalancer traffic redirecting routes: %w", err) + } } } } - if p.proxyAll { - if err := p.routeClient.AddLoadBalancer(loadBalancerIPStrings); err != nil { - return fmt.Errorf("failed to install Service LoadBalancer traffic redirecting flows: %w", err) + return nil +} + +func (p *proxier) addRouteForServiceIP(svcInfoStr string, ip net.IP, addRouteFn func(net.IP) error) error { + ipStr := ip.String() + references, exists := p.serviceIPRouteReferences[ipStr] + // If the IP was not referenced by any Service port, install a route for it. + // Otherwise, just reference it. + if !exists { + if err := addRouteFn(ip); err != nil { + return err } + references = sets.NewString(svcInfoStr) + p.serviceIPRouteReferences[ipStr] = references + } else { + references.Insert(svcInfoStr) } - return nil } -func (p *proxier) uninstallLoadBalancerService(loadBalancerIPStrings []string, svcPort uint16, protocol binding.Protocol) error { +func (p *proxier) uninstallLoadBalancerService(svcInfoStr string, loadBalancerIPStrings []string, svcPort uint16, protocol binding.Protocol) error { for _, ingress := range loadBalancerIPStrings { if ingress != "" { - if err := p.ofClient.UninstallServiceFlows(net.ParseIP(ingress), svcPort, protocol); err != nil { - return fmt.Errorf("failed to remove Service LoadBalancer load balancing flows: %w", err) + ip := net.ParseIP(ingress) + if err := p.ofClient.UninstallServiceFlows(ip, svcPort, protocol); err != nil { + return fmt.Errorf("failed to remove LoadBalancer load balancing flows: %w", err) + } + if p.proxyAll { + if err := p.deleteRouteForServiceIP(svcInfoStr, ip, p.routeClient.DeleteLoadBalancer); err != nil { + return fmt.Errorf("failed to remove LoadBalancer traffic redirecting routes: %w", err) + } } } } - if p.proxyAll { - if err := p.routeClient.DeleteLoadBalancer(loadBalancerIPStrings); err != nil { - return fmt.Errorf("failed to remove Service LoadBalancer traffic redirecting flows: %w", err) + return nil +} + +func (p *proxier) deleteRouteForServiceIP(svcInfoStr string, ip net.IP, deleteRouteFn func(net.IP) error) error { + ipStr := ip.String() + references, exists := p.serviceIPRouteReferences[ipStr] + // If the IP was not referenced by this Service port, skip it. + if exists && references.Has(svcInfoStr) { + // Delete the IP only if this Service port is the last one referencing it. + // Otherwise, just dereference it. + if references.Len() == 1 { + if err := deleteRouteFn(ip); err != nil { + return err + } + delete(p.serviceIPRouteReferences, ipStr) + } else { + references.Delete(svcInfoStr) } } - return nil } func (p *proxier) installServices() { for svcPortName, svcPort := range p.serviceMap { svcInfo := svcPort.(*types.ServiceInfo) + svcInfoStr := svcInfo.String() endpointsInstalled, ok := p.endpointsInstalledMap[svcPortName] if !ok { endpointsInstalled = map[string]k8sproxy.Endpoint{} @@ -544,9 +594,9 @@ func (p *proxier) installServices() { } } // If previous Service which has ClusterIP should be removed, remove ClusterIP routes. - if svcInfo.ClusterIP() != nil { - if err := p.routeClient.DeleteClusterIPRoute(pSvcInfo.ClusterIP()); err != nil { - klog.ErrorS(err, "Failed to remove ClusterIP Service routes", "Service", svcPortName) + if pSvcInfo.ClusterIP() != nil { + if err := p.deleteRouteForServiceIP(pSvcInfo.String(), pSvcInfo.ClusterIP(), p.routeClient.DeleteClusterIPRoute); err != nil { + klog.ErrorS(err, "Error when uninstalling ClusterIP route for Service", "ServicePortName", svcPortName, "ServiceInfo", svcInfoStr) continue } } @@ -566,8 +616,8 @@ func (p *proxier) installServices() { // is created, the routing target IP block will be recalculated for expansion to be able to route the new // created ClusterIP. Deleting a ClusterIP will not shrink the target routing IP block. The Service CIDR // can be finally calculated after creating enough ClusterIPs. - if err := p.routeClient.AddClusterIPRoute(svcInfo.ClusterIP()); err != nil { - klog.ErrorS(err, "Failed to install ClusterIP route of Service", "Service", svcPortName) + if err := p.addRouteForServiceIP(svcInfo.String(), svcInfo.ClusterIP(), p.routeClient.AddClusterIPRoute); err != nil { + klog.ErrorS(err, "Error when installing ClusterIP route for Service", "ServicePortName", svcPortName, "ServiceInfo", svcInfoStr) continue } @@ -594,15 +644,15 @@ func (p *proxier) installServices() { } // Remove LoadBalancer flows and configurations. if len(toDelete) > 0 { - if err := p.uninstallLoadBalancerService(toDelete, uint16(pSvcInfo.Port()), pSvcInfo.OFProtocol); err != nil { - klog.ErrorS(err, "Failed to remove flows and configurations of Service", "Service", svcPortName) + if err := p.uninstallLoadBalancerService(pSvcInfo.String(), toDelete, uint16(pSvcInfo.Port()), pSvcInfo.OFProtocol); err != nil { + klog.ErrorS(err, "Error when uninstalling LoadBalancer flows and configurations for Service", "ServicePortName", svcPortName, "ServiceInfo", svcInfoStr) continue } } // Install LoadBalancer flows and configurations. if len(toAdd) > 0 { - if err := p.installLoadBalancerService(nGroupID, toAdd, uint16(svcInfo.Port()), svcInfo.OFProtocol, uint16(affinityTimeout), svcInfo.NodeLocalExternal()); err != nil { - klog.ErrorS(err, "Failed to install LoadBalancer flows and configurations of Service", "Service", svcPortName) + if err := p.installLoadBalancerService(svcInfo.String(), nGroupID, toAdd, uint16(svcInfo.Port()), svcInfo.OFProtocol, uint16(affinityTimeout), svcInfo.NodeLocalExternal()); err != nil { + klog.ErrorS(err, "Error when installing LoadBalancer flows and configurations for Service", "Service", "ServicePortName", svcPortName, "ServiceInfo", svcInfoStr) continue } } @@ -948,6 +998,7 @@ func NewProxier( endpointsInstalledMap: types.EndpointsMap{}, endpointsMap: types.EndpointsMap{}, endpointReferenceCounter: map[string]int{}, + serviceIPRouteReferences: map[string]sets.String{}, nodeLabels: map[string]string{}, serviceStringMap: map[string]k8sproxy.ServicePortName{}, groupCounter: groupCounter, diff --git a/pkg/agent/proxy/proxier_test.go b/pkg/agent/proxy/proxier_test.go index a4fa058144a..f601b0dc10f 100644 --- a/pkg/agent/proxy/proxier_test.go +++ b/pkg/agent/proxy/proxier_test.go @@ -15,22 +15,27 @@ package proxy import ( - "fmt" "math" "net" + "strconv" + "strings" "testing" "time" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" + discovery "k8s.io/api/discovery/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" apimachinerytypes "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/client-go/tools/record" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/component-base/metrics/legacyregistry" "k8s.io/component-base/metrics/testutil" + "k8s.io/utils/pointer" + agentconfig "antrea.io/antrea/pkg/agent/config" "antrea.io/antrea/pkg/agent/openflow" ofmock "antrea.io/antrea/pkg/agent/openflow/testing" "antrea.io/antrea/pkg/agent/proxy/metrics" @@ -42,24 +47,37 @@ import ( ) var ( - svcIPv4 = net.ParseIP("10.20.30.41") - svcIPv6 = net.ParseIP("2001::10:20:30:41") - ep1IPv4 = net.ParseIP("10.180.0.1") - ep1IPv6 = net.ParseIP("2001::10:180:0:1") - ep2IPv4 = net.ParseIP("10.180.0.2") - ep2IPv6 = net.ParseIP("2001::10:180:0:2") - loadBalancerIPv4 = net.ParseIP("169.254.169.1") - loadBalancerIPv6 = net.ParseIP("fec0::169:254:169:1") - svcNodePortIPv4 = net.ParseIP("192.168.77.100") - svcNodePortIPv6 = net.ParseIP("2001::192:168:77:100") - hostname = "localhost" - + svc1IPv4 = net.ParseIP("10.20.30.41") + svc2IPv4 = net.ParseIP("10.20.30.42") + svc1IPv6 = net.ParseIP("2001::10:20:30:41") + svc2IPv6 = net.ParseIP("2001::10:20:30:42") + ep1IPv4 = net.ParseIP("10.180.0.1") + ep1IPv6 = net.ParseIP("2001::10:180:0:1") + ep2IPv4 = net.ParseIP("10.180.0.2") + ep2IPv6 = net.ParseIP("2001::10:180:0:2") + loadBalancerIPv4 = net.ParseIP("169.254.169.1") + loadBalancerIPv6 = net.ParseIP("fec0::169:254:169:1") + svcNodePortIPv4 = net.ParseIP("192.168.77.100") + svcNodePortIPv6 = net.ParseIP("2001::192:168:77:100") nodePortAddressesIPv4 = []net.IP{svcNodePortIPv4} nodePortAddressesIPv6 = []net.IP{svcNodePortIPv6} + + svcPort = 80 + svcNodePort = 30008 + svcPortName = makeSvcPortName("ns", "svc", strconv.Itoa(svcPort), corev1.ProtocolTCP) + + hostname = "localhostName" + + skippedServiceNN = "kube-system/kube-dns" + skippedClusterIP = "192.168.1.2" ) -func makeNamespaceName(namespace, name string) apimachinerytypes.NamespacedName { - return apimachinerytypes.NamespacedName{Namespace: namespace, Name: name} +func makeSvcPortName(namespace, name, port string, protocol corev1.Protocol) k8sproxy.ServicePortName { + return k8sproxy.ServicePortName{ + NamespacedName: apimachinerytypes.NamespacedName{Namespace: namespace, Name: name}, + Port: port, + Protocol: protocol, + } } func makeServiceMap(proxier *proxier, allServices ...*corev1.Service) { @@ -90,20 +108,190 @@ func makeEndpointsMap(proxier *proxier, allEndpoints ...*corev1.Endpoints) { proxier.endpointsChanges.OnEndpointsSynced() } -func makeTestEndpoints(namespace, name string, eptFunc func(*corev1.Endpoints)) *corev1.Endpoints { - ept := &corev1.Endpoints{ +func makeEndpointSliceMap(proxier *proxier, allEndpoints ...*discovery.EndpointSlice) { + for i := range allEndpoints { + proxier.endpointsChanges.OnEndpointSliceUpdate(allEndpoints[i], false) + } + proxier.endpointsChanges.OnEndpointsSynced() +} + +func makeTestEndpointSlice(namespace, name string, eps []discovery.Endpoint, ports []discovery.EndpointPort, isIPv6 bool) *discovery.EndpointSlice { + addrType := discovery.AddressTypeIPv4 + if isIPv6 { + addrType = discovery.AddressTypeIPv6 + } + endpointSlice := &discovery.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: namespace, + Labels: map[string]string{ + discovery.LabelServiceName: name, + }, + }, + } + endpointSlice.Endpoints = eps + endpointSlice.Ports = ports + endpointSlice.AddressType = addrType + return endpointSlice +} + +func makeTestClusterIPService(svcPortName *k8sproxy.ServicePortName, + clusterIP net.IP, + svcPort int32, + protocol corev1.Protocol, + affinitySeconds *int32, + internalTrafficPolicy *corev1.ServiceInternalTrafficPolicyType) *corev1.Service { + return makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *corev1.Service) { + svc.Spec.ClusterIP = clusterIP.String() + svc.Spec.Ports = []corev1.ServicePort{{ + Name: svcPortName.Port, + Port: svcPort, + Protocol: protocol, + }} + if internalTrafficPolicy != nil { + svc.Spec.InternalTrafficPolicy = internalTrafficPolicy + } + if affinitySeconds != nil { + svc.Spec.SessionAffinity = corev1.ServiceAffinityClientIP + svc.Spec.SessionAffinityConfig = &corev1.SessionAffinityConfig{ + ClientIP: &corev1.ClientIPConfig{ + TimeoutSeconds: affinitySeconds, + }, + } + } + }) +} + +func makeTestNodePortService(svcPortName *k8sproxy.ServicePortName, + clusterIP net.IP, + svcPort, + svcNodePort int32, + protocol corev1.Protocol, + affinitySeconds *int32, + internalTrafficPolicy corev1.ServiceInternalTrafficPolicyType, + externalTrafficPolicy corev1.ServiceExternalTrafficPolicyType) *corev1.Service { + return makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *corev1.Service) { + svc.Spec.ClusterIP = clusterIP.String() + svc.Spec.Type = corev1.ServiceTypeNodePort + svc.Spec.Ports = []corev1.ServicePort{{ + NodePort: svcNodePort, + Name: svcPortName.Port, + Port: svcPort, + Protocol: protocol, + }} + svc.Spec.ExternalTrafficPolicy = externalTrafficPolicy + svc.Spec.InternalTrafficPolicy = &internalTrafficPolicy + if affinitySeconds != nil { + svc.Spec.SessionAffinity = corev1.ServiceAffinityClientIP + svc.Spec.SessionAffinityConfig = &corev1.SessionAffinityConfig{ + ClientIP: &corev1.ClientIPConfig{ + TimeoutSeconds: affinitySeconds, + }, + } + } + }) +} + +func makeTestLoadBalancerService(svcPortName *k8sproxy.ServicePortName, + clusterIP net.IP, + loadBalancerIPs []net.IP, + svcPort, + svcNodePort int32, + protocol corev1.Protocol, + affinitySeconds *int32, + internalTrafficPolicy *corev1.ServiceInternalTrafficPolicyType, + externalTrafficPolicy corev1.ServiceExternalTrafficPolicyType) *corev1.Service { + return makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *corev1.Service) { + svc.Spec.ClusterIP = clusterIP.String() + svc.Spec.Type = corev1.ServiceTypeLoadBalancer + var ingress []corev1.LoadBalancerIngress + for _, ip := range loadBalancerIPs { + ingress = append(ingress, corev1.LoadBalancerIngress{IP: ip.String()}) + } + svc.Status.LoadBalancer.Ingress = ingress + svc.Spec.Ports = []corev1.ServicePort{{ + NodePort: svcNodePort, + Name: svcPortName.Port, + Port: svcPort, + Protocol: protocol, + }} + svc.Spec.ExternalTrafficPolicy = externalTrafficPolicy + if internalTrafficPolicy != nil { + svc.Spec.InternalTrafficPolicy = internalTrafficPolicy + } + if affinitySeconds != nil { + svc.Spec.SessionAffinity = corev1.ServiceAffinityClientIP + svc.Spec.SessionAffinityConfig = &corev1.SessionAffinityConfig{ + ClientIP: &corev1.ClientIPConfig{ + TimeoutSeconds: affinitySeconds, + }, + } + } + }) +} + +func makeTestEndpointSubset(svcPortName *k8sproxy.ServicePortName, + epIP net.IP, + port int32, + protocol corev1.Protocol, + isLocal bool) *corev1.EndpointSubset { + var nodeName *string + if isLocal { + nodeName = &hostname + } + return &corev1.EndpointSubset{ + Addresses: []corev1.EndpointAddress{{ + IP: epIP.String(), + NodeName: nodeName, + }}, + Ports: []corev1.EndpointPort{{ + Name: svcPortName.Port, + Port: port, + Protocol: protocol, + }}, + } +} + +func makeTestEndpointSliceEndpointAndPort(svcPortName *k8sproxy.ServicePortName, + epIP net.IP, + port int32, + protocol corev1.Protocol, + isLocal bool) (*discovery.Endpoint, *discovery.EndpointPort) { + ready := true + var nodeName *string + if isLocal { + nodeName = &hostname + } + return &discovery.Endpoint{ + Addresses: []string{ + epIP.String(), + }, + Conditions: discovery.EndpointConditions{ + Ready: &ready, + }, + Hostname: nodeName, + NodeName: nodeName, + }, &discovery.EndpointPort{ + Name: &svcPortName.Port, + Port: &port, + Protocol: &protocol, + } +} + +func makeTestEndpoints(svcPortName *k8sproxy.ServicePortName, epSubsets []corev1.EndpointSubset) *corev1.Endpoints { + return &corev1.Endpoints{ + ObjectMeta: metav1.ObjectMeta{ + Name: svcPortName.Name, + Namespace: svcPortName.Namespace, }, + Subsets: epSubsets, } - eptFunc(ept) - return ept } type proxyOptions struct { proxyAllEnabled bool proxyLoadBalancerIPs bool + endpointSliceEnabled bool } type proxyOptionsFn func(*proxyOptions) @@ -116,110 +304,95 @@ func withoutProxyLoadBalancerIPs(o *proxyOptions) { o.proxyLoadBalancerIPs = false } -func NewFakeProxier(routeClient route.Interface, ofClient openflow.Client, nodePortAddresses []net.IP, groupIDAllocator openflow.GroupAllocator, isIPv6 bool, options ...proxyOptionsFn) *proxier { - hostname := "localhost" - eventBroadcaster := record.NewBroadcaster() - recorder := eventBroadcaster.NewRecorder( - runtime.NewScheme(), - corev1.EventSource{Component: componentName, Host: hostname}, - ) - - ipFamily := corev1.IPv4Protocol - if isIPv6 { - ipFamily = corev1.IPv6Protocol - } +func withEndpointSlice(o *proxyOptions) { + o.endpointSliceEnabled = true +} + +func getMockClients(ctrl *gomock.Controller) (*ofmock.MockClient, *routemock.MockInterface) { + mockOFClient := ofmock.NewMockClient(ctrl) + mockRouteClient := routemock.NewMockInterface(ctrl) + return mockOFClient, mockRouteClient +} +func NewFakeProxier(routeClient route.Interface, ofClient openflow.Client, nodePortAddresses []net.IP, groupIDAllocator openflow.GroupAllocator, isIPv6 bool, options ...proxyOptionsFn) *proxier { o := &proxyOptions{ proxyAllEnabled: false, proxyLoadBalancerIPs: true, + endpointSliceEnabled: false, } for _, fn := range options { fn(o) } - p := &proxier{ - endpointsChanges: newEndpointsChangesTracker(hostname, false, isIPv6), - serviceChanges: newServiceChangesTracker(recorder, ipFamily, []string{"kube-system/kube-dns", "192.168.1.2"}), - serviceMap: k8sproxy.ServiceMap{}, - serviceInstalledMap: k8sproxy.ServiceMap{}, - endpointsInstalledMap: types.EndpointsMap{}, - endpointReferenceCounter: map[string]int{}, - endpointsMap: types.EndpointsMap{}, - groupCounter: types.NewGroupCounter(groupIDAllocator, make(chan string, 100)), - ofClient: ofClient, - routeClient: routeClient, - serviceStringMap: map[string]k8sproxy.ServicePortName{}, - isIPv6: isIPv6, - nodePortAddresses: nodePortAddresses, - proxyAll: o.proxyAllEnabled, - proxyLoadBalancerIPs: o.proxyLoadBalancerIPs, - numLocalEndpoints: map[apimachinerytypes.NamespacedName]int{}, - } + p := NewProxier(hostname, + informers.NewSharedInformerFactory(fake.NewSimpleClientset(), 0), + ofClient, + isIPv6, + routeClient, + nodePortAddresses, + o.proxyAllEnabled, + []string{skippedServiceNN, skippedClusterIP}, + o.proxyLoadBalancerIPs, + types.NewGroupCounter(groupIDAllocator, make(chan string, 100))) p.runner = k8sproxy.NewBoundedFrequencyRunner(componentName, p.syncProxyRules, time.Second, 30*time.Second, 2) + if o.endpointSliceEnabled { + p.endpointsChanges = newEndpointsChangesTracker(hostname, o.endpointSliceEnabled, isIPv6) + } return p } -func testClusterIP(t *testing.T, svcIP net.IP, ep1IP, ep2IP net.IP, isIPv6, nodeLocalInternal bool, extraSvcs []*corev1.Service, extraEps []*corev1.Endpoints) { +func testClusterIPAdd(t *testing.T, + svcIP net.IP, + ep1IP net.IP, + ep2IP net.IP, + isIPv6 bool, + nodeLocalInternal bool, + extraSvcs []*corev1.Service, + extraEps []*corev1.Endpoints, + endpointSliceEnabled bool) { ctrl := gomock.NewController(t) defer ctrl.Finish() - mockOFClient := ofmock.NewMockClient(ctrl) - mockRouteClient := routemock.NewMockInterface(ctrl) + mockOFClient, mockRouteClient := getMockClients(ctrl) groupAllocator := openflow.NewGroupAllocator(isIPv6) - fp := NewFakeProxier(mockRouteClient, mockOFClient, nil, groupAllocator, isIPv6, withProxyAll) - - svcPort := 80 - svcPortName := k8sproxy.ServicePortName{ - NamespacedName: makeNamespaceName("ns1", "svc1"), - Port: "80", - Protocol: corev1.ProtocolTCP, + options := []proxyOptionsFn{withProxyAll} + if endpointSliceEnabled { + options = append(options, withEndpointSlice) } + fp := NewFakeProxier(mockRouteClient, mockOFClient, nil, groupAllocator, isIPv6, options...) + internalTrafficPolicy := corev1.ServiceInternalTrafficPolicyCluster if nodeLocalInternal { internalTrafficPolicy = corev1.ServiceInternalTrafficPolicyLocal } - - allServices := append(extraSvcs, - makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *corev1.Service) { - svc.Spec.ClusterIP = svcIP.String() - svc.Spec.Ports = []corev1.ServicePort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: corev1.ProtocolTCP, - }} - svc.Spec.InternalTrafficPolicy = &internalTrafficPolicy - })) - makeServiceMap(fp, allServices...) - - remoteEndpoint := corev1.EndpointSubset{ - Addresses: []corev1.EndpointAddress{{ - IP: ep1IP.String(), - }}, - Ports: []corev1.EndpointPort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: corev1.ProtocolTCP, - }}, - } - localEndpoint := corev1.EndpointSubset{ - Addresses: []corev1.EndpointAddress{{ - IP: ep2IP.String(), - NodeName: &hostname, - }}, - Ports: []corev1.EndpointPort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: corev1.ProtocolTCP, - }}, + allSvcs := append(extraSvcs, makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, nil, &internalTrafficPolicy)) + makeServiceMap(fp, allSvcs...) + + if !endpointSliceEnabled { + remoteEpSubset := makeTestEndpointSubset(&svcPortName, ep1IP, int32(svcPort), corev1.ProtocolTCP, false) + localEpSubset := makeTestEndpointSubset(&svcPortName, ep2IP, int32(svcPort), corev1.ProtocolTCP, true) + allEps := append(extraEps, makeTestEndpoints(&svcPortName, []corev1.EndpointSubset{*remoteEpSubset, *localEpSubset})) + makeEndpointsMap(fp, allEps...) + } else { + remoteEp, remoteEpPort := makeTestEndpointSliceEndpointAndPort(&svcPortName, ep1IP, int32(svcPort), corev1.ProtocolTCP, false) + localEp, localEpPort := makeTestEndpointSliceEndpointAndPort(&svcPortName, ep2IP, int32(svcPort), corev1.ProtocolTCP, true) + endpointSlice := makeTestEndpointSlice(svcPortName.Namespace, + svcPortName.Name, + []discovery.Endpoint{*remoteEp, *localEp}, + []discovery.EndpointPort{*remoteEpPort, *localEpPort}, + isIPv6) + makeEndpointSliceMap(fp, endpointSlice) } - epFunc := func(ept *corev1.Endpoints) { ept.Subsets = []corev1.EndpointSubset{localEndpoint, remoteEndpoint} } - allEps := append(extraEps, makeTestEndpoints(svcPortName.Namespace, svcPortName.Name, epFunc)) - makeEndpointsMap(fp, allEps...) - expectedLocalEps := []k8sproxy.Endpoint{k8sproxy.NewBaseEndpointInfo(ep2IP.String(), "", "", svcPort, true, true, false, false, nil)} - expectedAllEps := expectedLocalEps + var nodeName string + var serving bool + if endpointSliceEnabled { + nodeName = hostname + serving = true + } + expectedAllEps := []k8sproxy.Endpoint{k8sproxy.NewBaseEndpointInfo(ep2IP.String(), nodeName, "", svcPort, true, true, serving, false, nil)} if !nodeLocalInternal { - expectedAllEps = append(expectedAllEps, k8sproxy.NewBaseEndpointInfo(ep1IP.String(), "", "", svcPort, false, true, false, false, nil)) + expectedAllEps = append(expectedAllEps, k8sproxy.NewBaseEndpointInfo(ep1IP.String(), "", "", svcPort, false, true, serving, false, nil)) } bindingProtocol := binding.ProtocolTCP @@ -236,25 +409,30 @@ func testClusterIP(t *testing.T, svcIP net.IP, ep1IP, ep2IP net.IP, isIPv6, node fp.syncProxyRules() } -func testLoadBalancer(t *testing.T, nodePortAddresses []net.IP, svcIP, ep1IP, ep2IP, loadBalancerIP net.IP, isIPv6, nodeLocalInternal, nodeLocalExternal, proxyLoadBalancerIPs bool) { +func testLoadBalancerAdd(t *testing.T, + nodePortAddresses []net.IP, + svcIP net.IP, + ep1IP net.IP, + ep2IP net.IP, + loadBalancerIP net.IP, + isIPv6 bool, + nodeLocalInternal bool, + nodeLocalExternal bool, + proxyLoadBalancerIPs bool, + endpointSliceEnabled bool) { ctrl := gomock.NewController(t) defer ctrl.Finish() - mockOFClient := ofmock.NewMockClient(ctrl) - mockRouteClient := routemock.NewMockInterface(ctrl) + mockOFClient, mockRouteClient := getMockClients(ctrl) groupAllocator := openflow.NewGroupAllocator(isIPv6) options := []proxyOptionsFn{withProxyAll} if !proxyLoadBalancerIPs { options = append(options, withoutProxyLoadBalancerIPs) } + if endpointSliceEnabled { + options = append(options, withEndpointSlice) + } fp := NewFakeProxier(mockRouteClient, mockOFClient, nodePortAddresses, groupAllocator, isIPv6, options...) - svcPort := 80 - svcNodePort := 30008 - svcPortName := k8sproxy.ServicePortName{ - NamespacedName: makeNamespaceName("ns1", "svc1"), - Port: "80", - Protocol: corev1.ProtocolTCP, - } externalTrafficPolicy := corev1.ServiceExternalTrafficPolicyTypeCluster if nodeLocalExternal { externalTrafficPolicy = corev1.ServiceExternalTrafficPolicyTypeLocal @@ -263,60 +441,49 @@ func testLoadBalancer(t *testing.T, nodePortAddresses []net.IP, svcIP, ep1IP, ep if nodeLocalInternal { internalTrafficPolicy = corev1.ServiceInternalTrafficPolicyLocal } - - makeServiceMap(fp, - makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *corev1.Service) { - svc.Spec.ClusterIP = svcIP.String() - svc.Spec.LoadBalancerIP = loadBalancerIP.String() - svc.Spec.Type = corev1.ServiceTypeLoadBalancer - ingress := []corev1.LoadBalancerIngress{{IP: loadBalancerIP.String()}} - svc.Status.LoadBalancer.Ingress = ingress - svc.Spec.Ports = []corev1.ServicePort{{ - NodePort: int32(svcNodePort), - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: corev1.ProtocolTCP, - }} - svc.Spec.ExternalTrafficPolicy = externalTrafficPolicy - svc.Spec.InternalTrafficPolicy = &internalTrafficPolicy - }), - ) - - remoteEndpoint := corev1.EndpointSubset{ - Addresses: []corev1.EndpointAddress{{ - IP: ep1IP.String(), - }}, - Ports: []corev1.EndpointPort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: corev1.ProtocolTCP, - }}, - } - localEndpoint := corev1.EndpointSubset{ - Addresses: []corev1.EndpointAddress{{ - IP: ep2IP.String(), - NodeName: &hostname, - }}, - Ports: []corev1.EndpointPort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: corev1.ProtocolTCP, - }}, + svc := makeTestLoadBalancerService(&svcPortName, svcIP, + []net.IP{loadBalancerIP}, + int32(svcPort), + int32(svcNodePort), + corev1.ProtocolTCP, + nil, + &internalTrafficPolicy, + externalTrafficPolicy) + makeServiceMap(fp, svc) + + if !endpointSliceEnabled { + remoteEpSubset := makeTestEndpointSubset(&svcPortName, ep1IP, int32(svcPort), corev1.ProtocolTCP, false) + localEpSubset := makeTestEndpointSubset(&svcPortName, ep2IP, int32(svcPort), corev1.ProtocolTCP, true) + eps := makeTestEndpoints(&svcPortName, []corev1.EndpointSubset{*remoteEpSubset, *localEpSubset}) + makeEndpointsMap(fp, eps) + } else { + remoteEp, remoteEpPort := makeTestEndpointSliceEndpointAndPort(&svcPortName, ep1IP, int32(svcPort), corev1.ProtocolTCP, false) + localEp, localEpPort := makeTestEndpointSliceEndpointAndPort(&svcPortName, ep2IP, int32(svcPort), corev1.ProtocolTCP, true) + endpointSlice := makeTestEndpointSlice(svcPortName.Namespace, + svcPortName.Name, + []discovery.Endpoint{*remoteEp, *localEp}, + []discovery.EndpointPort{*remoteEpPort, *localEpPort}, + isIPv6) + makeEndpointSliceMap(fp, endpointSlice) } - epFunc := func(ept *corev1.Endpoints) { ept.Subsets = []corev1.EndpointSubset{localEndpoint, remoteEndpoint} } - eps := []*corev1.Endpoints{makeTestEndpoints(svcPortName.Namespace, svcPortName.Name, epFunc)} - makeEndpointsMap(fp, eps...) - - expectedLocalEps := []k8sproxy.Endpoint{k8sproxy.NewBaseEndpointInfo(ep2IP.String(), "", "", svcPort, true, true, false, false, nil)} + var nodeName string + var serving bool + if endpointSliceEnabled { + nodeName = hostname + serving = true + } + expectedLocalEps := []k8sproxy.Endpoint{k8sproxy.NewBaseEndpointInfo(ep2IP.String(), nodeName, "", svcPort, true, true, serving, false, nil)} expectedAllEps := expectedLocalEps if !(nodeLocalInternal && nodeLocalExternal) { - expectedAllEps = append(expectedAllEps, k8sproxy.NewBaseEndpointInfo(ep1IP.String(), "", "", svcPort, false, true, false, false, nil)) + expectedAllEps = append(expectedAllEps, k8sproxy.NewBaseEndpointInfo(ep1IP.String(), "", "", svcPort, false, true, serving, false, nil)) } bindingProtocol := binding.ProtocolTCP + vIP := agentconfig.VirtualNodePortDNATIPv4 if isIPv6 { bindingProtocol = binding.ProtocolTCPv6 + vIP = agentconfig.VirtualNodePortDNATIPv6 } mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, gomock.InAnyOrder(expectedAllEps)).Times(1) @@ -334,7 +501,7 @@ func testLoadBalancer(t *testing.T, nodePortAddresses []net.IP, svcIP, ep1IP, ep mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), nodeLocalExternal, corev1.ServiceTypeClusterIP).Times(1) groupID = fp.groupCounter.AllocateIfNotExist(svcPortName, nodeLocalExternal) mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.InAnyOrder(nodePortEps)).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, gomock.Any(), uint16(svcNodePort), bindingProtocol, uint16(0), nodeLocalExternal, corev1.ServiceTypeNodePort).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), nodeLocalExternal, corev1.ServiceTypeNodePort).Times(1) if proxyLoadBalancerIPs { mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), nodeLocalExternal, corev1.ServiceTypeLoadBalancer).Times(1) } @@ -343,7 +510,7 @@ func testLoadBalancer(t *testing.T, nodePortAddresses []net.IP, svcIP, ep1IP, ep groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, nodeLocalVal) mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.InAnyOrder(expectedAllEps)).Times(1) mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), nodeLocalExternal, corev1.ServiceTypeClusterIP).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, gomock.Any(), uint16(svcNodePort), bindingProtocol, uint16(0), nodeLocalExternal, corev1.ServiceTypeNodePort).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), nodeLocalExternal, corev1.ServiceTypeNodePort).Times(1) if proxyLoadBalancerIPs { mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), nodeLocalExternal, corev1.ServiceTypeLoadBalancer).Times(1) } @@ -352,28 +519,32 @@ func testLoadBalancer(t *testing.T, nodePortAddresses []net.IP, svcIP, ep1IP, ep } mockRouteClient.EXPECT().AddClusterIPRoute(svcIP).Times(1) if proxyLoadBalancerIPs { - mockRouteClient.EXPECT().AddLoadBalancer([]string{loadBalancerIP.String()}).Times(1) + mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) } mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) fp.syncProxyRules() } -func testNodePort(t *testing.T, nodePortAddresses []net.IP, svcIP, ep1IP, ep2IP net.IP, isIPv6, nodeLocalInternal, nodeLocalExternal bool) { +func testNodePortAdd(t *testing.T, + nodePortAddresses []net.IP, + svcIP net.IP, + ep1IP net.IP, + ep2IP net.IP, + isIPv6 bool, + nodeLocalInternal bool, + nodeLocalExternal bool, + endpointSliceEnabled bool) { ctrl := gomock.NewController(t) defer ctrl.Finish() - mockOFClient := ofmock.NewMockClient(ctrl) - mockRouteClient := routemock.NewMockInterface(ctrl) + mockOFClient, mockRouteClient := getMockClients(ctrl) groupAllocator := openflow.NewGroupAllocator(isIPv6) - fp := NewFakeProxier(mockRouteClient, mockOFClient, nodePortAddresses, groupAllocator, isIPv6, withProxyAll) - - svcPort := 80 - svcNodePort := 31000 - svcPortName := k8sproxy.ServicePortName{ - NamespacedName: makeNamespaceName("ns1", "svc1"), - Port: "80", - Protocol: corev1.ProtocolTCP, + options := []proxyOptionsFn{withProxyAll} + if endpointSliceEnabled { + options = append(options, withEndpointSlice) } + fp := NewFakeProxier(mockRouteClient, mockOFClient, nodePortAddresses, groupAllocator, isIPv6, options...) + externalTrafficPolicy := corev1.ServiceExternalTrafficPolicyTypeCluster if nodeLocalExternal { externalTrafficPolicy = corev1.ServiceExternalTrafficPolicyTypeLocal @@ -382,64 +553,48 @@ func testNodePort(t *testing.T, nodePortAddresses []net.IP, svcIP, ep1IP, ep2IP if nodeLocalInternal { internalTrafficPolicy = corev1.ServiceInternalTrafficPolicyLocal } - - makeServiceMap(fp, - makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *corev1.Service) { - svc.Spec.ClusterIP = svcIP.String() - svc.Spec.Type = corev1.ServiceTypeNodePort - svc.Spec.Ports = []corev1.ServicePort{{ - NodePort: int32(svcNodePort), - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: corev1.ProtocolTCP, - }} - svc.Spec.ExternalTrafficPolicy = externalTrafficPolicy - svc.Spec.InternalTrafficPolicy = &internalTrafficPolicy - }), - ) - - remoteEndpoint := corev1.EndpointSubset{ - Addresses: []corev1.EndpointAddress{{ - IP: ep1IP.String(), - }}, - Ports: []corev1.EndpointPort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: corev1.ProtocolTCP, - }}, - } - localEndpoint := corev1.EndpointSubset{ - Addresses: []corev1.EndpointAddress{{ - IP: ep2IP.String(), - NodeName: &hostname, - }}, - Ports: []corev1.EndpointPort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: corev1.ProtocolTCP, - }}, + svc := makeTestNodePortService(&svcPortName, svcIP, + int32(svcPort), + int32(svcNodePort), + corev1.ProtocolTCP, + nil, + internalTrafficPolicy, + externalTrafficPolicy) + makeServiceMap(fp, svc) + + if !endpointSliceEnabled { + remoteEpSubset := makeTestEndpointSubset(&svcPortName, ep1IP, int32(svcPort), corev1.ProtocolTCP, false) + localEpSubset := makeTestEndpointSubset(&svcPortName, ep2IP, int32(svcPort), corev1.ProtocolTCP, true) + eps := makeTestEndpoints(&svcPortName, []corev1.EndpointSubset{*remoteEpSubset, *localEpSubset}) + makeEndpointsMap(fp, eps) + } else { + remoteEp, remoteEpPort := makeTestEndpointSliceEndpointAndPort(&svcPortName, ep1IP, int32(svcPort), corev1.ProtocolTCP, false) + localEp, localEpPort := makeTestEndpointSliceEndpointAndPort(&svcPortName, ep2IP, int32(svcPort), corev1.ProtocolTCP, true) + endpointSlice := makeTestEndpointSlice(svcPortName.Namespace, + svcPortName.Name, + []discovery.Endpoint{*remoteEp, *localEp}, + []discovery.EndpointPort{*remoteEpPort, *localEpPort}, + isIPv6) + makeEndpointSliceMap(fp, endpointSlice) } - var eps []*corev1.Endpoints - epFunc := func(ept *corev1.Endpoints) { - if nodeLocalInternal && nodeLocalExternal { - ept.Subsets = []corev1.EndpointSubset{localEndpoint} - } else { - ept.Subsets = []corev1.EndpointSubset{localEndpoint, remoteEndpoint} - } + var nodeName string + var serving bool + if endpointSliceEnabled { + nodeName = hostname + serving = true } - eps = append(eps, makeTestEndpoints(svcPortName.Namespace, svcPortName.Name, epFunc)) - makeEndpointsMap(fp, eps...) - - expectedLocalEps := []k8sproxy.Endpoint{k8sproxy.NewBaseEndpointInfo(ep2IP.String(), "", "", svcPort, true, true, false, false, nil)} + expectedLocalEps := []k8sproxy.Endpoint{k8sproxy.NewBaseEndpointInfo(ep2IP.String(), nodeName, "", svcPort, true, true, serving, false, nil)} expectedAllEps := expectedLocalEps if !(nodeLocalInternal && nodeLocalExternal) { - expectedAllEps = append(expectedAllEps, k8sproxy.NewBaseEndpointInfo(ep1IP.String(), "", "", svcPort, false, true, false, false, nil)) + expectedAllEps = append(expectedAllEps, k8sproxy.NewBaseEndpointInfo(ep1IP.String(), "", "", svcPort, false, true, serving, false, nil)) } bindingProtocol := binding.ProtocolTCP + vIP := agentconfig.VirtualNodePortDNATIPv4 if isIPv6 { bindingProtocol = binding.ProtocolTCPv6 + vIP = agentconfig.VirtualNodePortDNATIPv6 } mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, gomock.InAnyOrder(expectedAllEps)).Times(1) @@ -458,192 +613,397 @@ func testNodePort(t *testing.T, nodePortAddresses []net.IP, svcIP, ep1IP, ep2IP groupID = fp.groupCounter.AllocateIfNotExist(svcPortName, nodeLocalExternal) mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.InAnyOrder(nodePortEps)).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, gomock.Any(), uint16(svcNodePort), bindingProtocol, uint16(0), nodeLocalExternal, corev1.ServiceTypeNodePort).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), nodeLocalExternal, corev1.ServiceTypeNodePort).Times(1) } else { nodeLocalVal := nodeLocalInternal && nodeLocalExternal groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, nodeLocalVal) mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.InAnyOrder(expectedAllEps)).Times(1) mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), nodeLocalExternal, corev1.ServiceTypeClusterIP).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, gomock.Any(), uint16(svcNodePort), bindingProtocol, uint16(0), nodeLocalExternal, corev1.ServiceTypeNodePort).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), nodeLocalExternal, corev1.ServiceTypeNodePort).Times(1) groupID = fp.groupCounter.AllocateIfNotExist(svcPortName, !nodeLocalVal) mockOFClient.EXPECT().UninstallServiceGroup(groupID).Times(1) } mockRouteClient.EXPECT().AddClusterIPRoute(svcIP).Times(1) - mockRouteClient.EXPECT().AddNodePort(gomock.Any(), uint16(svcNodePort), bindingProtocol).Times(1) + mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) fp.syncProxyRules() } -func TestCluster(t *testing.T) { +func TestClusterIPAdd(t *testing.T) { t.Run("IPv4", func(t *testing.T) { - t.Run("InternalTrafficPolicy Cluster", func(t *testing.T) { - testClusterIP(t, svcIPv4, ep1IPv4, ep2IPv4, false, false, []*corev1.Service{}, []*corev1.Endpoints{}) + t.Run("Endpoints", func(t *testing.T) { + t.Run("InternalTrafficPolicy Cluster", func(t *testing.T) { + testClusterIPAdd(t, svc1IPv4, ep1IPv4, ep2IPv4, false, false, []*corev1.Service{}, []*corev1.Endpoints{}, false) + }) + t.Run("InternalTrafficPolicy Local", func(t *testing.T) { + testClusterIPAdd(t, svc1IPv4, ep1IPv4, ep2IPv4, false, true, []*corev1.Service{}, []*corev1.Endpoints{}, false) + }) }) - t.Run("InternalTrafficPolicy Local", func(t *testing.T) { - testClusterIP(t, svcIPv4, ep1IPv4, ep2IPv4, false, true, []*corev1.Service{}, []*corev1.Endpoints{}) + t.Run("EndpointSlice", func(t *testing.T) { + t.Run("InternalTrafficPolicy Cluster", func(t *testing.T) { + testClusterIPAdd(t, svc1IPv4, ep1IPv4, ep2IPv4, false, false, []*corev1.Service{}, []*corev1.Endpoints{}, true) + }) + t.Run("InternalTrafficPolicy Local", func(t *testing.T) { + testClusterIPAdd(t, svc1IPv4, ep1IPv4, ep2IPv4, false, true, []*corev1.Service{}, []*corev1.Endpoints{}, true) + }) }) }) t.Run("IPv6", func(t *testing.T) { - t.Run("InternalTrafficPolicy Cluster", func(t *testing.T) { - testClusterIP(t, svcIPv6, ep1IPv6, ep2IPv6, true, false, []*corev1.Service{}, []*corev1.Endpoints{}) + t.Run("Endpoints", func(t *testing.T) { + t.Run("InternalTrafficPolicy Cluster", func(t *testing.T) { + testClusterIPAdd(t, svc1IPv6, ep1IPv6, ep2IPv6, true, false, []*corev1.Service{}, []*corev1.Endpoints{}, false) + }) + t.Run("InternalTrafficPolicy Local", func(t *testing.T) { + testClusterIPAdd(t, svc1IPv6, ep1IPv6, ep2IPv6, true, true, []*corev1.Service{}, []*corev1.Endpoints{}, false) + }) }) - t.Run("InternalTrafficPolicy Local", func(t *testing.T) { - testClusterIP(t, svcIPv6, ep1IPv6, ep2IPv6, true, true, []*corev1.Service{}, []*corev1.Endpoints{}) + t.Run("EndpointSlice", func(t *testing.T) { + t.Run("InternalTrafficPolicy Cluster", func(t *testing.T) { + testClusterIPAdd(t, svc1IPv6, ep1IPv6, ep2IPv6, true, false, []*corev1.Service{}, []*corev1.Endpoints{}, true) + }) + t.Run("InternalTrafficPolicy Local", func(t *testing.T) { + testClusterIPAdd(t, svc1IPv6, ep1IPv6, ep2IPv6, true, true, []*corev1.Service{}, []*corev1.Endpoints{}, true) + }) }) }) } -func TestLoadBalancer(t *testing.T) { +func TestLoadBalancerAdd(t *testing.T) { t.Run("IPv4", func(t *testing.T) { - t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testLoadBalancer(t, nodePortAddressesIPv4, svcIPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, false, false, true) + t.Run("Endpoints", func(t *testing.T) { + t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Cluster", func(t *testing.T) { + testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, false, false, true, false) + }) + t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Local", func(t *testing.T) { + testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, false, true, true, false) + }) + t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Cluster", func(t *testing.T) { + testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, true, false, true, false) + }) + t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Local", func(t *testing.T) { + testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, true, true, true, false) + }) + t.Run("No External IPs", func(t *testing.T) { + testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, false, false, false, false) + }) }) - t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Local", func(t *testing.T) { - testLoadBalancer(t, nodePortAddressesIPv4, svcIPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, false, true, true) - }) - t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testLoadBalancer(t, nodePortAddressesIPv4, svcIPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, true, false, true) - }) - t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Local", func(t *testing.T) { - testLoadBalancer(t, nodePortAddressesIPv4, svcIPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, true, true, true) - }) - t.Run("No External IPs", func(t *testing.T) { - testLoadBalancer(t, nodePortAddressesIPv4, svcIPv4, ep1IPv4, nil, loadBalancerIPv4, false, false, false, false) + t.Run("EndpointSlice", func(t *testing.T) { + t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Cluster", func(t *testing.T) { + testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, false, false, true, true) + }) + t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Local", func(t *testing.T) { + testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, false, true, true, true) + }) + t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Cluster", func(t *testing.T) { + testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, true, false, true, true) + }) + t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Local", func(t *testing.T) { + testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, true, true, true, true) + }) + t.Run("No External IPs", func(t *testing.T) { + testLoadBalancerAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, loadBalancerIPv4, false, false, false, false, true) + }) }) }) t.Run("IPv6", func(t *testing.T) { - t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testLoadBalancer(t, nodePortAddressesIPv6, svcIPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, false, false, true) - }) - t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Local", func(t *testing.T) { - testLoadBalancer(t, nodePortAddressesIPv6, svcIPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, false, true, true) + t.Run("Endpoints", func(t *testing.T) { + t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Cluster", func(t *testing.T) { + testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, false, false, true, false) + }) + t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Local", func(t *testing.T) { + testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, false, true, true, false) + }) + t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Cluster", func(t *testing.T) { + testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, true, false, true, false) + }) + t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Local", func(t *testing.T) { + testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, true, true, true, false) + }) + t.Run("No External IPs", func(t *testing.T) { + testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, false, false, false, false) + }) }) - t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testLoadBalancer(t, nodePortAddressesIPv6, svcIPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, true, false, true) - }) - t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Local", func(t *testing.T) { - testLoadBalancer(t, nodePortAddressesIPv6, svcIPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, true, true, true) - }) - t.Run("No External IPs", func(t *testing.T) { - testLoadBalancer(t, nodePortAddressesIPv6, svcIPv6, ep1IPv6, nil, loadBalancerIPv6, true, false, false, false) + t.Run("EndpointSlice", func(t *testing.T) { + t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Cluster", func(t *testing.T) { + testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, false, false, true, true) + }) + t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Local", func(t *testing.T) { + testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, false, true, true, true) + }) + t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Cluster", func(t *testing.T) { + testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, true, false, true, true) + }) + t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Local", func(t *testing.T) { + testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, true, true, true, true) + }) + t.Run("No External IPs", func(t *testing.T) { + testLoadBalancerAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, loadBalancerIPv6, true, false, false, false, true) + }) }) }) } -func TestNodePort(t *testing.T) { +func TestLoadBalancerServiceWithMultiplePorts(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockOFClient, mockRouteClient := getMockClients(ctrl) + groupAllocator := openflow.NewGroupAllocator(false) + nodePortAddresses := []net.IP{net.ParseIP("0.0.0.0")} + fp := NewFakeProxier(mockRouteClient, mockOFClient, nodePortAddresses, groupAllocator, false, withProxyAll, withEndpointSlice) + + port80Str := "port80" + port80Int32 := int32(80) + port443Str := "port443" + port443Int32 := int32(443) + port30001Int32 := int32(30001) + port30002Int32 := int32(30002) + protocolTCP := corev1.ProtocolTCP + endpoint1Address := "192.168.0.11" + endpoint2Address := "192.168.1.11" + endpoint1NodeName := fp.hostname + endpoint2NodeName := "node2" + + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "svc", + Namespace: "default", + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: port80Str, + Protocol: protocolTCP, + Port: port80Int32, + TargetPort: intstr.FromInt(int(port80Int32)), + NodePort: port30001Int32, + }, + { + Name: port443Str, + Protocol: protocolTCP, + Port: port443Int32, + TargetPort: intstr.FromInt(int(port443Int32)), + NodePort: port30002Int32, + }, + }, + ClusterIP: svc1IPv4.String(), + ClusterIPs: []string{svc1IPv4.String()}, + Type: corev1.ServiceTypeLoadBalancer, + ExternalTrafficPolicy: corev1.ServiceExternalTrafficPolicyTypeLocal, + HealthCheckNodePort: 40000, + IPFamilies: []corev1.IPFamily{corev1.IPv4Protocol}, + }, + Status: corev1.ServiceStatus{ + LoadBalancer: corev1.LoadBalancerStatus{Ingress: []corev1.LoadBalancerIngress{ + {IP: loadBalancerIPv4.String()}, + }}, + }, + } + makeServiceMap(fp, svc) + + endpointSlice := &discovery.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "svc-x5ks2", + Namespace: svc.Namespace, + Labels: map[string]string{ + discovery.LabelServiceName: svc.Name, + }, + }, + AddressType: discovery.AddressTypeIPv4, + Endpoints: []discovery.Endpoint{ + { + Addresses: []string{ + endpoint1Address, + }, + Conditions: discovery.EndpointConditions{ + Ready: pointer.Bool(true), + Serving: pointer.Bool(true), + Terminating: pointer.Bool(false), + }, + NodeName: &endpoint1NodeName, + }, + { + Addresses: []string{ + endpoint2Address, + }, + Conditions: discovery.EndpointConditions{ + Ready: pointer.Bool(true), + Serving: pointer.Bool(true), + Terminating: pointer.Bool(false), + }, + NodeName: &endpoint2NodeName, + }, + }, + Ports: []discovery.EndpointPort{ + { + Name: &port80Str, + Port: &port80Int32, + Protocol: &protocolTCP, + }, + { + Name: &port443Str, + Port: &port443Int32, + Protocol: &protocolTCP, + }, + }, + } + makeEndpointSliceMap(fp, endpointSlice) + + localEndpointForPort80 := k8sproxy.NewBaseEndpointInfo(endpoint1Address, endpoint1NodeName, "", int(port80Int32), true, true, true, false, nil) + localEndpointForPort443 := k8sproxy.NewBaseEndpointInfo(endpoint1Address, endpoint1NodeName, "", int(port443Int32), true, true, true, false, nil) + remoteEndpointForPort80 := k8sproxy.NewBaseEndpointInfo(endpoint2Address, endpoint2NodeName, "", int(port80Int32), false, true, true, false, nil) + remoteEndpointForPort443 := k8sproxy.NewBaseEndpointInfo(endpoint2Address, endpoint2NodeName, "", int(port443Int32), false, true, true, false, nil) + + mockOFClient.EXPECT().InstallEndpointFlows(binding.ProtocolTCP, gomock.InAnyOrder([]k8sproxy.Endpoint{localEndpointForPort80, remoteEndpointForPort80})).Times(1) + mockOFClient.EXPECT().InstallServiceGroup(gomock.Any(), false, []k8sproxy.Endpoint{localEndpointForPort80}).Times(1) + mockOFClient.EXPECT().InstallServiceGroup(gomock.Any(), false, gomock.InAnyOrder([]k8sproxy.Endpoint{localEndpointForPort80, remoteEndpointForPort80})).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), svc1IPv4, uint16(port80Int32), binding.ProtocolTCP, uint16(0), true, corev1.ServiceTypeClusterIP).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), agentconfig.VirtualNodePortDNATIPv4, uint16(port30001Int32), binding.ProtocolTCP, uint16(0), true, corev1.ServiceTypeNodePort).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), loadBalancerIPv4, uint16(port80Int32), binding.ProtocolTCP, uint16(0), true, corev1.ServiceTypeLoadBalancer).Times(1) + mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(port30001Int32), binding.ProtocolTCP).Times(1) + // The route for the ClusterIP and the LoadBalancer IP should only be installed once. + mockRouteClient.EXPECT().AddClusterIPRoute(svc1IPv4).Times(1) + mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIPv4).Times(1) + + mockOFClient.EXPECT().InstallEndpointFlows(binding.ProtocolTCP, gomock.InAnyOrder([]k8sproxy.Endpoint{localEndpointForPort443, remoteEndpointForPort443})).Times(1) + mockOFClient.EXPECT().InstallServiceGroup(gomock.Any(), false, []k8sproxy.Endpoint{localEndpointForPort443}).Times(1) + mockOFClient.EXPECT().InstallServiceGroup(gomock.Any(), false, gomock.InAnyOrder([]k8sproxy.Endpoint{localEndpointForPort443, remoteEndpointForPort443})).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), svc1IPv4, uint16(port443Int32), binding.ProtocolTCP, uint16(0), true, corev1.ServiceTypeClusterIP).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), agentconfig.VirtualNodePortDNATIPv4, uint16(port30002Int32), binding.ProtocolTCP, uint16(0), true, corev1.ServiceTypeNodePort).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), loadBalancerIPv4, uint16(port443Int32), binding.ProtocolTCP, uint16(0), true, corev1.ServiceTypeLoadBalancer).Times(1) + mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(port30002Int32), binding.ProtocolTCP).Times(1) + + fp.syncProxyRules() + + // Remove the service. + fp.serviceChanges.OnServiceUpdate(svc, nil) + fp.endpointsChanges.OnEndpointSliceUpdate(endpointSlice, true) + + mockOFClient.EXPECT().UninstallEndpointFlows(binding.ProtocolTCP, localEndpointForPort80) + mockOFClient.EXPECT().UninstallEndpointFlows(binding.ProtocolTCP, remoteEndpointForPort80) + mockOFClient.EXPECT().UninstallServiceGroup(gomock.Any()).Times(2) + mockOFClient.EXPECT().UninstallServiceFlows(svc1IPv4, uint16(port80Int32), binding.ProtocolTCP) + mockOFClient.EXPECT().UninstallServiceFlows(agentconfig.VirtualNodePortDNATIPv4, uint16(port30001Int32), binding.ProtocolTCP) + mockOFClient.EXPECT().UninstallServiceFlows(loadBalancerIPv4, uint16(port80Int32), binding.ProtocolTCP) + mockRouteClient.EXPECT().DeleteNodePort(nodePortAddresses, uint16(port30001Int32), binding.ProtocolTCP) + + mockOFClient.EXPECT().UninstallEndpointFlows(binding.ProtocolTCP, localEndpointForPort443) + mockOFClient.EXPECT().UninstallEndpointFlows(binding.ProtocolTCP, remoteEndpointForPort443) + mockOFClient.EXPECT().UninstallServiceGroup(gomock.Any()).Times(2) + mockOFClient.EXPECT().UninstallServiceFlows(svc1IPv4, uint16(port443Int32), binding.ProtocolTCP) + mockOFClient.EXPECT().UninstallServiceFlows(agentconfig.VirtualNodePortDNATIPv4, uint16(port30002Int32), binding.ProtocolTCP) + mockOFClient.EXPECT().UninstallServiceFlows(loadBalancerIPv4, uint16(port443Int32), binding.ProtocolTCP) + mockRouteClient.EXPECT().DeleteNodePort(nodePortAddresses, uint16(port30002Int32), binding.ProtocolTCP) + // The route for the ClusterIP and the LoadBalancer IP should only be uninstalled once. + mockRouteClient.EXPECT().DeleteClusterIPRoute(svc1IPv4) + mockRouteClient.EXPECT().DeleteLoadBalancer(loadBalancerIPv4) + + fp.syncProxyRules() + + assert.Emptyf(t, fp.serviceIPRouteReferences, "serviceIPRouteReferences was not cleaned up after Service was removed") +} + +func TestNodePortAdd(t *testing.T) { t.Run("IPv4", func(t *testing.T) { - t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testNodePort(t, nodePortAddressesIPv4, svcIPv4, ep1IPv4, ep2IPv4, false, false, false) - }) - t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Local", func(t *testing.T) { - testNodePort(t, nodePortAddressesIPv4, svcIPv4, ep1IPv4, ep2IPv4, false, false, true) + t.Run("Endpoints", func(t *testing.T) { + t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Cluster", func(t *testing.T) { + testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, false, false, false, false) + }) + t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Local", func(t *testing.T) { + testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, false, false, true, false) + }) + t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Cluster", func(t *testing.T) { + testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, false, true, false, false) + }) + t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Local", func(t *testing.T) { + testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, false, true, true, false) + }) }) - t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testNodePort(t, nodePortAddressesIPv4, svcIPv4, ep1IPv4, ep2IPv4, false, true, false) - }) - t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Local", func(t *testing.T) { - testNodePort(t, nodePortAddressesIPv4, svcIPv4, ep1IPv4, ep2IPv4, false, true, true) + t.Run("EndpointSlice", func(t *testing.T) { + t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Cluster", func(t *testing.T) { + testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, false, false, false, true) + }) + t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Local", func(t *testing.T) { + testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, false, false, true, true) + }) + t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Cluster", func(t *testing.T) { + testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, false, true, false, true) + }) + t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Local", func(t *testing.T) { + testNodePortAdd(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, ep2IPv4, false, true, true, true) + }) }) }) t.Run("IPv6", func(t *testing.T) { - t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testNodePort(t, nodePortAddressesIPv6, svcIPv6, ep1IPv6, ep2IPv6, true, false, false) - }) - t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Local", func(t *testing.T) { - testNodePort(t, nodePortAddressesIPv6, svcIPv6, ep1IPv6, ep2IPv6, true, false, true) + t.Run("Endpoints", func(t *testing.T) { + t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Cluster", func(t *testing.T) { + testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, true, false, false, false) + }) + t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Local", func(t *testing.T) { + testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, true, false, true, false) + }) + t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Cluster", func(t *testing.T) { + testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, true, true, false, false) + }) + t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Local", func(t *testing.T) { + testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, true, true, true, false) + }) }) - t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Cluster", func(t *testing.T) { - testNodePort(t, nodePortAddressesIPv6, svcIPv6, ep1IPv6, ep2IPv6, true, true, false) - }) - t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Local", func(t *testing.T) { - testNodePort(t, nodePortAddressesIPv6, svcIPv6, ep1IPv6, ep2IPv6, true, true, true) + t.Run("EndpointSlice", func(t *testing.T) { + t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Cluster", func(t *testing.T) { + testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, true, false, false, true) + }) + t.Run("InternalTrafficPolicy:Cluster ExternalTrafficPolicy:Local", func(t *testing.T) { + testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, true, false, true, true) + }) + t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Cluster", func(t *testing.T) { + testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, true, true, false, true) + }) + t.Run("InternalTrafficPolicy:Local ExternalTrafficPolicy:Local", func(t *testing.T) { + testNodePortAdd(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, ep2IPv6, true, true, true, true) + }) }) }) } func TestClusterSkipServices(t *testing.T) { svc1Port := 53 - svc1PortName := k8sproxy.ServicePortName{ - NamespacedName: makeNamespaceName("kube-system", "kube-dns"), - Port: "53", - Protocol: corev1.ProtocolTCP, - } svc2Port := 88 - svc2PortName := k8sproxy.ServicePortName{ - NamespacedName: makeNamespaceName("kube-system", "test"), - Port: "88", - Protocol: corev1.ProtocolTCP, - } - svc1 := makeTestService(svc1PortName.Namespace, svc1PortName.Name, func(svc *corev1.Service) { - svc.Spec.ClusterIP = "10.96.10.12" - svc.Spec.Ports = []corev1.ServicePort{{ - Name: svc1PortName.Port, - Port: int32(svc1Port), - Protocol: corev1.ProtocolTCP, - }} - }) - svc2 := makeTestService(svc2PortName.Namespace, svc2PortName.Name, func(svc *corev1.Service) { - svc.Spec.ClusterIP = "192.168.1.2" - svc.Spec.Ports = []corev1.ServicePort{{ - Name: svc2PortName.Port, - Port: int32(svc2Port), - Protocol: corev1.ProtocolTCP, - }} - }) + svc1ClusterIP := net.ParseIP("10.96.10.12") + svc2ClusterIP := net.ParseIP(skippedClusterIP) + ep1IP := net.ParseIP("172.16.1.2") + ep2IP := net.ParseIP("172.16.1.3") + + skippedServiceNamespace := strings.Split(skippedServiceNN, "/")[0] + skippedServiceName := strings.Split(skippedServiceNN, "/")[1] + svc1PortName := makeSvcPortName(skippedServiceNamespace, skippedServiceName, strconv.Itoa(svc1Port), corev1.ProtocolTCP) + svc2PortName := makeSvcPortName("kube-system", "test", strconv.Itoa(svc2Port), corev1.ProtocolTCP) + svc1 := makeTestClusterIPService(&svc1PortName, svc1ClusterIP, int32(svc1Port), corev1.ProtocolTCP, nil, nil) + svc2 := makeTestClusterIPService(&svc2PortName, svc2ClusterIP, int32(svc2Port), corev1.ProtocolTCP, nil, nil) svcs := []*corev1.Service{svc1, svc2} - ep1 := makeTestEndpoints(svc1PortName.Namespace, svc1PortName.Name, func(ept *corev1.Endpoints) { - ept.Subsets = []corev1.EndpointSubset{{ - Addresses: []corev1.EndpointAddress{{ - IP: "172.16.1.2", - }}, - Ports: []corev1.EndpointPort{{ - Name: svc1PortName.Port, - Port: int32(svc1Port), - Protocol: corev1.ProtocolTCP, - }}, - }} - }) - ep2 := makeTestEndpoints(svc2PortName.Namespace, svc2PortName.Name, func(ept *corev1.Endpoints) { - ept.Subsets = []corev1.EndpointSubset{{ - Addresses: []corev1.EndpointAddress{{ - IP: "172.16.1.3", - }}, - Ports: []corev1.EndpointPort{{ - Name: svc2PortName.Port, - Port: int32(svc2Port), - Protocol: corev1.ProtocolTCP, - }}, - }} - }) + epSubset := makeTestEndpointSubset(&svc1PortName, ep1IP, int32(svc1Port), corev1.ProtocolTCP, false) + ep1 := makeTestEndpoints(&svc1PortName, []corev1.EndpointSubset{*epSubset}) + epSubset = makeTestEndpointSubset(&svc1PortName, ep2IP, int32(svc2Port), corev1.ProtocolTCP, false) + ep2 := makeTestEndpoints(&svc2PortName, []corev1.EndpointSubset{*epSubset}) eps := []*corev1.Endpoints{ep1, ep2} - testClusterIP(t, svcIPv4, ep1IPv4, ep2IPv4, false, false, svcs, eps) + + testClusterIPAdd(t, svc1IPv4, ep1IPv4, ep2IPv4, false, false, svcs, eps, false) } func TestDualStackService(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - mockOFClient := ofmock.NewMockClient(ctrl) - mockRouteClient := routemock.NewMockInterface(ctrl) - ipv4GGroupAllocator := openflow.NewGroupAllocator(false) - ipv6GGroupAllocator := openflow.NewGroupAllocator(true) - fpv4 := NewFakeProxier(mockRouteClient, mockOFClient, nil, ipv4GGroupAllocator, false) - fpv6 := NewFakeProxier(mockRouteClient, mockOFClient, nil, ipv6GGroupAllocator, true) + mockOFClient, mockRouteClient := getMockClients(ctrl) + ipv4GroupAllocator := openflow.NewGroupAllocator(false) + ipv6GroupAllocator := openflow.NewGroupAllocator(true) + fpv4 := NewFakeProxier(mockRouteClient, mockOFClient, nil, ipv4GroupAllocator, false) + fpv6 := NewFakeProxier(mockRouteClient, mockOFClient, nil, ipv6GroupAllocator, true) metaProxier := k8sproxy.NewMetaProxier(fpv4, fpv6) - svcPort := 80 - svcPortName := k8sproxy.ServicePortName{ - NamespacedName: makeNamespaceName("ns1", "svc1"), - Port: "80", - Protocol: corev1.ProtocolTCP, - } - svcIPv4 := net.ParseIP("10.20.30.41") - svcIPv6 := net.ParseIP("10:20::41") - - s := makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *corev1.Service) { - svc.Spec.ClusterIP = svcIPv4.String() - svc.Spec.ClusterIPs = []string{svcIPv4.String(), svcIPv6.String()} + svc := makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *corev1.Service) { + svc.Spec.ClusterIP = svc1IPv4.String() + svc.Spec.ClusterIPs = []string{svc1IPv4.String(), svc1IPv6.String()} svc.Spec.IPFamilies = []corev1.IPFamily{corev1.IPv4Protocol, corev1.IPv6Protocol} svc.Spec.Ports = []corev1.ServicePort{{ Name: svcPortName.Port, @@ -652,33 +1012,12 @@ func TestDualStackService(t *testing.T) { }} }) - epv4 := makeTestEndpoints(svcPortName.Namespace, svcPortName.Name, func(ept *corev1.Endpoints) { - ept.Subsets = []corev1.EndpointSubset{{ - Addresses: []corev1.EndpointAddress{{ - IP: "10.180.30.41", - }}, - Ports: []corev1.EndpointPort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: corev1.ProtocolTCP, - }}, - }} - }) - - epv6 := makeTestEndpoints(svcPortName.Namespace, svcPortName.Name, func(ept *corev1.Endpoints) { - ept.Subsets = []corev1.EndpointSubset{{ - Addresses: []corev1.EndpointAddress{{ - IP: "10:180::1", - }}, - Ports: []corev1.EndpointPort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: corev1.ProtocolTCP, - }}, - }} - }) + epSubset := makeTestEndpointSubset(&svcPortName, ep1IPv4, int32(svcPort), corev1.ProtocolTCP, false) + epv4 := makeTestEndpoints(&svcPortName, []corev1.EndpointSubset{*epSubset}) + epSubset = makeTestEndpointSubset(&svcPortName, ep1IPv6, int32(svcPort), corev1.ProtocolTCP, false) + epv6 := makeTestEndpoints(&svcPortName, []corev1.EndpointSubset{*epSubset}) - metaProxier.OnServiceUpdate(nil, s) + metaProxier.OnServiceUpdate(nil, svc) metaProxier.OnServiceSynced() metaProxier.OnEndpointsUpdate(nil, epv4) metaProxier.OnEndpointsUpdate(nil, epv6) @@ -689,59 +1028,35 @@ func TestDualStackService(t *testing.T) { mockOFClient.EXPECT().InstallServiceGroup(groupIDv4, false, gomock.Any()).Times(1) mockOFClient.EXPECT().InstallEndpointFlows(binding.ProtocolTCP, gomock.Any()).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupIDv4, svcIPv4, uint16(svcPort), binding.ProtocolTCP, uint16(0), false, corev1.ServiceTypeClusterIP).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupIDv4, svc1IPv4, uint16(svcPort), binding.ProtocolTCP, uint16(0), false, corev1.ServiceTypeClusterIP).Times(1) mockOFClient.EXPECT().InstallServiceGroup(groupIDv6, false, gomock.Any()).Times(1) mockOFClient.EXPECT().InstallEndpointFlows(binding.ProtocolTCPv6, gomock.Any()).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupIDv6, svcIPv6, uint16(svcPort), binding.ProtocolTCPv6, uint16(0), false, corev1.ServiceTypeClusterIP).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupIDv6, svc1IPv6, uint16(svcPort), binding.ProtocolTCPv6, uint16(0), false, corev1.ServiceTypeClusterIP).Times(1) fpv4.syncProxyRules() fpv6.syncProxyRules() } -func testClusterIPRemoval(t *testing.T, svcIP net.IP, epIP net.IP, isIPv6 bool) { +func testClusterIPRemove(t *testing.T, svcIP net.IP, epIP net.IP, isIPv6 bool) { ctrl := gomock.NewController(t) defer ctrl.Finish() - mockOFClient := ofmock.NewMockClient(ctrl) - mockRouteClient := routemock.NewMockInterface(ctrl) + mockOFClient, mockRouteClient := getMockClients(ctrl) groupAllocator := openflow.NewGroupAllocator(isIPv6) fp := NewFakeProxier(mockRouteClient, mockOFClient, nil, groupAllocator, isIPv6, withProxyAll) - svcPort := 80 - svcPortName := k8sproxy.ServicePortName{ - NamespacedName: makeNamespaceName("ns1", "svc1"), - Port: fmt.Sprint(svcPort), - Protocol: corev1.ProtocolTCP, - } - service := makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *corev1.Service) { - svc.Spec.ClusterIP = svcIP.String() - svc.Spec.Ports = []corev1.ServicePort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: corev1.ProtocolTCP, - }} - }) - makeServiceMap(fp, service) + svc := makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, nil, nil) + makeServiceMap(fp, svc) - epFunc := func(ept *corev1.Endpoints) { - ept.Subsets = []corev1.EndpointSubset{{ - Addresses: []corev1.EndpointAddress{{ - IP: epIP.String(), - }}, - Ports: []corev1.EndpointPort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: corev1.ProtocolTCP, - }}, - }} - } + epSubset := makeTestEndpointSubset(&svcPortName, epIP, int32(svcPort), corev1.ProtocolTCP, false) + ep := makeTestEndpoints(&svcPortName, []corev1.EndpointSubset{*epSubset}) + makeEndpointsMap(fp, ep) bindingProtocol := binding.ProtocolTCP if isIPv6 { bindingProtocol = binding.ProtocolTCPv6 } - ep := makeTestEndpoints(svcPortName.Namespace, svcPortName.Name, epFunc) - makeEndpointsMap(fp, ep) + groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, false) mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.Any()).Times(1) mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, gomock.Any()).Times(1) @@ -750,131 +1065,205 @@ func testClusterIPRemoval(t *testing.T, svcIP net.IP, epIP net.IP, isIPv6 bool) mockOFClient.EXPECT().UninstallServiceFlows(svcIP, uint16(svcPort), bindingProtocol).Times(1) mockOFClient.EXPECT().UninstallEndpointFlows(bindingProtocol, gomock.Any()).Times(1) mockOFClient.EXPECT().UninstallServiceGroup(gomock.Any()).Times(1) + mockRouteClient.EXPECT().DeleteClusterIPRoute(svcIP).Times(1) fp.syncProxyRules() - fp.serviceChanges.OnServiceUpdate(service, nil) + fp.serviceChanges.OnServiceUpdate(svc, nil) fp.endpointsChanges.OnEndpointUpdate(ep, nil) fp.syncProxyRules() } -func TestClusterIPRemovalIPv4(t *testing.T) { - testClusterIPRemoval(t, net.ParseIP("10.20.30.41"), net.ParseIP("10.180.0.1"), false) -} - -func TestClusterIPRemovalIPv6(t *testing.T) { - testClusterIPRemoval(t, net.ParseIP("10:20::41"), net.ParseIP("10:180::1"), true) -} - -func testClusterIPNoEndpoint(t *testing.T, svcIP net.IP, isIPv6 bool) { +func testNodePortRemove(t *testing.T, nodePortAddresses []net.IP, svcIP net.IP, epIP net.IP, isIPv6 bool) { ctrl := gomock.NewController(t) defer ctrl.Finish() - mockOFClient := ofmock.NewMockClient(ctrl) - mockRouteClient := routemock.NewMockInterface(ctrl) + mockOFClient, mockRouteClient := getMockClients(ctrl) groupAllocator := openflow.NewGroupAllocator(isIPv6) - fp := NewFakeProxier(mockRouteClient, mockOFClient, nil, groupAllocator, isIPv6) + fp := NewFakeProxier(mockRouteClient, mockOFClient, nodePortAddresses, groupAllocator, isIPv6, withProxyAll) - svcPort := 80 - svcNodePort := 3001 - svcPortName := k8sproxy.ServicePortName{ - NamespacedName: makeNamespaceName("ns1", "svc1"), - Port: "80", - Protocol: corev1.ProtocolTCP, - } - - makeServiceMap(fp, - makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *corev1.Service) { - svc.Spec.ClusterIP = svcIP.String() - svc.Spec.Ports = []corev1.ServicePort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: corev1.ProtocolTCP, - NodePort: int32(svcNodePort), - }} - }), - ) - makeEndpointsMap(fp) + svc := makeTestNodePortService(&svcPortName, svcIP, + int32(svcPort), + int32(svcNodePort), + corev1.ProtocolTCP, + nil, + corev1.ServiceInternalTrafficPolicyCluster, + corev1.ServiceExternalTrafficPolicyTypeLocal) + makeServiceMap(fp, svc) + + epSubset := makeTestEndpointSubset(&svcPortName, epIP, int32(svcPort), corev1.ProtocolTCP, true) + ep := makeTestEndpoints(&svcPortName, []corev1.EndpointSubset{*epSubset}) + makeEndpointsMap(fp, ep) + + bindingProtocol := binding.ProtocolTCP + vIP := agentconfig.VirtualNodePortDNATIPv4 + if isIPv6 { + bindingProtocol = binding.ProtocolTCPv6 + vIP = agentconfig.VirtualNodePortDNATIPv6 + } + expectedEp := k8sproxy.NewBaseEndpointInfo(epIP.String(), "", "", svcPort, true, true, false, false, nil) + expectedAllEps := []k8sproxy.Endpoint{expectedEp} + + mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, expectedAllEps).Times(1) + groupIDLocal := fp.groupCounter.AllocateIfNotExist(svcPortName, true) + groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, false) + mockOFClient.EXPECT().InstallServiceGroup(groupIDLocal, false, expectedAllEps).Times(1) + mockOFClient.EXPECT().InstallServiceGroup(groupID, false, expectedAllEps).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), true, corev1.ServiceTypeClusterIP).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), true, corev1.ServiceTypeNodePort).Times(1) + mockRouteClient.EXPECT().AddClusterIPRoute(svcIP).Times(1) + mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) + + mockOFClient.EXPECT().UninstallEndpointFlows(bindingProtocol, expectedEp).Times(1) + mockOFClient.EXPECT().UninstallServiceFlows(svcIP, uint16(svcPort), bindingProtocol).Times(1) + mockOFClient.EXPECT().UninstallServiceFlows(vIP, uint16(svcNodePort), bindingProtocol).Times(1) + mockOFClient.EXPECT().UninstallServiceGroup(gomock.Any()).Times(2) + mockRouteClient.EXPECT().DeleteClusterIPRoute(svcIP).Times(1) + mockRouteClient.EXPECT().DeleteNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) + fp.syncProxyRules() + + fp.serviceChanges.OnServiceUpdate(svc, nil) + fp.endpointsChanges.OnEndpointUpdate(ep, nil) + fp.syncProxyRules() +} + +func testLoadBalancerRemove(t *testing.T, nodePortAddresses []net.IP, svcIP net.IP, epIP net.IP, loadBalancerIP net.IP, isIPv6 bool) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockOFClient, mockRouteClient := getMockClients(ctrl) + groupAllocator := openflow.NewGroupAllocator(isIPv6) + fp := NewFakeProxier(mockRouteClient, mockOFClient, nodePortAddresses, groupAllocator, isIPv6, withProxyAll) + + externalTrafficPolicy := corev1.ServiceExternalTrafficPolicyTypeLocal + internalTrafficPolicy := corev1.ServiceInternalTrafficPolicyCluster + + svc := makeTestLoadBalancerService(&svcPortName, svcIP, + []net.IP{loadBalancerIP}, + int32(svcPort), + int32(svcNodePort), + corev1.ProtocolTCP, + nil, + &internalTrafficPolicy, + externalTrafficPolicy) + makeServiceMap(fp, svc) + + epSubset := makeTestEndpointSubset(&svcPortName, epIP, int32(svcPort), corev1.ProtocolTCP, true) + ep := makeTestEndpoints(&svcPortName, []corev1.EndpointSubset{*epSubset}) + makeEndpointsMap(fp, ep) + + bindingProtocol := binding.ProtocolTCP + vIP := agentconfig.VirtualNodePortDNATIPv4 + if isIPv6 { + bindingProtocol = binding.ProtocolTCPv6 + vIP = agentconfig.VirtualNodePortDNATIPv6 + } + expectedEp := k8sproxy.NewBaseEndpointInfo(epIP.String(), "", "", svcPort, true, true, false, false, nil) + expectedAllEps := []k8sproxy.Endpoint{expectedEp} + + mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, expectedAllEps).Times(1) + groupIDLocal := fp.groupCounter.AllocateIfNotExist(svcPortName, true) + groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, false) + mockOFClient.EXPECT().InstallServiceGroup(groupID, false, expectedAllEps).Times(1) + mockOFClient.EXPECT().InstallServiceGroup(groupIDLocal, false, expectedAllEps).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), true, corev1.ServiceTypeClusterIP).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), true, corev1.ServiceTypeNodePort).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), true, corev1.ServiceTypeLoadBalancer).Times(1) + + mockRouteClient.EXPECT().AddClusterIPRoute(svcIP).Times(1) + mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) + mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) + + mockOFClient.EXPECT().UninstallEndpointFlows(bindingProtocol, expectedEp).Times(1) + mockOFClient.EXPECT().UninstallServiceFlows(svcIP, uint16(svcPort), bindingProtocol).Times(1) + mockOFClient.EXPECT().UninstallServiceFlows(vIP, uint16(svcNodePort), bindingProtocol).Times(1) + mockOFClient.EXPECT().UninstallServiceFlows(loadBalancerIP, uint16(svcPort), bindingProtocol).Times(1) + mockOFClient.EXPECT().UninstallServiceGroup(gomock.Any()).Times(2) + mockRouteClient.EXPECT().DeleteClusterIPRoute(svcIP).Times(1) + mockRouteClient.EXPECT().DeleteNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) + mockRouteClient.EXPECT().DeleteLoadBalancer(loadBalancerIP).Times(1) + fp.syncProxyRules() + + fp.serviceChanges.OnServiceUpdate(svc, nil) + fp.endpointsChanges.OnEndpointUpdate(ep, nil) fp.syncProxyRules() } -func TestClusterIPNoEndpointIPv4(t *testing.T) { - testClusterIPNoEndpoint(t, net.ParseIP("10.20.30.41"), false) +func TestClusterIPRemove(t *testing.T) { + t.Run("IPv4", func(t *testing.T) { + testClusterIPRemove(t, svc1IPv4, ep1IPv4, false) + }) + t.Run("IPv6", func(t *testing.T) { + testClusterIPRemove(t, svc1IPv6, ep1IPv6, true) + }) } -func TestClusterIPNoEndpointIPv6(t *testing.T) { - testClusterIPNoEndpoint(t, net.ParseIP("10:20::41"), true) +func TestNodePortRemove(t *testing.T) { + t.Run("IPv4", func(t *testing.T) { + testNodePortRemove(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, false) + }) + t.Run("IPv6", func(t *testing.T) { + testNodePortRemove(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, true) + }) } -func testClusterIPRemoveSamePortEndpoint(t *testing.T, svcIP net.IP, epIP net.IP, isIPv6 bool) { +func TestLoadBalancerRemove(t *testing.T) { + t.Run("IPv4", func(t *testing.T) { + testLoadBalancerRemove(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, loadBalancerIPv4, false) + }) + t.Run("IPv6", func(t *testing.T) { + testLoadBalancerRemove(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, loadBalancerIPv6, true) + }) + +} + +func testClusterIPNoEndpoint(t *testing.T, svcIP net.IP, isIPv6 bool) { ctrl := gomock.NewController(t) defer ctrl.Finish() - mockOFClient := ofmock.NewMockClient(ctrl) - mockRouteClient := routemock.NewMockInterface(ctrl) + mockOFClient, mockRouteClient := getMockClients(ctrl) groupAllocator := openflow.NewGroupAllocator(isIPv6) fp := NewFakeProxier(mockRouteClient, mockOFClient, nil, groupAllocator, isIPv6) - svcPort := 80 - svcPortName := k8sproxy.ServicePortName{ - NamespacedName: makeNamespaceName("ns1", "svc1"), - Port: "80", - Protocol: corev1.ProtocolTCP, - } - svcPortNameUDP := k8sproxy.ServicePortName{ - NamespacedName: makeNamespaceName("ns1", "svc1-udp"), - Port: "80", - Protocol: corev1.ProtocolUDP, - } - makeServiceMap(fp, - makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *corev1.Service) { - svc.Spec.ClusterIP = svcIP.String() - svc.Spec.Ports = []corev1.ServicePort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: corev1.ProtocolTCP, - }} - }), - makeTestService(svcPortName.Namespace, svcPortNameUDP.Name, func(svc *corev1.Service) { - svc.Spec.ClusterIP = svcIP.String() - svc.Spec.Ports = []corev1.ServicePort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: corev1.ProtocolUDP, - }} - }), - ) - - ep := makeTestEndpoints(svcPortName.Namespace, svcPortName.Name, func(ept *corev1.Endpoints) { - ept.Subsets = []corev1.EndpointSubset{{ - Addresses: []corev1.EndpointAddress{{ - IP: epIP.String(), - }}, - Ports: []corev1.EndpointPort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: corev1.ProtocolTCP, - }}, - }} + svc := makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, nil, nil) + makeServiceMap(fp, svc) + makeEndpointsMap(fp) + fp.syncProxyRules() +} + +func TestClusterIPNoEndpoint(t *testing.T) { + t.Run("IPv4", func(t *testing.T) { + testClusterIPNoEndpoint(t, svc1IPv4, false) }) - epUDP := makeTestEndpoints(svcPortName.Namespace, svcPortNameUDP.Name, func(ept *corev1.Endpoints) { - ept.Subsets = []corev1.EndpointSubset{{ - Addresses: []corev1.EndpointAddress{{ - IP: epIP.String(), - }}, - Ports: []corev1.EndpointPort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: corev1.ProtocolUDP, - }}, - }} + t.Run("IPv6", func(t *testing.T) { + testClusterIPNoEndpoint(t, svc1IPv6, true) }) +} + +func testClusterIPRemoveSamePortEndpoint(t *testing.T, svcIP net.IP, epIP net.IP, isIPv6 bool) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockOFClient, mockRouteClient := getMockClients(ctrl) + groupAllocator := openflow.NewGroupAllocator(isIPv6) + fp := NewFakeProxier(mockRouteClient, mockOFClient, nil, groupAllocator, isIPv6) + + svcPortNameTCP := makeSvcPortName("ns", "svc-tcp", strconv.Itoa(svcPort), corev1.ProtocolTCP) + svcPortNameUDP := makeSvcPortName("ns", "svc-udp", strconv.Itoa(svcPort), corev1.ProtocolUDP) + + svcTCP := makeTestClusterIPService(&svcPortNameTCP, svcIP, int32(svcPort), corev1.ProtocolTCP, nil, nil) + svcUDP := makeTestClusterIPService(&svcPortNameUDP, svcIP, int32(svcPort), corev1.ProtocolUDP, nil, nil) + makeServiceMap(fp, svcTCP, svcUDP) + + epSubset := makeTestEndpointSubset(&svcPortNameTCP, epIP, int32(svcPort), corev1.ProtocolTCP, false) + epTCP := makeTestEndpoints(&svcPortNameTCP, []corev1.EndpointSubset{*epSubset}) + epSubset = makeTestEndpointSubset(&svcPortNameUDP, epIP, int32(svcPort), corev1.ProtocolUDP, false) + epUDP := makeTestEndpoints(&svcPortNameUDP, []corev1.EndpointSubset{*epSubset}) + makeEndpointsMap(fp, epTCP, epUDP) + protocolTCP := binding.ProtocolTCP protocolUDP := binding.ProtocolUDP if isIPv6 { protocolTCP = binding.ProtocolTCPv6 protocolUDP = binding.ProtocolUDPv6 } - makeEndpointsMap(fp, ep, epUDP) - groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, false) + groupID := fp.groupCounter.AllocateIfNotExist(svcPortNameTCP, false) groupIDUDP := fp.groupCounter.AllocateIfNotExist(svcPortNameUDP, false) mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.Any()).Times(1) mockOFClient.EXPECT().InstallServiceGroup(groupIDUDP, false, gomock.Any()).Times(2) @@ -889,56 +1278,34 @@ func testClusterIPRemoveSamePortEndpoint(t *testing.T, svcIP net.IP, epIP net.IP fp.syncProxyRules() } -func TestClusterIPRemoveSamePortEndpointIPv4(t *testing.T) { - testClusterIPRemoveSamePortEndpoint(t, net.ParseIP("10.20.30.41"), net.ParseIP("10.180.0.1"), false) -} - -func TestClusterIPRemoveSamePortEndpointIPv6(t *testing.T) { - testClusterIPRemoveSamePortEndpoint(t, net.ParseIP("10:20::41"), net.ParseIP("10:180::1"), true) +func TestClusterIPRemoveSamePortEndpoint(t *testing.T) { + t.Run("IPv4", func(t *testing.T) { + testClusterIPRemoveSamePortEndpoint(t, svc1IPv4, ep1IPv4, false) + }) + t.Run("IPv6", func(t *testing.T) { + testClusterIPRemoveSamePortEndpoint(t, svc1IPv6, ep1IPv6, true) + }) } func testClusterIPRemoveEndpoints(t *testing.T, svcIP net.IP, epIP net.IP, isIPv6 bool) { ctrl := gomock.NewController(t) defer ctrl.Finish() - mockOFClient := ofmock.NewMockClient(ctrl) - mockRouteClient := routemock.NewMockInterface(ctrl) + mockOFClient, mockRouteClient := getMockClients(ctrl) groupAllocator := openflow.NewGroupAllocator(isIPv6) fp := NewFakeProxier(mockRouteClient, mockOFClient, nil, groupAllocator, isIPv6) - svcPort := 80 - svcPortName := k8sproxy.ServicePortName{ - NamespacedName: makeNamespaceName("ns1", "svc1"), - Port: "80", - Protocol: corev1.ProtocolTCP, - } - makeServiceMap(fp, - makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *corev1.Service) { - svc.Spec.ClusterIP = svcIP.String() - svc.Spec.Ports = []corev1.ServicePort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: corev1.ProtocolTCP, - }} - }), - ) - - ep := makeTestEndpoints(svcPortName.Namespace, svcPortName.Name, func(ept *corev1.Endpoints) { - ept.Subsets = []corev1.EndpointSubset{{ - Addresses: []corev1.EndpointAddress{{ - IP: epIP.String(), - }}, - Ports: []corev1.EndpointPort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: corev1.ProtocolTCP, - }}, - }} - }) + svc := makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, nil, nil) + makeServiceMap(fp, svc) + + epSubset := makeTestEndpointSubset(&svcPortName, epIP, int32(svcPort), corev1.ProtocolTCP, false) + ep := makeTestEndpoints(&svcPortName, []corev1.EndpointSubset{*epSubset}) + makeEndpointsMap(fp, ep) + bindingProtocol := binding.ProtocolTCP if isIPv6 { bindingProtocol = binding.ProtocolTCPv6 } - makeEndpointsMap(fp, ep) + groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, false) mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.Any()).Times(2) mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, gomock.Any()).Times(2) @@ -950,63 +1317,44 @@ func testClusterIPRemoveEndpoints(t *testing.T, svcIP net.IP, epIP net.IP, isIPv fp.syncProxyRules() } -func TestClusterIPRemoveEndpointsIPv4(t *testing.T) { - testClusterIPRemoveEndpoints(t, net.ParseIP("10.20.30.41"), net.ParseIP("10.180.0.1"), false) -} - -func TestClusterIPRemoveEndpointsIPv6(t *testing.T) { - testClusterIPRemoveEndpoints(t, net.ParseIP("10:20::41"), net.ParseIP("10:180::1"), true) +func TestClusterIPRemoveEndpoints(t *testing.T) { + t.Run("IPv4", func(t *testing.T) { + testClusterIPRemoveEndpoints(t, svc1IPv4, ep1IPv4, false) + }) + t.Run("IPv6", func(t *testing.T) { + testClusterIPRemoveEndpoints(t, svc1IPv6, ep1IPv6, true) + }) } -func testSessionAffinity(t *testing.T, svcExternalIPs net.IP, svcIP net.IP, epIP net.IP, affinitySeconds int32, isIPv6 bool) { +func testSessionAffinity(t *testing.T, svcIP net.IP, epIP net.IP, affinitySeconds int32, isIPv6 bool) { ctrl := gomock.NewController(t) defer ctrl.Finish() - mockOFClient := ofmock.NewMockClient(ctrl) - mockRouteClient := routemock.NewMockInterface(ctrl) + mockOFClient, mockRouteClient := getMockClients(ctrl) groupAllocator := openflow.NewGroupAllocator(isIPv6) fp := NewFakeProxier(mockRouteClient, mockOFClient, nil, groupAllocator, isIPv6) - svcPort := 80 - svcNodePort := 3001 - svcPortName := k8sproxy.ServicePortName{ - NamespacedName: makeNamespaceName("ns1", "svc1"), - Port: "80", - Protocol: corev1.ProtocolTCP, - } + svc := makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *corev1.Service) { + svc.Spec.Type = corev1.ServiceTypeNodePort + svc.Spec.ClusterIP = svcIP.String() + svc.Spec.SessionAffinity = corev1.ServiceAffinityClientIP + svc.Spec.SessionAffinityConfig = &corev1.SessionAffinityConfig{ + ClientIP: &corev1.ClientIPConfig{ + TimeoutSeconds: &affinitySeconds, + }, + } + svc.Spec.Ports = []corev1.ServicePort{{ + Name: svcPortName.Port, + Port: int32(svcPort), + Protocol: corev1.ProtocolTCP, + NodePort: int32(svcNodePort), + }} + }) + makeServiceMap(fp, svc) + + epSubset := makeTestEndpointSubset(&svcPortName, epIP, int32(svcPort), corev1.ProtocolTCP, false) + ep := makeTestEndpoints(&svcPortName, []corev1.EndpointSubset{*epSubset}) + makeEndpointsMap(fp, ep) - makeServiceMap(fp, - makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *corev1.Service) { - svc.Spec.Type = corev1.ServiceTypeNodePort - svc.Spec.ClusterIP = svcIP.String() - svc.Spec.ExternalIPs = []string{svcExternalIPs.String()} - svc.Spec.SessionAffinity = corev1.ServiceAffinityClientIP - svc.Spec.SessionAffinityConfig = &corev1.SessionAffinityConfig{ - ClientIP: &corev1.ClientIPConfig{ - TimeoutSeconds: &affinitySeconds, - }, - } - svc.Spec.Ports = []corev1.ServicePort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: corev1.ProtocolTCP, - NodePort: int32(svcNodePort), - }} - }), - ) - makeEndpointsMap(fp, - makeTestEndpoints(svcPortName.Namespace, svcPortName.Name, func(ept *corev1.Endpoints) { - ept.Subsets = []corev1.EndpointSubset{{ - Addresses: []corev1.EndpointAddress{{ - IP: epIP.String(), - }}, - Ports: []corev1.EndpointPort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: corev1.ProtocolTCP, - }}, - }} - }), - ) bindingProtocol := binding.ProtocolTCP if isIPv6 { bindingProtocol = binding.ProtocolTCPv6 @@ -1025,202 +1373,822 @@ func testSessionAffinity(t *testing.T, svcExternalIPs net.IP, svcIP net.IP, epIP fp.syncProxyRules() } -func TestSessionAffinityIPv4(t *testing.T) { - affinitySeconds := corev1.DefaultClientIPServiceAffinitySeconds - testSessionAffinity(t, net.ParseIP("50.60.70.81"), net.ParseIP("10.20.30.41"), net.ParseIP("10.180.0.1"), affinitySeconds, false) -} - -func TestSessionAffinityIPv6(t *testing.T) { +func TestSessionAffinity(t *testing.T) { affinitySeconds := corev1.DefaultClientIPServiceAffinitySeconds - testSessionAffinity(t, net.ParseIP("5060:70::81"), net.ParseIP("10:20::41"), net.ParseIP("10:180::1"), affinitySeconds, true) + t.Run("IPv4", func(t *testing.T) { + testSessionAffinity(t, svc1IPv4, ep1IPv4, affinitySeconds, false) + }) + t.Run("IPv6", func(t *testing.T) { + testSessionAffinity(t, svc1IPv6, ep1IPv6, affinitySeconds, true) + }) } func TestSessionAffinityOverflow(t *testing.T) { // Ensure that the SessionAffinity timeout is truncated to the max supported value, instead // of wrapping around. affinitySeconds := int32(math.MaxUint16 + 10) - testSessionAffinity(t, net.ParseIP("50.60.70.81"), net.ParseIP("10.20.30.41"), net.ParseIP("10.180.0.1"), affinitySeconds, false) + testSessionAffinity(t, svc1IPv4, ep1IPv4, affinitySeconds, false) } func testSessionAffinityNoEndpoint(t *testing.T, svcExternalIPs net.IP, svcIP net.IP, isIPv6 bool) { ctrl := gomock.NewController(t) defer ctrl.Finish() - mockOFClient := ofmock.NewMockClient(ctrl) - mockRouteClient := routemock.NewMockInterface(ctrl) + mockOFClient, mockRouteClient := getMockClients(ctrl) groupAllocator := openflow.NewGroupAllocator(isIPv6) fp := NewFakeProxier(mockRouteClient, mockOFClient, nil, groupAllocator, isIPv6) - svcPort := 80 - svcNodePort := 3001 - svcPortName := k8sproxy.ServicePortName{ - NamespacedName: makeNamespaceName("ns1", "svc1"), - Port: "80", - Protocol: corev1.ProtocolTCP, - } timeoutSeconds := corev1.DefaultClientIPServiceAffinitySeconds - makeServiceMap(fp, - makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *corev1.Service) { - svc.Spec.Type = corev1.ServiceTypeNodePort - svc.Spec.ClusterIP = svcIP.String() - svc.Spec.ExternalIPs = []string{svcExternalIPs.String()} - svc.Spec.SessionAffinity = corev1.ServiceAffinityClientIP - svc.Spec.SessionAffinityConfig = &corev1.SessionAffinityConfig{ - ClientIP: &corev1.ClientIPConfig{ - TimeoutSeconds: &timeoutSeconds, - }, - } - svc.Spec.Ports = []corev1.ServicePort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: corev1.ProtocolTCP, - NodePort: int32(svcNodePort), - }} - }), - ) + svc := makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *corev1.Service) { + svc.Spec.Type = corev1.ServiceTypeNodePort + svc.Spec.ClusterIP = svcIP.String() + svc.Spec.ExternalIPs = []string{svcExternalIPs.String()} + svc.Spec.SessionAffinity = corev1.ServiceAffinityClientIP + svc.Spec.SessionAffinityConfig = &corev1.SessionAffinityConfig{ + ClientIP: &corev1.ClientIPConfig{ + TimeoutSeconds: &timeoutSeconds, + }, + } + svc.Spec.Ports = []corev1.ServicePort{{ + Name: svcPortName.Port, + Port: int32(svcPort), + Protocol: corev1.ProtocolTCP, + NodePort: int32(svcNodePort), + }} + }) + makeServiceMap(fp, svc) makeEndpointsMap(fp) fp.syncProxyRules() } -func TestSessionAffinityNoEndpointIPv4(t *testing.T) { - testSessionAffinityNoEndpoint(t, net.ParseIP("50.60.70.81"), net.ParseIP("10.20.30.41"), false) +func TestSessionAffinityNoEndpoint(t *testing.T) { + t.Run("IPv4", func(t *testing.T) { + testSessionAffinityNoEndpoint(t, net.ParseIP("50.60.70.81"), svc1IPv4, false) + }) + t.Run("IPv6", func(t *testing.T) { + testSessionAffinityNoEndpoint(t, net.ParseIP("5060:70::81"), svc1IPv6, true) + }) } -func TestSessionAffinityIPv6NoEndpoint(t *testing.T) { - testSessionAffinityNoEndpoint(t, net.ParseIP("5060:70::81"), net.ParseIP("10:20::41"), true) +func testServiceClusterIPUpdate(t *testing.T, + nodePortAddresses []net.IP, + svcIP net.IP, + updatedSvcIP net.IP, + loadBalancerIP net.IP, + epIP net.IP, + svcType corev1.ServiceType, + isIPv6 bool) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockOFClient, mockRouteClient := getMockClients(ctrl) + groupAllocator := openflow.NewGroupAllocator(isIPv6) + fp := NewFakeProxier(mockRouteClient, mockOFClient, nodePortAddresses, groupAllocator, isIPv6, withProxyAll) + + var svc, updatedSvc *corev1.Service + switch svcType { + case corev1.ServiceTypeClusterIP: + svc = makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, nil, nil) + updatedSvc = makeTestClusterIPService(&svcPortName, updatedSvcIP, int32(svcPort), corev1.ProtocolTCP, nil, nil) + case corev1.ServiceTypeNodePort: + svc = makeTestNodePortService(&svcPortName, svcIP, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestNodePortService(&svcPortName, updatedSvcIP, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) + case corev1.ServiceTypeLoadBalancer: + svc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestLoadBalancerService(&svcPortName, updatedSvcIP, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + } + makeServiceMap(fp, svc) + + epSubset := makeTestEndpointSubset(&svcPortName, epIP, int32(svcPort), corev1.ProtocolTCP, false) + eps := makeTestEndpoints(&svcPortName, []corev1.EndpointSubset{*epSubset}) + makeEndpointsMap(fp, eps) + + expectedEps := []k8sproxy.Endpoint{k8sproxy.NewBaseEndpointInfo(epIP.String(), "", "", svcPort, false, true, false, false, nil)} + + bindingProtocol := binding.ProtocolTCP + vIP := agentconfig.VirtualNodePortDNATIPv4 + if isIPv6 { + bindingProtocol = binding.ProtocolTCPv6 + vIP = agentconfig.VirtualNodePortDNATIPv6 + } + + groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, false) + mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, expectedEps).Times(1) + mockOFClient.EXPECT().InstallServiceGroup(groupID, false, expectedEps).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP).Times(1) + mockRouteClient.EXPECT().AddClusterIPRoute(svcIP).Times(1) + + s1 := mockOFClient.EXPECT().UninstallServiceFlows(svcIP, uint16(svcPort), bindingProtocol).Times(1) + s2 := mockOFClient.EXPECT().InstallServiceFlows(groupID, updatedSvcIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP).Times(1) + s2.After(s1) + mockRouteClient.EXPECT().DeleteClusterIPRoute(svcIP).Times(1) + mockRouteClient.EXPECT().AddClusterIPRoute(updatedSvcIP).Times(1) + + if svcType == corev1.ServiceTypeNodePort || svcType == corev1.ServiceTypeLoadBalancer { + mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), false, corev1.ServiceTypeNodePort).Times(1) + mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) + + mockOFClient.EXPECT().UninstallServiceFlows(vIP, uint16(svcNodePort), bindingProtocol).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), false, corev1.ServiceTypeNodePort).Times(1) + mockRouteClient.EXPECT().DeleteNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) + mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) + } + if svcType == corev1.ServiceTypeLoadBalancer { + mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeLoadBalancer).Times(1) + mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) + + mockOFClient.EXPECT().UninstallServiceFlows(loadBalancerIP, uint16(svcPort), bindingProtocol).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeLoadBalancer).Times(1) + mockRouteClient.EXPECT().DeleteLoadBalancer(loadBalancerIP).Times(1) + mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) + } + + fp.syncProxyRules() + fp.serviceChanges.OnServiceUpdate(svc, updatedSvc) + fp.syncProxyRules() } -func testPortChange(t *testing.T, svcIP net.IP, epIP net.IP, isIPv6 bool) { +func TestServiceClusterIPUpdate(t *testing.T) { + t.Run("IPv4", func(t *testing.T) { + t.Run("ClusterIP", func(t *testing.T) { + testServiceClusterIPUpdate(t, nil, svc1IPv4, svc2IPv4, nil, ep1IPv4, corev1.ServiceTypeClusterIP, false) + }) + t.Run("NodePort", func(t *testing.T) { + testServiceClusterIPUpdate(t, nodePortAddressesIPv4, svc1IPv4, svc2IPv4, nil, ep1IPv4, corev1.ServiceTypeNodePort, false) + }) + t.Run("LoadBalancer", func(t *testing.T) { + testServiceClusterIPUpdate(t, nodePortAddressesIPv4, svc1IPv4, svc2IPv4, loadBalancerIPv4, ep1IPv4, corev1.ServiceTypeLoadBalancer, false) + }) + }) + t.Run("IPv6", func(t *testing.T) { + t.Run("ClusterIP", func(t *testing.T) { + testServiceClusterIPUpdate(t, nil, svc1IPv6, svc2IPv6, nil, ep1IPv6, corev1.ServiceTypeClusterIP, true) + }) + t.Run("NodePort", func(t *testing.T) { + testServiceClusterIPUpdate(t, nodePortAddressesIPv6, svc1IPv6, svc2IPv6, nil, ep1IPv6, corev1.ServiceTypeNodePort, true) + }) + t.Run("LoadBalancer", func(t *testing.T) { + testServiceClusterIPUpdate(t, nodePortAddressesIPv6, svc1IPv6, svc2IPv6, loadBalancerIPv6, ep1IPv6, corev1.ServiceTypeLoadBalancer, true) + }) + }) +} + +func testServicePortUpdate(t *testing.T, + nodePortAddresses []net.IP, + svcIP net.IP, + loadBalancerIP net.IP, + epIP net.IP, + svcType corev1.ServiceType, + isIPv6 bool) { ctrl := gomock.NewController(t) defer ctrl.Finish() - mockOFClient := ofmock.NewMockClient(ctrl) - mockRouteClient := routemock.NewMockInterface(ctrl) + mockOFClient, mockRouteClient := getMockClients(ctrl) groupAllocator := openflow.NewGroupAllocator(isIPv6) - fp := NewFakeProxier(mockRouteClient, mockOFClient, nil, groupAllocator, isIPv6) + fp := NewFakeProxier(mockRouteClient, mockOFClient, nodePortAddresses, groupAllocator, isIPv6, withProxyAll) - svcPort1 := 80 - svcPort2 := 8080 - svcPortName := k8sproxy.ServicePortName{ - NamespacedName: makeNamespaceName("ns1", "svc1"), - Port: "80", - Protocol: corev1.ProtocolTCP, + var svc, updatedSvc *corev1.Service + switch svcType { + case corev1.ServiceTypeClusterIP: + svc = makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, nil, nil) + updatedSvc = makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort+1), corev1.ProtocolTCP, nil, nil) + case corev1.ServiceTypeNodePort: + svc = makeTestNodePortService(&svcPortName, svcIP, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestNodePortService(&svcPortName, svcIP, int32(svcPort+1), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) + case corev1.ServiceTypeLoadBalancer: + svc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{loadBalancerIP}, int32(svcPort+1), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) } - service := makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *corev1.Service) { - svc.Spec.ClusterIP = svcIP.String() - svc.Spec.Ports = []corev1.ServicePort{{ - Name: svcPortName.Port, - Port: int32(svcPort1), - TargetPort: intstr.FromInt(80), - Protocol: corev1.ProtocolTCP, - }} + makeServiceMap(fp, svc) + + epSubset := makeTestEndpointSubset(&svcPortName, epIP, int32(svcPort), corev1.ProtocolTCP, false) + eps := makeTestEndpoints(&svcPortName, []corev1.EndpointSubset{*epSubset}) + makeEndpointsMap(fp, eps) + + expectedEps := []k8sproxy.Endpoint{k8sproxy.NewBaseEndpointInfo(epIP.String(), "", "", svcPort, false, true, false, false, nil)} + + bindingProtocol := binding.ProtocolTCP + vIP := agentconfig.VirtualNodePortDNATIPv4 + if isIPv6 { + bindingProtocol = binding.ProtocolTCPv6 + vIP = agentconfig.VirtualNodePortDNATIPv6 + } + + groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, false) + mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, expectedEps).Times(1) + mockOFClient.EXPECT().InstallServiceGroup(groupID, false, expectedEps).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP).Times(1) + mockRouteClient.EXPECT().AddClusterIPRoute(svcIP).Times(1) + + s1 := mockOFClient.EXPECT().UninstallServiceFlows(svcIP, uint16(svcPort), bindingProtocol).Times(1) + s2 := mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort+1), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP).Times(1) + s2.After(s1) + + mockRouteClient.EXPECT().DeleteClusterIPRoute(svcIP).Times(1) + mockRouteClient.EXPECT().AddClusterIPRoute(svcIP).Times(1) + + if svcType == corev1.ServiceTypeNodePort || svcType == corev1.ServiceTypeLoadBalancer { + mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), false, corev1.ServiceTypeNodePort).Times(1) + mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) + + mockOFClient.EXPECT().UninstallServiceFlows(vIP, uint16(svcNodePort), bindingProtocol).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), false, corev1.ServiceTypeNodePort).Times(1) + mockRouteClient.EXPECT().DeleteNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) + mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) + } + if svcType == corev1.ServiceTypeLoadBalancer { + mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeLoadBalancer).Times(1) + mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) + + s1 = mockOFClient.EXPECT().UninstallServiceFlows(loadBalancerIP, uint16(svcPort), bindingProtocol) + s2 = mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort+1), bindingProtocol, uint16(0), false, corev1.ServiceTypeLoadBalancer).Times(1) + s2.After(s1) + + mockRouteClient.EXPECT().DeleteLoadBalancer(loadBalancerIP).Times(1) + mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) + } + fp.syncProxyRules() + fp.serviceChanges.OnServiceUpdate(svc, updatedSvc) + fp.syncProxyRules() +} + +func TestServicePortUpdate(t *testing.T) { + t.Run("IPv4", func(t *testing.T) { + t.Run("ClusterIP", func(t *testing.T) { + testServicePortUpdate(t, nil, svc1IPv4, nil, ep1IPv4, corev1.ServiceTypeClusterIP, false) + }) + t.Run("NodePort", func(t *testing.T) { + testServicePortUpdate(t, nodePortAddressesIPv4, svc1IPv4, nil, ep1IPv4, corev1.ServiceTypeNodePort, false) + }) + t.Run("LoadBalancer", func(t *testing.T) { + testServicePortUpdate(t, nodePortAddressesIPv4, svc1IPv4, loadBalancerIPv4, ep1IPv4, corev1.ServiceTypeLoadBalancer, false) + }) + }) + t.Run("IPv6", func(t *testing.T) { + t.Run("ClusterIP", func(t *testing.T) { + testServicePortUpdate(t, nil, svc1IPv6, nil, ep1IPv6, corev1.ServiceTypeClusterIP, true) + }) + t.Run("NodePort", func(t *testing.T) { + testServicePortUpdate(t, nodePortAddressesIPv6, svc1IPv6, nil, ep1IPv6, corev1.ServiceTypeNodePort, true) + }) + t.Run("LoadBalancer", func(t *testing.T) { + testServicePortUpdate(t, nodePortAddressesIPv6, svc1IPv6, loadBalancerIPv6, ep1IPv6, corev1.ServiceTypeLoadBalancer, true) + }) }) - makeServiceMap(fp, service) +} - epFunc := func(ept *corev1.Endpoints) { - ept.Subsets = []corev1.EndpointSubset{{ - Addresses: []corev1.EndpointAddress{{ - IP: epIP.String(), - }}, - Ports: []corev1.EndpointPort{{ - Name: svcPortName.Port, - Port: int32(80), - Protocol: corev1.ProtocolTCP, - }}, - }} +func testServiceNodePortUpdate(t *testing.T, + nodePortAddresses []net.IP, + svcIP net.IP, + loadBalancerIP net.IP, + epIP net.IP, + svcType corev1.ServiceType, + isIPv6 bool) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockOFClient, mockRouteClient := getMockClients(ctrl) + groupAllocator := openflow.NewGroupAllocator(isIPv6) + fp := NewFakeProxier(mockRouteClient, mockOFClient, nodePortAddresses, groupAllocator, isIPv6, withProxyAll) + + var svc, updatedSvc *corev1.Service + switch svcType { + case corev1.ServiceTypeNodePort: + svc = makeTestNodePortService(&svcPortName, svcIP, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestNodePortService(&svcPortName, svcIP, int32(svcPort), int32(svcNodePort+1), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) + case corev1.ServiceTypeLoadBalancer: + svc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort+1), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) } + makeServiceMap(fp, svc) + + epSubset := makeTestEndpointSubset(&svcPortName, epIP, int32(svcPort), corev1.ProtocolTCP, false) + eps := makeTestEndpoints(&svcPortName, []corev1.EndpointSubset{*epSubset}) + makeEndpointsMap(fp, eps) + + expectedEps := []k8sproxy.Endpoint{k8sproxy.NewBaseEndpointInfo(epIP.String(), "", "", svcPort, false, true, false, false, nil)} bindingProtocol := binding.ProtocolTCP + vIP := agentconfig.VirtualNodePortDNATIPv4 if isIPv6 { bindingProtocol = binding.ProtocolTCPv6 + vIP = agentconfig.VirtualNodePortDNATIPv6 } - ep := makeTestEndpoints(svcPortName.Namespace, svcPortName.Name, epFunc) - makeEndpointsMap(fp, ep) + groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, false) - mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.Any()).Times(1) - mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, gomock.Any()).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort1), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP) - mockOFClient.EXPECT().UninstallServiceFlows(svcIP, uint16(svcPort1), bindingProtocol) - mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort2), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP) + mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, expectedEps).Times(1) + mockOFClient.EXPECT().InstallServiceGroup(groupID, false, expectedEps).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP).Times(1) + mockRouteClient.EXPECT().AddClusterIPRoute(svcIP).Times(1) + + mockOFClient.EXPECT().UninstallServiceFlows(svcIP, uint16(svcPort), bindingProtocol).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP).Times(1) + mockRouteClient.EXPECT().DeleteClusterIPRoute(svcIP).Times(1) + mockRouteClient.EXPECT().AddClusterIPRoute(svcIP).Times(1) + + if svcType == corev1.ServiceTypeNodePort || svcType == corev1.ServiceTypeLoadBalancer { + mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), false, corev1.ServiceTypeNodePort).Times(1) + mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) + s1 := mockOFClient.EXPECT().UninstallServiceFlows(vIP, uint16(svcNodePort), bindingProtocol) + mockRouteClient.EXPECT().DeleteNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) + s2 := mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort+1), bindingProtocol, uint16(0), false, corev1.ServiceTypeNodePort).Times(1) + mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort+1), bindingProtocol).Times(1) + s2.After(s1) + } + if svcType == corev1.ServiceTypeLoadBalancer { + mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeLoadBalancer).Times(1) + mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) + + mockOFClient.EXPECT().UninstallServiceFlows(loadBalancerIP, uint16(svcPort), bindingProtocol) + mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeLoadBalancer).Times(1) + mockRouteClient.EXPECT().DeleteLoadBalancer(loadBalancerIP).Times(1) + mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) + } + + fp.syncProxyRules() + fp.serviceChanges.OnServiceUpdate(svc, updatedSvc) fp.syncProxyRules() +} - serviceNew := makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *corev1.Service) { - svc.Spec.ClusterIP = svcIP.String() - svc.Spec.Ports = []corev1.ServicePort{{ - Name: svcPortName.Port, - Port: int32(svcPort2), - Protocol: corev1.ProtocolTCP, - }} +func TestServiceNodePortUpdate(t *testing.T) { + t.Run("IPv4", func(t *testing.T) { + t.Run("NodePort", func(t *testing.T) { + testServiceNodePortUpdate(t, nodePortAddressesIPv4, svc1IPv4, nil, ep1IPv4, corev1.ServiceTypeNodePort, false) + }) + t.Run("LoadBalancer", func(t *testing.T) { + testServiceNodePortUpdate(t, nodePortAddressesIPv4, svc1IPv4, loadBalancerIPv4, ep1IPv4, corev1.ServiceTypeLoadBalancer, false) + }) }) + t.Run("IPv6", func(t *testing.T) { + t.Run("NodePort", func(t *testing.T) { + testServiceNodePortUpdate(t, nodePortAddressesIPv6, svc1IPv6, nil, ep1IPv6, corev1.ServiceTypeNodePort, true) + }) + t.Run("LoadBalancer", func(t *testing.T) { + testServiceNodePortUpdate(t, nodePortAddressesIPv6, svc1IPv6, loadBalancerIPv6, ep1IPv6, corev1.ServiceTypeLoadBalancer, true) + }) + }) +} + +func testServiceExternalTrafficPolicyUpdate(t *testing.T, + nodePortAddresses []net.IP, + svcIP net.IP, + loadBalancerIP net.IP, + ep1IP net.IP, + ep2IP net.IP, + svcType corev1.ServiceType, + isIPv6 bool) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockOFClient, mockRouteClient := getMockClients(ctrl) + groupAllocator := openflow.NewGroupAllocator(isIPv6) + fp := NewFakeProxier(mockRouteClient, mockOFClient, nodePortAddresses, groupAllocator, isIPv6, withProxyAll) - fp.serviceChanges.OnServiceUpdate(service, serviceNew) + var svc, updatedSvc *corev1.Service + switch svcType { + case corev1.ServiceTypeNodePort: + svc = makeTestNodePortService(&svcPortName, svcIP, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestNodePortService(&svcPortName, svcIP, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeLocal) + case corev1.ServiceTypeLoadBalancer: + svc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeLocal) + } + makeServiceMap(fp, svc) + + remoteEpSubset := makeTestEndpointSubset(&svcPortName, ep1IP, int32(svcPort), corev1.ProtocolTCP, false) + localEpSubset := makeTestEndpointSubset(&svcPortName, ep2IP, int32(svcPort), corev1.ProtocolTCP, true) + eps := makeTestEndpoints(&svcPortName, []corev1.EndpointSubset{*remoteEpSubset, *localEpSubset}) + makeEndpointsMap(fp, eps) + + expectedLocalEps := []k8sproxy.Endpoint{k8sproxy.NewBaseEndpointInfo(ep2IP.String(), "", "", svcPort, true, true, false, false, nil)} + expectedAllEps := append(expectedLocalEps, k8sproxy.NewBaseEndpointInfo(ep1IP.String(), "", "", svcPort, false, true, false, false, nil)) + + bindingProtocol := binding.ProtocolTCP + vIP := agentconfig.VirtualNodePortDNATIPv4 + if isIPv6 { + bindingProtocol = binding.ProtocolTCPv6 + vIP = agentconfig.VirtualNodePortDNATIPv6 + } + + groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, false) + mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, gomock.InAnyOrder(expectedAllEps)).Times(1) + mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.InAnyOrder(expectedAllEps)).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP).Times(1) + mockRouteClient.EXPECT().AddClusterIPRoute(svcIP).Times(1) + + if svcType == corev1.ServiceTypeNodePort || svcType == corev1.ServiceTypeLoadBalancer { + mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), false, corev1.ServiceTypeNodePort).Times(1) + mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) + } + if svcType == corev1.ServiceTypeLoadBalancer { + mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeLoadBalancer).Times(1) + mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) + } fp.syncProxyRules() + + fp.serviceChanges.OnServiceUpdate(svc, updatedSvc) + groupIDLocal := fp.groupCounter.AllocateIfNotExist(svcPortName, true) + + mockOFClient.EXPECT().UninstallServiceFlows(svcIP, uint16(svcPort), bindingProtocol).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), true, corev1.ServiceTypeClusterIP).Times(1) + mockRouteClient.EXPECT().DeleteClusterIPRoute(svcIP).Times(1) + mockRouteClient.EXPECT().AddClusterIPRoute(svcIP).Times(1) + + if svcType == corev1.ServiceTypeNodePort || svcType == corev1.ServiceTypeLoadBalancer { + mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, gomock.InAnyOrder(expectedAllEps)).Times(1) + mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.InAnyOrder(expectedAllEps)).Times(1) + mockOFClient.EXPECT().InstallServiceGroup(groupIDLocal, false, expectedLocalEps).Times(1) + + s1 := mockOFClient.EXPECT().UninstallServiceFlows(vIP, uint16(svcNodePort), bindingProtocol).Times(1) + s2 := mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), true, corev1.ServiceTypeNodePort).Times(1) + s2.After(s1) + + mockRouteClient.EXPECT().DeleteNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) + mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) + } + if svcType == corev1.ServiceTypeLoadBalancer { + s1 := mockOFClient.EXPECT().UninstallServiceFlows(loadBalancerIP, uint16(svcPort), bindingProtocol).Times(1) + s2 := mockOFClient.EXPECT().InstallServiceFlows(groupIDLocal, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), true, corev1.ServiceTypeLoadBalancer).Times(1) + s2.After(s1) + + mockRouteClient.EXPECT().DeleteLoadBalancer(loadBalancerIP).Times(1) + mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) + } + fp.syncProxyRules() +} + +func TestServiceExternalTrafficPolicyUpdate(t *testing.T) { + t.Run("IPv4", func(t *testing.T) { + t.Run("NodePort", func(t *testing.T) { + testServiceExternalTrafficPolicyUpdate(t, nodePortAddressesIPv4, svc1IPv4, nil, ep1IPv4, ep2IPv4, corev1.ServiceTypeNodePort, false) + }) + t.Run("LoadBalancer", func(t *testing.T) { + testServiceExternalTrafficPolicyUpdate(t, nodePortAddressesIPv4, svc1IPv4, loadBalancerIPv4, ep1IPv4, ep2IPv4, corev1.ServiceTypeLoadBalancer, false) + }) + }) + t.Run("IPv6", func(t *testing.T) { + t.Run("NodePort", func(t *testing.T) { + testServiceExternalTrafficPolicyUpdate(t, nodePortAddressesIPv6, svc1IPv6, nil, ep1IPv6, ep2IPv6, corev1.ServiceTypeNodePort, true) + }) + t.Run("LoadBalancer", func(t *testing.T) { + testServiceExternalTrafficPolicyUpdate(t, nodePortAddressesIPv6, svc1IPv6, loadBalancerIPv6, ep1IPv6, ep2IPv6, corev1.ServiceTypeLoadBalancer, true) + }) + }) } -func TestPortChangeIPv4(t *testing.T) { - testPortChange(t, net.ParseIP("10.20.30.41"), net.ParseIP("10.180.0.1"), false) +func testServiceInternalTrafficPolicyUpdate(t *testing.T, + svcIP net.IP, + ep1IP net.IP, + ep2IP net.IP, + isIPv6 bool) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockOFClient, mockRouteClient := getMockClients(ctrl) + groupAllocator := openflow.NewGroupAllocator(isIPv6) + fp := NewFakeProxier(mockRouteClient, mockOFClient, nil, groupAllocator, isIPv6, withProxyAll) + + internalTrafficPolicyCluster := corev1.ServiceInternalTrafficPolicyCluster + internalTrafficPolicyLocal := corev1.ServiceInternalTrafficPolicyLocal + + svc := makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, nil, &internalTrafficPolicyCluster) + updatedSvc := makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, nil, &internalTrafficPolicyLocal) + makeServiceMap(fp, svc) + + remoteEpSubset := makeTestEndpointSubset(&svcPortName, ep1IP, int32(svcPort), corev1.ProtocolTCP, false) + localEpSubset := makeTestEndpointSubset(&svcPortName, ep2IP, int32(svcPort), corev1.ProtocolTCP, true) + eps := makeTestEndpoints(&svcPortName, []corev1.EndpointSubset{*remoteEpSubset, *localEpSubset}) + makeEndpointsMap(fp, eps) + + expectedLocalEps := []k8sproxy.Endpoint{k8sproxy.NewBaseEndpointInfo(ep2IP.String(), "", "", svcPort, true, true, false, false, nil)} + expectedAllEps := append(expectedLocalEps, k8sproxy.NewBaseEndpointInfo(ep1IP.String(), "", "", svcPort, false, true, false, false, nil)) + + bindingProtocol := binding.ProtocolTCP + if isIPv6 { + bindingProtocol = binding.ProtocolTCPv6 + } + + groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, false) + mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, gomock.InAnyOrder(expectedAllEps)).Times(1) + mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.InAnyOrder(expectedAllEps)).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP).Times(1) + mockRouteClient.EXPECT().AddClusterIPRoute(svcIP).Times(1) + fp.syncProxyRules() + + fp.serviceChanges.OnServiceUpdate(svc, updatedSvc) + groupIDLocal := fp.groupCounter.AllocateIfNotExist(svcPortName, true) + + mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, gomock.InAnyOrder(expectedLocalEps)).Times(1) + mockOFClient.EXPECT().InstallServiceGroup(groupIDLocal, false, expectedLocalEps).Times(1) + fp.syncProxyRules() } -func TestPortChangeIPv6(t *testing.T) { - testPortChange(t, net.ParseIP("10:20::41"), net.ParseIP("10:180::1"), true) +func TestServiceInternalTrafficPolicyUpdate(t *testing.T) { + t.Run("IPv4", func(t *testing.T) { + t.Run("ClusterIP", func(t *testing.T) { + testServiceInternalTrafficPolicyUpdate(t, svc1IPv4, ep1IPv4, ep2IPv4, false) + }) + }) + t.Run("IPv6", func(t *testing.T) { + t.Run("ClusterIP", func(t *testing.T) { + testServiceInternalTrafficPolicyUpdate(t, svc1IPv6, ep1IPv6, ep2IPv6, true) + }) + }) } -func TestServicesWithSameEndpoints(t *testing.T) { +func testServiceIngressIPsUpdate(t *testing.T, + nodePortAddresses []net.IP, + svcIP net.IP, + epIP net.IP, + loadBalancerIPs []net.IP, + updatedLoadBalancerIPs []net.IP, + isIPv6 bool) { ctrl := gomock.NewController(t) defer ctrl.Finish() - mockOFClient := ofmock.NewMockClient(ctrl) - mockRouteClient := routemock.NewMockInterface(ctrl) - groupAllocator := openflow.NewGroupAllocator(false) - fp := NewFakeProxier(mockRouteClient, mockOFClient, nil, groupAllocator, false) - epIP := net.ParseIP("10.50.60.71") - svcIP1 := net.ParseIP("10.180.30.41") - svcIP2 := net.ParseIP("10.180.30.42") - svcPort := 80 - svcPortName1 := k8sproxy.ServicePortName{ - NamespacedName: makeNamespaceName("ns1", "svc1"), - Port: "80", - Protocol: corev1.ProtocolTCP, - } - svcPortName2 := k8sproxy.ServicePortName{ - NamespacedName: makeNamespaceName("ns1", "svc2"), - Port: "80", - Protocol: corev1.ProtocolTCP, - } - svcMapFactory := func(svcPortName k8sproxy.ServicePortName, svcIP string) *corev1.Service { - svc := makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *corev1.Service) { - svc.Spec.ClusterIP = svcIP - svc.Spec.Ports = []corev1.ServicePort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: corev1.ProtocolTCP, - }} + mockOFClient, mockRouteClient := getMockClients(ctrl) + groupAllocator := openflow.NewGroupAllocator(isIPv6) + fp := NewFakeProxier(mockRouteClient, mockOFClient, nodePortAddresses, groupAllocator, isIPv6, withProxyAll) + + var loadBalancerIPStrs, updatedLoadBalancerIPStrs []string + for _, ip := range loadBalancerIPs { + loadBalancerIPStrs = append(loadBalancerIPStrs, ip.String()) + } + for _, ip := range updatedLoadBalancerIPs { + updatedLoadBalancerIPStrs = append(updatedLoadBalancerIPStrs, ip.String()) + } + + svc := makeTestLoadBalancerService(&svcPortName, svcIP, loadBalancerIPs, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc := makeTestLoadBalancerService(&svcPortName, svcIP, updatedLoadBalancerIPs, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + makeServiceMap(fp, svc) + + epSubset := makeTestEndpointSubset(&svcPortName, epIP, int32(svcPort), corev1.ProtocolTCP, false) + eps := makeTestEndpoints(&svcPortName, []corev1.EndpointSubset{*epSubset}) + makeEndpointsMap(fp, eps) + + expectedEps := []k8sproxy.Endpoint{k8sproxy.NewBaseEndpointInfo(epIP.String(), "", "", svcPort, false, true, false, false, nil)} + + bindingProtocol := binding.ProtocolTCP + vIP := agentconfig.VirtualNodePortDNATIPv4 + if isIPv6 { + bindingProtocol = binding.ProtocolTCPv6 + vIP = agentconfig.VirtualNodePortDNATIPv6 + } + + groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, false) + mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, gomock.InAnyOrder(expectedEps)).Times(1) + mockOFClient.EXPECT().InstallServiceGroup(groupID, false, gomock.InAnyOrder(expectedEps)).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), false, corev1.ServiceTypeNodePort).Times(1) + for _, ip := range loadBalancerIPs { + mockOFClient.EXPECT().InstallServiceFlows(groupID, ip, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeLoadBalancer).Times(1) + } + mockRouteClient.EXPECT().AddClusterIPRoute(svcIP).Times(1) + mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) + for _, ip := range loadBalancerIPs { + mockRouteClient.EXPECT().AddLoadBalancer(ip).Times(1) + } + + toDeleteLoadBalancerIPs := smallSliceDifference(loadBalancerIPStrs, updatedLoadBalancerIPStrs) + toAddLoadBalancerIPs := smallSliceDifference(updatedLoadBalancerIPStrs, loadBalancerIPStrs) + for _, ipStr := range toDeleteLoadBalancerIPs { + mockOFClient.EXPECT().UninstallServiceFlows(net.ParseIP(ipStr), uint16(svcPort), bindingProtocol).Times(1) + mockRouteClient.EXPECT().DeleteLoadBalancer(net.ParseIP(ipStr)).Times(1) + } + for _, ipStr := range toAddLoadBalancerIPs { + mockOFClient.EXPECT().InstallServiceFlows(groupID, net.ParseIP(ipStr), uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeLoadBalancer).Times(1) + mockRouteClient.EXPECT().AddLoadBalancer(net.ParseIP(ipStr)).Times(1) + } + + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP).Times(1) + + fp.syncProxyRules() + fp.serviceChanges.OnServiceUpdate(svc, updatedSvc) + fp.syncProxyRules() +} + +func TestServiceIngressIPsUpdate(t *testing.T) { + t.Run("IPv4", func(t *testing.T) { + t.Run("LoadBalancer", func(t *testing.T) { + loadBalancerIPs := []net.IP{net.ParseIP("169.254.1.1"), net.ParseIP("169.254.1.2")} + updatedLoadBalancerIPs := []net.IP{net.ParseIP("169.254.1.2"), net.ParseIP("169.254.1.3")} + testServiceIngressIPsUpdate(t, nodePortAddressesIPv4, svc1IPv4, ep1IPv4, loadBalancerIPs, updatedLoadBalancerIPs, false) + }) + }) + t.Run("IPv6", func(t *testing.T) { + t.Run("LoadBalancer", func(t *testing.T) { + loadBalancerIPs := []net.IP{net.ParseIP("fec0::169:254:1:1"), net.ParseIP("fec0::169:254:1:2")} + updatedLoadBalancerIPs := []net.IP{net.ParseIP("fec0::169:254:1:2"), net.ParseIP("fec0::169:254:1:3")} + testServiceIngressIPsUpdate(t, nodePortAddressesIPv6, svc1IPv6, ep1IPv6, loadBalancerIPs, updatedLoadBalancerIPs, true) + }) + }) +} + +func testServiceStickyMaxAgeSecondsUpdate(t *testing.T, + nodePortAddresses []net.IP, + svcIP net.IP, + loadBalancerIP net.IP, + epIP net.IP, + svcType corev1.ServiceType, + isIPv6 bool) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockOFClient, mockRouteClient := getMockClients(ctrl) + groupAllocator := openflow.NewGroupAllocator(isIPv6) + fp := NewFakeProxier(mockRouteClient, mockOFClient, nodePortAddresses, groupAllocator, isIPv6, withProxyAll) + + var svc, updatedSvc *corev1.Service + affinitySeconds := int32(10) + updatedAffinitySeconds := int32(100) + switch svcType { + case corev1.ServiceTypeClusterIP: + svc = makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, &affinitySeconds, nil) + updatedSvc = makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, &updatedAffinitySeconds, nil) + case corev1.ServiceTypeNodePort: + svc = makeTestNodePortService(&svcPortName, svcIP, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &affinitySeconds, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestNodePortService(&svcPortName, svcIP, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &updatedAffinitySeconds, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) + case corev1.ServiceTypeLoadBalancer: + svc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &affinitySeconds, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &updatedAffinitySeconds, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + } + makeServiceMap(fp, svc) + + epSubset := makeTestEndpointSubset(&svcPortName, epIP, int32(svcPort), corev1.ProtocolTCP, false) + eps := makeTestEndpoints(&svcPortName, []corev1.EndpointSubset{*epSubset}) + makeEndpointsMap(fp, eps) + + expectedEps := []k8sproxy.Endpoint{k8sproxy.NewBaseEndpointInfo(epIP.String(), "", "", svcPort, false, true, false, false, nil)} + + bindingProtocol := binding.ProtocolTCP + vIP := agentconfig.VirtualNodePortDNATIPv4 + if isIPv6 { + bindingProtocol = binding.ProtocolTCPv6 + vIP = agentconfig.VirtualNodePortDNATIPv6 + } + + groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, false) + mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, expectedEps).Times(1) + mockOFClient.EXPECT().InstallServiceGroup(groupID, true, expectedEps).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(affinitySeconds), false, corev1.ServiceTypeClusterIP).Times(1) + mockRouteClient.EXPECT().AddClusterIPRoute(svcIP).Times(1) + + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(updatedAffinitySeconds), false, corev1.ServiceTypeClusterIP).Times(1) + + if svcType == corev1.ServiceTypeNodePort || svcType == corev1.ServiceTypeLoadBalancer { + mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(affinitySeconds), false, corev1.ServiceTypeNodePort).Times(1) + mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) + } + if svcType == corev1.ServiceTypeLoadBalancer { + mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(affinitySeconds), false, corev1.ServiceTypeLoadBalancer).Times(1) + mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) + } + + fp.syncProxyRules() + fp.serviceChanges.OnServiceUpdate(svc, updatedSvc) + fp.syncProxyRules() +} + +func TestServiceStickyMaxAgeSecondsUpdate(t *testing.T) { + t.Run("IPv4", func(t *testing.T) { + t.Run("ClusterIP", func(t *testing.T) { + testServiceStickyMaxAgeSecondsUpdate(t, nil, svc1IPv4, nil, ep1IPv4, corev1.ServiceTypeClusterIP, false) + }) + t.Run("NodePort", func(t *testing.T) { + testServiceStickyMaxAgeSecondsUpdate(t, nodePortAddressesIPv4, svc1IPv4, nil, ep1IPv4, corev1.ServiceTypeNodePort, false) }) - makeServiceMap(fp, svc) - return svc - } - - svc1 := svcMapFactory(svcPortName1, svcIP1.String()) - svc2 := svcMapFactory(svcPortName2, svcIP2.String()) - - epMapFactory := func(svcPortName k8sproxy.ServicePortName, epIP string) *corev1.Endpoints { - ep := makeTestEndpoints(svcPortName.Namespace, svcPortName.Name, func(ept *corev1.Endpoints) { - ept.Subsets = []corev1.EndpointSubset{{ - Addresses: []corev1.EndpointAddress{{ - IP: epIP, - }}, - Ports: []corev1.EndpointPort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: corev1.ProtocolTCP, - }}, - }} + t.Run("LoadBalancer", func(t *testing.T) { + testServiceStickyMaxAgeSecondsUpdate(t, nodePortAddressesIPv4, svc1IPv4, loadBalancerIPv4, ep1IPv4, corev1.ServiceTypeLoadBalancer, false) }) - makeEndpointsMap(fp, ep) - return ep + }) + t.Run("IPv6", func(t *testing.T) { + t.Run("ClusterIP", func(t *testing.T) { + testServiceStickyMaxAgeSecondsUpdate(t, nil, svc1IPv6, nil, ep1IPv6, corev1.ServiceTypeClusterIP, true) + }) + t.Run("NodePort", func(t *testing.T) { + testServiceStickyMaxAgeSecondsUpdate(t, nodePortAddressesIPv6, svc1IPv6, nil, ep1IPv6, corev1.ServiceTypeNodePort, true) + }) + t.Run("LoadBalancer", func(t *testing.T) { + testServiceStickyMaxAgeSecondsUpdate(t, nodePortAddressesIPv6, svc1IPv6, loadBalancerIPv6, ep1IPv6, corev1.ServiceTypeLoadBalancer, true) + }) + }) +} + +func testServiceSessionAffinityTypeUpdate(t *testing.T, + nodePortAddresses []net.IP, + svcIP net.IP, + loadBalancerIP net.IP, + epIP net.IP, + svcType corev1.ServiceType, + isIPv6 bool) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockOFClient, mockRouteClient := getMockClients(ctrl) + groupAllocator := openflow.NewGroupAllocator(isIPv6) + fp := NewFakeProxier(mockRouteClient, mockOFClient, nodePortAddresses, groupAllocator, isIPv6, withProxyAll) + + var svc, updatedSvc *corev1.Service + affinitySeconds := int32(100) + switch svcType { + case corev1.ServiceTypeClusterIP: + svc = makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, nil, nil) + updatedSvc = makeTestClusterIPService(&svcPortName, svcIP, int32(svcPort), corev1.ProtocolTCP, &affinitySeconds, nil) + case corev1.ServiceTypeNodePort: + svc = makeTestNodePortService(&svcPortName, svcIP, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestNodePortService(&svcPortName, svcIP, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &affinitySeconds, corev1.ServiceInternalTrafficPolicyCluster, corev1.ServiceExternalTrafficPolicyTypeCluster) + case corev1.ServiceTypeLoadBalancer: + svc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, nil, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) + updatedSvc = makeTestLoadBalancerService(&svcPortName, svcIP, []net.IP{loadBalancerIP}, int32(svcPort), int32(svcNodePort), corev1.ProtocolTCP, &affinitySeconds, nil, corev1.ServiceExternalTrafficPolicyTypeCluster) } + makeServiceMap(fp, svc) + + epSubset := makeTestEndpointSubset(&svcPortName, epIP, int32(svcPort), corev1.ProtocolTCP, false) + eps := makeTestEndpoints(&svcPortName, []corev1.EndpointSubset{*epSubset}) + makeEndpointsMap(fp, eps) - ep1 := epMapFactory(svcPortName1, epIP.String()) - ep2 := epMapFactory(svcPortName2, epIP.String()) + expectedEps := []k8sproxy.Endpoint{k8sproxy.NewBaseEndpointInfo(epIP.String(), "", "", svcPort, false, true, false, false, nil)} + + bindingProtocol := binding.ProtocolTCP + vIP := agentconfig.VirtualNodePortDNATIPv4 + if isIPv6 { + bindingProtocol = binding.ProtocolTCPv6 + vIP = agentconfig.VirtualNodePortDNATIPv6 + } + + groupID := fp.groupCounter.AllocateIfNotExist(svcPortName, false) + mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, expectedEps).Times(1) + mockOFClient.EXPECT().InstallServiceGroup(groupID, false, expectedEps).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP).Times(1) + mockRouteClient.EXPECT().AddClusterIPRoute(svcIP).Times(1) + + mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, expectedEps).Times(1) + mockOFClient.EXPECT().InstallServiceGroup(groupID, true, expectedEps).Times(1) + mockOFClient.EXPECT().UninstallServiceFlows(svcIP, uint16(svcPort), bindingProtocol).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, svcIP, uint16(svcPort), bindingProtocol, uint16(affinitySeconds), false, corev1.ServiceTypeClusterIP).Times(1) + mockRouteClient.EXPECT().DeleteClusterIPRoute(svcIP).Times(1) + mockRouteClient.EXPECT().AddClusterIPRoute(svcIP).Times(1) + + if svcType == corev1.ServiceTypeNodePort || svcType == corev1.ServiceTypeLoadBalancer { + mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(0), false, corev1.ServiceTypeNodePort).Times(1) + mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) + + mockOFClient.EXPECT().UninstallServiceFlows(vIP, uint16(svcNodePort), bindingProtocol).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID, vIP, uint16(svcNodePort), bindingProtocol, uint16(affinitySeconds), false, corev1.ServiceTypeNodePort).Times(1) + mockRouteClient.EXPECT().DeleteNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) + mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) + } + if svcType == corev1.ServiceTypeLoadBalancer { + mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeLoadBalancer).Times(1) + mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) + + mockOFClient.EXPECT().UninstallServiceFlows(loadBalancerIP, uint16(svcPort), bindingProtocol) + mockOFClient.EXPECT().InstallServiceFlows(groupID, loadBalancerIP, uint16(svcPort), bindingProtocol, uint16(affinitySeconds), false, corev1.ServiceTypeLoadBalancer).Times(1) + mockRouteClient.EXPECT().DeleteLoadBalancer(loadBalancerIP).Times(1) + mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) + } + + fp.syncProxyRules() + fp.serviceChanges.OnServiceUpdate(svc, updatedSvc) + fp.syncProxyRules() +} + +func TestServiceSessionAffinityTypeUpdate(t *testing.T) { + t.Run("IPv4", func(t *testing.T) { + t.Run("ClusterIP", func(t *testing.T) { + testServiceSessionAffinityTypeUpdate(t, nil, svc1IPv4, nil, ep1IPv4, corev1.ServiceTypeClusterIP, false) + }) + t.Run("NodePort", func(t *testing.T) { + testServiceSessionAffinityTypeUpdate(t, nodePortAddressesIPv4, svc1IPv4, nil, ep1IPv4, corev1.ServiceTypeNodePort, false) + }) + t.Run("LoadBalancer", func(t *testing.T) { + testServiceSessionAffinityTypeUpdate(t, nodePortAddressesIPv4, svc1IPv4, loadBalancerIPv4, ep1IPv4, corev1.ServiceTypeLoadBalancer, false) + }) + }) + t.Run("IPv6", func(t *testing.T) { + t.Run("ClusterIP", func(t *testing.T) { + testServiceSessionAffinityTypeUpdate(t, nil, svc1IPv6, nil, ep1IPv6, corev1.ServiceTypeClusterIP, true) + }) + t.Run("NodePort", func(t *testing.T) { + testServiceSessionAffinityTypeUpdate(t, nodePortAddressesIPv6, svc1IPv6, nil, ep1IPv6, corev1.ServiceTypeNodePort, true) + }) + t.Run("LoadBalancer", func(t *testing.T) { + testServiceSessionAffinityTypeUpdate(t, nodePortAddressesIPv6, svc1IPv6, loadBalancerIPv6, ep1IPv6, corev1.ServiceTypeLoadBalancer, true) + }) + }) +} + +func TestServicesWithSameEndpoints(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockOFClient, mockRouteClient := getMockClients(ctrl) + groupAllocator := openflow.NewGroupAllocator(false) + fp := NewFakeProxier(mockRouteClient, mockOFClient, nil, groupAllocator, false) + + svcPortName1 := makeSvcPortName("ns", "svc1", strconv.Itoa(svcPort), corev1.ProtocolTCP) + svcPortName2 := makeSvcPortName("ns", "svc2", strconv.Itoa(svcPort), corev1.ProtocolTCP) + svc1 := makeTestClusterIPService(&svcPortName1, svc1IPv4, int32(svcPort), corev1.ProtocolTCP, nil, nil) + svc2 := makeTestClusterIPService(&svcPortName2, svc2IPv4, int32(svcPort), corev1.ProtocolTCP, nil, nil) + makeServiceMap(fp, svc1, svc2) + + epSubset := makeTestEndpointSubset(&svcPortName1, ep1IPv4, int32(svcPort), corev1.ProtocolTCP, false) + ep1 := makeTestEndpoints(&svcPortName1, []corev1.EndpointSubset{*epSubset}) + epSubset = makeTestEndpointSubset(&svcPortName2, ep1IPv4, int32(svcPort), corev1.ProtocolTCP, false) + ep2 := makeTestEndpoints(&svcPortName2, []corev1.EndpointSubset{*epSubset}) + makeEndpointsMap(fp, ep1, ep2) groupID1 := fp.groupCounter.AllocateIfNotExist(svcPortName1, false) groupID2 := fp.groupCounter.AllocateIfNotExist(svcPortName2, false) @@ -1228,10 +2196,10 @@ func TestServicesWithSameEndpoints(t *testing.T) { mockOFClient.EXPECT().InstallServiceGroup(groupID2, false, gomock.Any()).Times(1) bindingProtocol := binding.ProtocolTCP mockOFClient.EXPECT().InstallEndpointFlows(bindingProtocol, gomock.Any()).Times(2) - mockOFClient.EXPECT().InstallServiceFlows(groupID1, svcIP1, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP).Times(1) - mockOFClient.EXPECT().InstallServiceFlows(groupID2, svcIP2, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP).Times(1) - mockOFClient.EXPECT().UninstallServiceFlows(svcIP1, uint16(svcPort), bindingProtocol).Times(1) - mockOFClient.EXPECT().UninstallServiceFlows(svcIP2, uint16(svcPort), bindingProtocol).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID1, svc1IPv4, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(groupID2, svc2IPv4, uint16(svcPort), bindingProtocol, uint16(0), false, corev1.ServiceTypeClusterIP).Times(1) + mockOFClient.EXPECT().UninstallServiceFlows(svc1IPv4, uint16(svcPort), bindingProtocol).Times(1) + mockOFClient.EXPECT().UninstallServiceFlows(svc2IPv4, uint16(svcPort), bindingProtocol).Times(1) mockOFClient.EXPECT().UninstallServiceGroup(groupID1).Times(1) mockOFClient.EXPECT().UninstallServiceGroup(groupID2).Times(1) // Since these two Services reference to the same Endpoint, there should only be one operation. @@ -1249,6 +2217,7 @@ func TestServicesWithSameEndpoints(t *testing.T) { } func TestMetrics(t *testing.T) { + legacyregistry.Reset() metrics.Register() for _, tc := range []struct { @@ -1256,8 +2225,8 @@ func TestMetrics(t *testing.T) { svcIP, ep1IP, ep2IP net.IP isIPv6 bool }{ - {"IPv4", svcIPv4, ep1IPv4, ep2IPv4, false}, - {"IPv6", svcIPv6, ep1IPv6, ep2IPv6, true}, + {"IPv4", svc1IPv4, ep1IPv4, ep2IPv4, false}, + {"IPv6", svc1IPv6, ep1IPv6, ep2IPv6, true}, } { t.Run(tc.name, func(t *testing.T) { endpointsUpdateTotalMetric := metrics.EndpointsUpdatesTotal.CounterMetric @@ -1270,7 +2239,8 @@ func TestMetrics(t *testing.T) { endpointsInstallMetric = metrics.EndpointsInstalledTotalV6.GaugeMetric servicesInstallMetric = metrics.ServicesInstalledTotalV6.GaugeMetric } - testClusterIP(t, tc.svcIP, tc.ep1IP, tc.ep2IP, tc.isIPv6, false, []*corev1.Service{}, []*corev1.Endpoints{}) + + testClusterIPAdd(t, tc.svcIP, tc.ep1IP, tc.ep2IP, tc.isIPv6, false, []*corev1.Service{}, []*corev1.Endpoints{}, false) v, err := testutil.GetCounterMetricValue(endpointsUpdateTotalMetric) assert.NoError(t, err) assert.Equal(t, 0, int(v)) @@ -1284,7 +2254,7 @@ func TestMetrics(t *testing.T) { assert.Equal(t, 2, int(v)) assert.NoError(t, err) - testClusterIPRemoval(t, tc.svcIP, tc.ep1IP, tc.isIPv6) + testClusterIPRemove(t, tc.svcIP, tc.ep1IP, tc.isIPv6) v, err = testutil.GetCounterMetricValue(endpointsUpdateTotalMetric) assert.NoError(t, err) @@ -1301,3 +2271,87 @@ func TestMetrics(t *testing.T) { }) } } + +func TestGetServiceFlowKeys(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockOFClient, mockRouteClient := getMockClients(ctrl) + groupAllocator := openflow.NewGroupAllocator(false) + svc := makeTestNodePortService(&svcPortName, + svc1IPv4, + int32(svcPort), + int32(svcNodePort), + corev1.ProtocolTCP, + nil, + corev1.ServiceInternalTrafficPolicyLocal, + corev1.ServiceExternalTrafficPolicyTypeCluster) + remoteEpSubset := makeTestEndpointSubset(&svcPortName, ep1IPv4, int32(svcPort), corev1.ProtocolTCP, false) + localEpSubset := makeTestEndpointSubset(&svcPortName, ep2IPv4, int32(svcPort), corev1.ProtocolTCP, true) + eps := makeTestEndpoints(&svcPortName, []corev1.EndpointSubset{*remoteEpSubset, *localEpSubset}) + + testCases := []struct { + name string + svc *corev1.Service + eps *corev1.Endpoints + serviceInstalled bool + expectedFound bool + }{ + { + name: "Installed Service with Endpoints", + svc: svc, + eps: eps, + serviceInstalled: true, + expectedFound: true, + }, + { + name: "Not installed Service without Endpoints", + svc: svc, + serviceInstalled: false, + expectedFound: false, + }, + { + name: "Not installed Service with Endpoints", + svc: svc, + eps: eps, + serviceInstalled: false, + expectedFound: false, + }, + { + name: "Not existing Service", + serviceInstalled: false, + expectedFound: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fp := NewFakeProxier(mockRouteClient, mockOFClient, nodePortAddressesIPv4, groupAllocator, false, withProxyAll) + if tc.svc != nil { + makeServiceMap(fp, svc) + } + if tc.eps != nil { + makeEndpointsMap(fp, eps) + } + if tc.svc != nil && tc.eps != nil && tc.serviceInstalled { + mockRouteClient.EXPECT().AddClusterIPRoute(svc1IPv4).Times(1) + mockRouteClient.EXPECT().AddNodePort(nodePortAddressesIPv4, uint16(svcNodePort), binding.ProtocolTCP).Times(1) + mockOFClient.EXPECT().InstallServiceGroup(gomock.Any(), gomock.Any(), gomock.Any()).Times(2) + mockOFClient.EXPECT().InstallEndpointFlows(binding.ProtocolTCP, gomock.Any()).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), gomock.Any(), uint16(svcNodePort), binding.ProtocolTCP, uint16(0), false, corev1.ServiceTypeNodePort).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), svc1IPv4, uint16(svcPort), binding.ProtocolTCP, uint16(0), false, corev1.ServiceTypeClusterIP).Times(1) + fp.syncProxyRules() + } + + var expectedGroupIDs []binding.GroupIDType + if tc.serviceInstalled { + expectedGroupIDs = append(expectedGroupIDs, fp.groupCounter.AllocateIfNotExist(svcPortName, false)) + expectedGroupIDs = append(expectedGroupIDs, fp.groupCounter.AllocateIfNotExist(svcPortName, true)) + mockOFClient.EXPECT().GetServiceFlowKeys(svc1IPv4, uint16(svcPort), binding.ProtocolTCP, gomock.Any()).Times(1) + } + + _, groupIDs, found := fp.GetServiceFlowKeys("svc", "ns") + assert.ElementsMatch(t, expectedGroupIDs, groupIDs) + assert.Equal(t, tc.expectedFound, found) + }) + } +} diff --git a/pkg/agent/route/interfaces.go b/pkg/agent/route/interfaces.go index f0e1fc04a57..c55e9f7ae5a 100644 --- a/pkg/agent/route/interfaces.go +++ b/pkg/agent/route/interfaces.go @@ -29,7 +29,7 @@ type Interface interface { // Reconcile should remove orphaned routes and related configuration based on the desired podCIDRs and Service IPs. // If IPv6 is enabled in the cluster, Reconcile should also remove the orphaned IPv6 neighbors. - Reconcile(podCIDRs []string, svcIPs map[string]bool) error + Reconcile(podCIDRs []string) error // AddRoutes should add routes to the provided podCIDR. // It should override the routes if they already exist, without error. @@ -65,11 +65,11 @@ type Interface interface { // ClusterIP Service traffic from host network. DeleteClusterIPRoute(svcIP net.IP) error - // AddLoadBalancer adds configurations when a LoadBalancer Service is created. - AddLoadBalancer(externalIPs []string) error + // AddLoadBalancer adds configurations when a LoadBalancer IP is added. + AddLoadBalancer(externalIP net.IP) error - // DeleteLoadBalancer deletes related configurations when a LoadBalancer Service is deleted. - DeleteLoadBalancer(externalIPs []string) error + // DeleteLoadBalancer deletes related configurations when a LoadBalancer IP is deleted. + DeleteLoadBalancer(externalIP net.IP) error // Run starts the sync loop. Run(stopCh <-chan struct{}) diff --git a/pkg/agent/route/route_linux.go b/pkg/agent/route/route_linux.go index 3ad4f5c7ba0..1c2e26a4914 100644 --- a/pkg/agent/route/route_linux.go +++ b/pkg/agent/route/route_linux.go @@ -843,7 +843,7 @@ func (c *Client) initServiceIPRoutes() error { // Reconcile removes orphaned podCIDRs from ipset and removes routes to orphaned podCIDRs // based on the desired podCIDRs. svcIPs are used for Windows only. -func (c *Client) Reconcile(podCIDRs []string, svcIPs map[string]bool) error { +func (c *Client) Reconcile(podCIDRs []string) error { desiredPodCIDRs := sets.NewString(podCIDRs...) // Get the peer IPv6 gateways from pod CIDRs desiredIPv6GWs := getIPv6Gateways(podCIDRs) @@ -1425,11 +1425,10 @@ func (c *Client) addVirtualNodePortDNATIPRoute(isIPv6 bool) error { return nil } -// addLoadBalancerIngressIPRoute is used to add routing entry which is used to route LoadBalancer ingress IP to Antrea +// AddLoadBalancer is used to add routing entry which is used to route LoadBalancer ingress IP to Antrea // gateway on host. -func (c *Client) addLoadBalancerIngressIPRoute(svcIPStr string) error { +func (c *Client) AddLoadBalancer(svcIP net.IP) error { linkIndex := c.nodeConfig.GatewayConfig.LinkIndex - svcIP := net.ParseIP(svcIPStr) isIPv6 := utilnet.IsIPv6(svcIP) var gw net.IP var mask int @@ -1451,11 +1450,10 @@ func (c *Client) addLoadBalancerIngressIPRoute(svcIPStr string) error { return nil } -// deleteLoadBalancerIngressIPRoute is used to delete routing entry which is used to route LoadBalancer ingress IP to Antrea +// DeleteLoadBalancer is used to delete routing entry which is used to route LoadBalancer ingress IP to Antrea // gateway on host. -func (c *Client) deleteLoadBalancerIngressIPRoute(svcIPStr string) error { +func (c *Client) DeleteLoadBalancer(svcIP net.IP) error { linkIndex := c.nodeConfig.GatewayConfig.LinkIndex - svcIP := net.ParseIP(svcIPStr) isIPv6 := utilnet.IsIPv6(svcIP) var gw net.IP var mask int @@ -1481,28 +1479,6 @@ func (c *Client) deleteLoadBalancerIngressIPRoute(svcIPStr string) error { return nil } -// AddLoadBalancer is used to add routing entries when a LoadBalancer Service is added. -func (c *Client) AddLoadBalancer(externalIPs []string) error { - for _, svcIPStr := range externalIPs { - if err := c.addLoadBalancerIngressIPRoute(svcIPStr); err != nil { - return err - } - } - - return nil -} - -// DeleteLoadBalancer is used to delete routing entries when a LoadBalancer Service is deleted. -func (c *Client) DeleteLoadBalancer(externalIPs []string) error { - for _, svcIPStr := range externalIPs { - if err := c.deleteLoadBalancerIngressIPRoute(svcIPStr); err != nil { - return err - } - } - - return nil -} - // AddLocalAntreaFlexibleIPAMPodRule is used to add IP to target ip set when an AntreaFlexibleIPAM Pod is added. An entry is added // for every Pod IP. func (c *Client) AddLocalAntreaFlexibleIPAMPodRule(podAddresses []net.IP) error { diff --git a/pkg/agent/route/route_linux_test.go b/pkg/agent/route/route_linux_test.go index b8c7503bc90..44afcd40587 100644 --- a/pkg/agent/route/route_linux_test.go +++ b/pkg/agent/route/route_linux_test.go @@ -622,7 +622,7 @@ func TestReconcile(t *testing.T) { {IP: net.ParseIP("2001:ab03:cd04:55ee:100b::1")}, // non-existing podCIDR, should be deleted. }, nil) mockNetlink.EXPECT().NeighDel(&netlink.Neigh{IP: net.ParseIP("2001:ab03:cd04:55ee:100b::1")}) - assert.NoError(t, c.Reconcile(podCIDRs, nil)) + assert.NoError(t, c.Reconcile(podCIDRs)) } func TestAddRoutes(t *testing.T) { @@ -1334,12 +1334,12 @@ func TestAddLoadBalancer(t *testing.T) { nodeConfig := &config.NodeConfig{GatewayConfig: &config.GatewayConfig{LinkIndex: 10}} tests := []struct { name string - externalIPs []string + externalIP string expectedCalls func(mockNetlink *netlinktest.MockInterfaceMockRecorder) }{ { - name: "IPv4", - externalIPs: []string{"1.1.1.1", "1.1.1.2"}, + name: "IPv4", + externalIP: "1.1.1.1", expectedCalls: func(mockNetlink *netlinktest.MockInterfaceMockRecorder) { mockNetlink.RouteReplace(&netlink.Route{ Dst: &net.IPNet{ @@ -1350,20 +1350,11 @@ func TestAddLoadBalancer(t *testing.T) { Scope: netlink.SCOPE_UNIVERSE, LinkIndex: 10, }) - mockNetlink.RouteReplace(&netlink.Route{ - Dst: &net.IPNet{ - IP: net.ParseIP("1.1.1.2"), - Mask: net.CIDRMask(32, 32), - }, - Gw: config.VirtualServiceIPv4, - Scope: netlink.SCOPE_UNIVERSE, - LinkIndex: 10, - }) }, }, { - name: "IPv6", - externalIPs: []string{"fd00:1234:5678:dead:beaf::1", "fd00:1234:5678:dead:beaf::a"}, + name: "IPv6", + externalIP: "fd00:1234:5678:dead:beaf::1", expectedCalls: func(mockNetlink *netlinktest.MockInterfaceMockRecorder) { mockNetlink.RouteReplace(&netlink.Route{ Dst: &net.IPNet{IP: net.ParseIP("fd00:1234:5678:dead:beaf::1"), Mask: net.CIDRMask(128, 128)}, @@ -1371,12 +1362,6 @@ func TestAddLoadBalancer(t *testing.T) { Scope: netlink.SCOPE_UNIVERSE, LinkIndex: 10, }) - mockNetlink.RouteReplace(&netlink.Route{ - Dst: &net.IPNet{IP: net.ParseIP("fd00:1234:5678:dead:beaf::a"), Mask: net.CIDRMask(128, 128)}, - Gw: config.VirtualServiceIPv6, - Scope: netlink.SCOPE_UNIVERSE, - LinkIndex: 10, - }) }, }, } @@ -1391,7 +1376,7 @@ func TestAddLoadBalancer(t *testing.T) { } tt.expectedCalls(mockNetlink.EXPECT()) - assert.NoError(t, c.AddLoadBalancer(tt.externalIPs)) + assert.NoError(t, c.AddLoadBalancer(net.ParseIP(tt.externalIP))) }) } } @@ -1400,12 +1385,12 @@ func TestDeleteLoadBalancer(t *testing.T) { nodeConfig := &config.NodeConfig{GatewayConfig: &config.GatewayConfig{LinkIndex: 10}} tests := []struct { name string - externalIPs []string + externalIP string expectedCalls func(mockNetlink *netlinktest.MockInterfaceMockRecorder) }{ { - name: "IPv4", - externalIPs: []string{"1.1.1.1", "1.1.1.2"}, + name: "IPv4", + externalIP: "1.1.1.1", expectedCalls: func(mockNetlink *netlinktest.MockInterfaceMockRecorder) { mockNetlink.RouteDel(&netlink.Route{ Dst: &net.IPNet{ @@ -1416,20 +1401,11 @@ func TestDeleteLoadBalancer(t *testing.T) { Scope: netlink.SCOPE_UNIVERSE, LinkIndex: 10, }) - mockNetlink.RouteDel(&netlink.Route{ - Dst: &net.IPNet{ - IP: net.ParseIP("1.1.1.2"), - Mask: net.CIDRMask(32, 32), - }, - Gw: config.VirtualServiceIPv4, - Scope: netlink.SCOPE_UNIVERSE, - LinkIndex: 10, - }) }, }, { - name: "IPv6", - externalIPs: []string{"fd00:1234:5678:dead:beaf::1", "fd00:1234:5678:dead:beaf::a"}, + name: "IPv6", + externalIP: "fd00:1234:5678:dead:beaf::1", expectedCalls: func(mockNetlink *netlinktest.MockInterfaceMockRecorder) { mockNetlink.RouteDel(&netlink.Route{ Dst: &net.IPNet{IP: net.ParseIP("fd00:1234:5678:dead:beaf::1"), Mask: net.CIDRMask(128, 128)}, @@ -1437,12 +1413,6 @@ func TestDeleteLoadBalancer(t *testing.T) { Scope: netlink.SCOPE_UNIVERSE, LinkIndex: 10, }) - mockNetlink.RouteDel(&netlink.Route{ - Dst: &net.IPNet{IP: net.ParseIP("fd00:1234:5678:dead:beaf::a"), Mask: net.CIDRMask(128, 128)}, - Gw: config.VirtualServiceIPv6, - Scope: netlink.SCOPE_UNIVERSE, - LinkIndex: 10, - }) }, }, } @@ -1457,7 +1427,7 @@ func TestDeleteLoadBalancer(t *testing.T) { } tt.expectedCalls(mockNetlink.EXPECT()) - assert.NoError(t, c.DeleteLoadBalancer(tt.externalIPs)) + assert.NoError(t, c.DeleteLoadBalancer(net.ParseIP(tt.externalIP))) }) } } diff --git a/pkg/agent/route/route_windows.go b/pkg/agent/route/route_windows.go index 3e58e1556bd..7722774f68c 100644 --- a/pkg/agent/route/route_windows.go +++ b/pkg/agent/route/route_windows.go @@ -122,7 +122,7 @@ func (c *Client) initServiceIPRoutes() error { // Reconcile removes the orphaned routes and related configuration based on the desired podCIDRs and Service IPs. Only // the route entries on the host gateway interface are stored in the cache. -func (c *Client) Reconcile(podCIDRs []string, svcIPs map[string]bool) error { +func (c *Client) Reconcile(podCIDRs []string) error { desiredPodCIDRs := sets.NewString(podCIDRs...) routes, err := c.listRoutes() if err != nil { @@ -133,8 +133,8 @@ func (c *Client) Reconcile(podCIDRs []string, svcIPs map[string]bool) error { c.hostRoutes.Store(dst, rt) continue } - if _, ok := svcIPs[dst]; ok { - c.hostRoutes.Store(dst, rt) + // Don't delete the routes which are added by AntreaProxy. + if c.isServiceRoute(rt) { continue } err := util.RemoveNetRoute(rt) @@ -260,8 +260,8 @@ func (c *Client) addVirtualServiceIPRoute(isIPv6 bool) error { // TODO: Follow the code style in Linux that maintains one Service CIDR. func (c *Client) addServiceRoute(svcIP net.IP) error { - obj, found := c.hostRoutes.Load(svcIP.String()) svcIPNet := util.NewIPNet(svcIP) + obj, found := c.hostRoutes.Load(svcIPNet.String()) // Route: Service IP -> VirtualServiceIPv4 (169.254.0.253) route := &util.Route{ @@ -288,7 +288,7 @@ func (c *Client) addServiceRoute(svcIP net.IP) error { return err } - c.hostRoutes.Store(route.DestinationSubnet.String(), route) + c.hostRoutes.Store(svcIPNet.String(), route) klog.V(2).InfoS("Added Service route", "ServiceIP", route.DestinationSubnet, "GatewayIP", route.GatewayAddress) return nil } @@ -305,7 +305,7 @@ func (c *Client) deleteServiceRoute(svcIP net.IP) error { if err := util.RemoveNetRoute(rt); err != nil { return err } - c.hostRoutes.Delete(svcIP.String()) + c.hostRoutes.Delete(svcIPNet.String()) klog.V(2).InfoS("Deleted Service route from host gateway", "DestinationIP", svcIP) return nil } @@ -332,6 +332,15 @@ func (c *Client) UnMigrateRoutesFromGw(route *net.IPNet, linkName string) error func (c *Client) Run(stopCh <-chan struct{}) { } +func (c *Client) isServiceRoute(route *util.Route) bool { + // If the destination IP or gateway IP is the virtual Service IP, then it is the Service route added by AntreaProxy. + if route.DestinationSubnet != nil && route.DestinationSubnet.IP.Equal(config.VirtualServiceIPv4) || + route.GatewayAddress != nil && route.GatewayAddress.Equal(config.VirtualServiceIPv4) { + return true + } + return false +} + func (c *Client) listRoutes() (map[string]*util.Route, error) { routes, err := util.GetNetRoutesAll() if err != nil { @@ -394,22 +403,12 @@ func (c *Client) DeleteNodePort(nodePortAddresses []net.IP, port uint16, protoco return util.RemoveNetNatStaticMapping(antreaNatNodePort, "0.0.0.0", port, string(protocol)) } -func (c *Client) AddLoadBalancer(externalIPs []string) error { - for _, svcIPStr := range externalIPs { - if err := c.addServiceRoute(net.ParseIP(svcIPStr)); err != nil { - return err - } - } - return nil +func (c *Client) AddLoadBalancer(externalIP net.IP) error { + return c.addServiceRoute(externalIP) } -func (c *Client) DeleteLoadBalancer(externalIPs []string) error { - for _, svcIPStr := range externalIPs { - if err := c.deleteServiceRoute(net.ParseIP(svcIPStr)); err != nil { - return err - } - } - return nil +func (c *Client) DeleteLoadBalancer(externalIP net.IP) error { + return c.deleteServiceRoute(externalIP) } func (c *Client) AddLocalAntreaFlexibleIPAMPodRule(podAddresses []net.IP) error { diff --git a/pkg/agent/route/route_windows_test.go b/pkg/agent/route/route_windows_test.go index bfec5a310f7..b0e1c00020f 100644 --- a/pkg/agent/route/route_windows_test.go +++ b/pkg/agent/route/route_windows_test.go @@ -89,14 +89,20 @@ func TestRouteOperation(t *testing.T) { route3, err := util.GetNetRoutes(gwLink, svcIPNet1) require.Nil(t, err) assert.Equal(t, 1, len(route3)) + obj, found := client.hostRoutes.Load(svcIPNet1.String()) + assert.True(t, found) + assert.EqualValues(t, route3[0], *obj.(*util.Route)) err = client.AddClusterIPRoute(svcIP2) require.Nil(t, err) route4, err := util.GetNetRoutes(gwLink, svcIPNet2) require.Nil(t, err) assert.Equal(t, 1, len(route4)) + obj, found = client.hostRoutes.Load(svcIPNet2.String()) + assert.True(t, found) + assert.EqualValues(t, route4[0], *obj.(*util.Route)) - err = client.Reconcile([]string{dest2}, map[string]bool{svcIPNet1.String(): true}) + err = client.Reconcile([]string{dest2}) require.Nil(t, err) routes5, err := util.GetNetRoutes(gwLink, destCIDR1) @@ -105,7 +111,7 @@ func TestRouteOperation(t *testing.T) { routes6, err := util.GetNetRoutes(gwLink, svcIPNet2) require.Nil(t, err) - assert.Equal(t, 0, len(routes6)) + assert.Equal(t, 1, len(routes6)) err = client.DeleteRoutes(destCIDR2) require.Nil(t, err) @@ -118,4 +124,6 @@ func TestRouteOperation(t *testing.T) { routes8, err := util.GetNetRoutes(gwLink, svcIPNet1) require.Nil(t, err) assert.Equal(t, 0, len(routes8)) + _, found = client.hostRoutes.Load(svcIPNet1.String()) + assert.False(t, found) } diff --git a/pkg/agent/route/testing/mock_route.go b/pkg/agent/route/testing/mock_route.go index 67ee9270b63..ba8086898f5 100644 --- a/pkg/agent/route/testing/mock_route.go +++ b/pkg/agent/route/testing/mock_route.go @@ -1,4 +1,4 @@ -// Copyright 2021 Antrea Authors +// Copyright 2023 Antrea Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -65,7 +65,7 @@ func (mr *MockInterfaceMockRecorder) AddClusterIPRoute(arg0 interface{}) *gomock } // AddLoadBalancer mocks base method -func (m *MockInterface) AddLoadBalancer(arg0 []string) error { +func (m *MockInterface) AddLoadBalancer(arg0 net.IP) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "AddLoadBalancer", arg0) ret0, _ := ret[0].(error) @@ -149,7 +149,7 @@ func (mr *MockInterfaceMockRecorder) DeleteClusterIPRoute(arg0 interface{}) *gom } // DeleteLoadBalancer mocks base method -func (m *MockInterface) DeleteLoadBalancer(arg0 []string) error { +func (m *MockInterface) DeleteLoadBalancer(arg0 net.IP) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "DeleteLoadBalancer", arg0) ret0, _ := ret[0].(error) @@ -247,17 +247,17 @@ func (mr *MockInterfaceMockRecorder) MigrateRoutesToGw(arg0 interface{}) *gomock } // Reconcile mocks base method -func (m *MockInterface) Reconcile(arg0 []string, arg1 map[string]bool) error { +func (m *MockInterface) Reconcile(arg0 []string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Reconcile", arg0, arg1) + ret := m.ctrl.Call(m, "Reconcile", arg0) ret0, _ := ret[0].(error) return ret0 } // Reconcile indicates an expected call of Reconcile -func (mr *MockInterfaceMockRecorder) Reconcile(arg0, arg1 interface{}) *gomock.Call { +func (mr *MockInterfaceMockRecorder) Reconcile(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Reconcile", reflect.TypeOf((*MockInterface)(nil).Reconcile), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Reconcile", reflect.TypeOf((*MockInterface)(nil).Reconcile), arg0) } // Run mocks base method diff --git a/pkg/agent/util/net.go b/pkg/agent/util/net.go index 46c254d0365..8ecf026c4f1 100644 --- a/pkg/agent/util/net.go +++ b/pkg/agent/util/net.go @@ -383,7 +383,7 @@ func GetAllNodeAddresses(excludeDevices []string) ([]net.IP, []net.IP, error) { // NewIPNet generates an IPNet from an ip address using a netmask of 32 or 128. func NewIPNet(ip net.IP) *net.IPNet { if ip.To4() != nil { - return &net.IPNet{IP: ip, Mask: net.CIDRMask(32, 32)} + return &net.IPNet{IP: ip.To4(), Mask: net.CIDRMask(32, 32)} } return &net.IPNet{IP: ip, Mask: net.CIDRMask(128, 128)} } diff --git a/test/integration/agent/route_test.go b/test/integration/agent/route_test.go index 66d75911334..1c901a30d88 100644 --- a/test/integration/agent/route_test.go +++ b/test/integration/agent/route_test.go @@ -517,7 +517,6 @@ func TestReconcile(t *testing.T) { addedRoutes []peer desiredPeerCIDRs []string desiredNodeIPs []string - desiredServices map[string]bool // expectations expRoutes map[string]netlink.Link }{ @@ -530,7 +529,6 @@ func TestReconcile(t *testing.T) { }, desiredPeerCIDRs: []string{"10.10.20.0/24"}, desiredNodeIPs: []string{remotePeerIP.String()}, - desiredServices: map[string]bool{"200.200.10.10": true}, expRoutes: map[string]netlink.Link{"10.10.20.0/24": gwLink, "10.10.30.0/24": nil}, }, { @@ -542,7 +540,6 @@ func TestReconcile(t *testing.T) { }, desiredPeerCIDRs: []string{"10.10.20.0/24"}, desiredNodeIPs: []string{localPeerIP.String()}, - desiredServices: map[string]bool{"200.200.10.10": true}, expRoutes: map[string]netlink.Link{"10.10.20.0/24": nodeLink, "10.10.30.0/24": nil}, }, { @@ -556,7 +553,6 @@ func TestReconcile(t *testing.T) { }, desiredPeerCIDRs: []string{"10.10.20.0/24", "10.10.40.0/24"}, desiredNodeIPs: []string{localPeerIP.String(), remotePeerIP.String()}, - desiredServices: map[string]bool{"200.200.10.10": true}, expRoutes: map[string]netlink.Link{"10.10.20.0/24": nodeLink, "10.10.30.0/24": nil, "10.10.40.0/24": gwLink, "10.10.50.0/24": nil}, }, } @@ -574,7 +570,7 @@ func TestReconcile(t *testing.T) { assert.NoError(t, routeClient.AddRoutes(peerNet, tc.nodeName, route.peerIP, peerGwIP), "adding routes failed") } - assert.NoError(t, routeClient.Reconcile(tc.desiredPeerCIDRs, tc.desiredServices), "reconcile failed") + assert.NoError(t, routeClient.Reconcile(tc.desiredPeerCIDRs), "reconcile failed") for dst, uplink := range tc.expRoutes { expNum := 0