diff --git a/docs/user-guide.md b/docs/user-guide.md index 346c2b1d39..9983814dc2 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -124,6 +124,39 @@ and if you want to move back to kube-proxy then clean up config done by kube-rou and run kube-proxy with the configuration you have. - [General Setup](/README.md#getting-started) + +## Advertising IPs + +kube-router can advertise Cluster, External and LoadBalancer IPs to BGP peers. +It does this by: +* locally adding the advertised IPs to the nodes' `kube-dummy-if` network interface +* advertising the IPs to its BGP peers + +To set the default for all services use the `--advertise-cluster-ip`, +`--advertise-external-ip` and `--advertise-loadbalancer-ip` flags. + +To selectively enable or disable this feature per-service use the +`kube-router.io/service.advertise.cluster`, `kube-router.io/service.advertise.external` +and `kube-router.io/service.advertise.loadbalancer` annotations. + +e.g.: +`$ kubectl annotate service my-advertised-service "kube-router.io/service.advertise.cluster=true"` +`$ kubectl annotate service my-advertised-service "kube-router.io/service.advertise.external=true"` +`$ kubectl annotate service my-advertised-service "kube-router.io/service.advertise.loadbalancer=true"` + +`$ kubectl annotate service my-non-advertised-service "kube-router.io/service.advertise.cluster=false"` +`$ kubectl annotate service my-non-advertised-service "kube-router.io/service.advertise.external=false"` +`$ kubectl annotate service my-non-advertised-service "kube-router.io/service.advertise.loadbalancer=false"` + +By combining the flags with the per-service annotations you can choose either +a opt-in or opt-out strategy for advertising IPs. + +Advertising LoadBalancer IPs works by inspecting the services +`status.loadBalancer.ingress` IPs that are set by external LoadBalancers like +for example MetalLb. This has been successfully tested together with +[MetalLB](https://github.com/google/metallb) in ARP mode. + + ## Hairpin Mode Communication from a Pod that is behind a Service to its own ClusterIP:Port is @@ -206,25 +239,6 @@ For destination hashing scheduling use: kubectl annotate service my-service "kube-router.io/service.scheduler=dh" ``` -## LoadBalancer IPs - -If you want to also advertise loadbalancer set IPs -(`status.loadBalancer.ingress` IPs), e.g. when using it with MetalLb, -add the `--advertise-loadbalancer-ip` flag (`false` by default). - -To selectively disable this behaviour per-service, you can use -the `kube-router.io/service.skiplbips` annotation as e.g.: -`$ kubectl annotate service my-external-service "kube-router.io/service.skiplbips=true"` - -In concrete, unless the Service is annotated as per above, the -`--advertise-loadbalancer-ip` flag will make Service's Ingress IP(s) -set by the LoadBalancer to: -* be locally added to nodes' `kube-dummy-if` network interface -* be advertised to BGP peers - -FYI Above has been successfully tested together with -[MetalLB](https://github.com/google/metallb) in ARP mode. - ## HostPort support If you would like to use `HostPort` functionality below changes are required in the manifest. diff --git a/pkg/controllers/routing/ecmp_vip.go b/pkg/controllers/routing/ecmp_vip.go index b6510f7c66..4a9cf31478 100644 --- a/pkg/controllers/routing/ecmp_vip.go +++ b/pkg/controllers/routing/ecmp_vip.go @@ -268,12 +268,9 @@ func (nrc *NetworkRoutingController) getLoadBalancerIps(svc *v1core.Service) []s if svc.Spec.Type == "LoadBalancer" { // skip headless services if svc.Spec.ClusterIP != "None" && svc.Spec.ClusterIP != "" { - _, skiplbips := svc.ObjectMeta.Annotations["kube-router.io/service.skiplbips"] - if !skiplbips { - for _, lbIngress := range svc.Status.LoadBalancer.Ingress { - if len(lbIngress.IP) > 0 { - loadBalancerIpList = append(loadBalancerIpList, lbIngress.IP) - } + for _, lbIngress := range svc.Status.LoadBalancer.Ingress { + if len(lbIngress.IP) > 0 { + loadBalancerIpList = append(loadBalancerIpList, lbIngress.IP) } } } @@ -313,6 +310,16 @@ func (nrc *NetworkRoutingController) getVIPs(onlyActiveEndpoints bool) ([]string return toAdvertiseList, toWithdrawList, nil } +func (nrc *NetworkRoutingController) shouldAdvertiseService(svc *v1core.Service, annotation string, defaultValue bool) bool { + returnValue := defaultValue + stringValue, exists := svc.Annotations[annotation] + if exists { + // Service annotations overrides defaults. + returnValue, _ = strconv.ParseBool(stringValue) + } + return returnValue +} + func (nrc *NetworkRoutingController) getVIPsForService(svc *v1core.Service, onlyActiveEndpoints bool) ([]string, []string, error) { ipList := make([]string, 0) var err error @@ -328,16 +335,21 @@ func (nrc *NetworkRoutingController) getVIPsForService(svc *v1core.Service, only } } - if nrc.advertiseClusterIP { + if nrc.shouldAdvertiseService(svc, svcAdvertiseClusterAnnotation, nrc.advertiseClusterIP) { clusterIp := nrc.getClusterIp(svc) if clusterIp != "" { ipList = append(ipList, clusterIp) } } - if nrc.advertiseExternalIP { + + if nrc.shouldAdvertiseService(svc, svcAdvertiseExternalAnnotation, nrc.advertiseExternalIP) { ipList = append(ipList, nrc.getExternalIps(svc)...) } - if nrc.advertiseLoadBalancerIP { + + // Deprecated: Use service.advertise.loadbalancer=false instead of service.skiplbips. + _, skiplbips := svc.Annotations[svcSkipLbIpsAnnotation] + advertiseLoadBalancer := nrc.shouldAdvertiseService(svc, svcAdvertiseLoadBalancerAnnotation, nrc.advertiseLoadBalancerIP) + if advertiseLoadBalancer && !skiplbips { ipList = append(ipList, nrc.getLoadBalancerIps(svc)...) } diff --git a/pkg/controllers/routing/ecmp_vip_test.go b/pkg/controllers/routing/ecmp_vip_test.go new file mode 100644 index 0000000000..ce82762a88 --- /dev/null +++ b/pkg/controllers/routing/ecmp_vip_test.go @@ -0,0 +1,342 @@ +package routing + +import ( + "testing" + + v1core "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +// Compare 2 string slices by value. +func Equal(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i, v := range a { + if v != b[i] { + return false + } + } + return true +} + +type ServiceAdvertisedIPs struct { + service *v1core.Service + advertisedIPs []string + annotations map[string]string +} + +func Test_getVIPsForService(t *testing.T) { + services := map[string]*v1core.Service{ + "cluster": { + ObjectMeta: metav1.ObjectMeta{ + Name: "svc-cluster", + }, + Spec: v1core.ServiceSpec{ + Type: "ClusterIP", + ClusterIP: "10.0.0.1", + }, + }, + "external": { + ObjectMeta: metav1.ObjectMeta{ + Name: "svc-external", + }, + Spec: v1core.ServiceSpec{ + Type: "ClusterIP", + ClusterIP: "10.0.0.1", + ExternalIPs: []string{"1.1.1.1"}, + }, + }, + "nodeport": { + ObjectMeta: metav1.ObjectMeta{ + Name: "svc-nodeport", + }, + Spec: v1core.ServiceSpec{ + Type: "NodePort", + ClusterIP: "10.0.0.1", + ExternalIPs: []string{"1.1.1.1"}, + }, + }, + "loadbalancer": { + ObjectMeta: metav1.ObjectMeta{ + Name: "svc-loadbalancer", + }, + Spec: v1core.ServiceSpec{ + Type: "LoadBalancer", + ClusterIP: "10.0.0.1", + // External IPs are ignored since LoadBalancer services don't + // advertise external IPs. + ExternalIPs: []string{"1.1.1.1"}, + }, + Status: v1core.ServiceStatus{ + LoadBalancer: v1core.LoadBalancerStatus{ + Ingress: []v1core.LoadBalancerIngress{ + { + IP: "10.0.255.1", + }, + { + IP: "10.0.255.2", + }, + }, + }, + }, + }, + } + + tests := []struct { + name string + // cluster, external, loadbalancer + advertiseSettings [3]bool + serviceAdvertisedIPs []*ServiceAdvertisedIPs + }{ + { + "advertise all IPs", + [3]bool{true, true, true}, + []*ServiceAdvertisedIPs{ + { + services["cluster"], + []string{"10.0.0.1"}, + nil, + }, + { + services["external"], + []string{"10.0.0.1", "1.1.1.1"}, + nil, + }, + { + services["nodeport"], + []string{"10.0.0.1", "1.1.1.1"}, + nil, + }, + { + services["loadbalancer"], + []string{"10.0.0.1", "10.0.255.1", "10.0.255.2"}, + nil, + }, + }, + }, + { + "do not advertise any IPs", + [3]bool{false, false, false}, + []*ServiceAdvertisedIPs{ + { + services["cluster"], + []string{}, + nil, + }, + { + services["external"], + []string{}, + nil, + }, + { + services["nodeport"], + []string{}, + nil, + }, + { + services["loadbalancer"], + []string{}, + nil, + }, + }, + }, + { + "advertise cluster IPs", + [3]bool{true, false, false}, + []*ServiceAdvertisedIPs{ + { + services["cluster"], + []string{"10.0.0.1"}, + nil, + }, + { + services["external"], + []string{"10.0.0.1"}, + nil, + }, + { + services["nodeport"], + []string{"10.0.0.1"}, + nil, + }, + { + services["loadbalancer"], + []string{"10.0.0.1"}, + nil, + }, + }, + }, + { + "advertise external IPs", + [3]bool{false, true, false}, + []*ServiceAdvertisedIPs{ + { + services["cluster"], + []string{}, + nil, + }, + { + services["external"], + []string{"1.1.1.1"}, + nil, + }, + { + services["nodeport"], + []string{"1.1.1.1"}, + nil, + }, + { + services["loadbalancer"], + []string{}, + nil, + }, + }, + }, + { + "advertise loadbalancer IPs", + [3]bool{false, false, true}, + []*ServiceAdvertisedIPs{ + { + services["cluster"], + []string{}, + nil, + }, + { + services["external"], + []string{}, + nil, + }, + { + services["nodeport"], + []string{}, + nil, + }, + { + services["loadbalancer"], + []string{"10.0.255.1", "10.0.255.2"}, + nil, + }, + }, + }, + { + "opt in to advertise all IPs via annotations", + [3]bool{false, false, false}, + []*ServiceAdvertisedIPs{ + { + services["cluster"], + []string{"10.0.0.1"}, + map[string]string{ + svcAdvertiseClusterAnnotation: "true", + svcAdvertiseExternalAnnotation: "true", + svcAdvertiseLoadBalancerAnnotation: "true", + }, + }, + { + services["external"], + []string{"10.0.0.1", "1.1.1.1"}, + map[string]string{ + svcAdvertiseClusterAnnotation: "true", + svcAdvertiseExternalAnnotation: "true", + svcAdvertiseLoadBalancerAnnotation: "true", + }, + }, + { + services["nodeport"], + []string{"10.0.0.1", "1.1.1.1"}, + map[string]string{ + svcAdvertiseClusterAnnotation: "true", + svcAdvertiseExternalAnnotation: "true", + svcAdvertiseLoadBalancerAnnotation: "true", + }, + }, + { + services["loadbalancer"], + []string{"10.0.0.1", "10.0.255.1", "10.0.255.2"}, + map[string]string{ + svcAdvertiseClusterAnnotation: "true", + svcAdvertiseExternalAnnotation: "true", + svcAdvertiseLoadBalancerAnnotation: "true", + }, + }, + { + // Special case to test svcAdvertiseLoadBalancerAnnotation vs legacy svcSkipLbIpsAnnotation + services["loadbalancer"], + []string{"10.0.0.1"}, + map[string]string{ + svcAdvertiseClusterAnnotation: "true", + svcAdvertiseExternalAnnotation: "true", + svcAdvertiseLoadBalancerAnnotation: "true", + svcSkipLbIpsAnnotation: "true", + }, + }, + }, + }, + { + "opt out to advertise any IPs via annotations", + [3]bool{true, true, true}, + []*ServiceAdvertisedIPs{ + { + services["cluster"], + []string{}, + map[string]string{ + svcAdvertiseClusterAnnotation: "false", + svcAdvertiseExternalAnnotation: "false", + svcAdvertiseLoadBalancerAnnotation: "false", + }, + }, + { + services["external"], + []string{}, + map[string]string{ + svcAdvertiseClusterAnnotation: "false", + svcAdvertiseExternalAnnotation: "false", + svcAdvertiseLoadBalancerAnnotation: "false", + }, + }, + { + services["nodeport"], + []string{}, + map[string]string{ + svcAdvertiseClusterAnnotation: "false", + svcAdvertiseExternalAnnotation: "false", + svcAdvertiseLoadBalancerAnnotation: "false", + }, + }, + { + services["loadbalancer"], + []string{}, + map[string]string{ + svcAdvertiseClusterAnnotation: "false", + svcAdvertiseExternalAnnotation: "false", + svcAdvertiseLoadBalancerAnnotation: "false", + }, + }, + }, + }, + } + + for _, test := range tests { + nrc := NetworkRoutingController{} + t.Run(test.name, func(t *testing.T) { + nrc.advertiseClusterIP = test.advertiseSettings[0] + nrc.advertiseExternalIP = test.advertiseSettings[1] + nrc.advertiseLoadBalancerIP = test.advertiseSettings[2] + + for _, serviceAdvertisedIP := range test.serviceAdvertisedIPs { + clientset := fake.NewSimpleClientset() + + if serviceAdvertisedIP.annotations != nil { + serviceAdvertisedIP.service.ObjectMeta.Annotations = serviceAdvertisedIP.annotations + } + svc, _ := clientset.CoreV1().Services("default").Create(serviceAdvertisedIP.service) + advertisedIPs, _, _ := nrc.getVIPsForService(svc, false) + t.Logf("AdvertisedIPs: %v\n", advertisedIPs) + if !Equal(serviceAdvertisedIP.advertisedIPs, advertisedIPs) { + t.Errorf("Advertised IPs are incorrect, got: %v, want: %v.", serviceAdvertisedIP.advertisedIPs, advertisedIPs) + } + } + }) + } +} diff --git a/pkg/controllers/routing/network_routes_controller.go b/pkg/controllers/routing/network_routes_controller.go index b4b0278114..66799228f1 100644 --- a/pkg/controllers/routing/network_routes_controller.go +++ b/pkg/controllers/routing/network_routes_controller.go @@ -40,18 +40,24 @@ const ( podSubnetsIPSetName = "kube-router-pod-subnets" nodeAddrsIPSetName = "kube-router-node-ips" - nodeASNAnnotation = "kube-router.io/node.asn" - pathPrependASNAnnotation = "kube-router.io/path-prepend.as" - pathPrependRepeatNAnnotation = "kube-router.io/path-prepend.repeat-n" - peerASNAnnotation = "kube-router.io/peer.asns" - peerIPAnnotation = "kube-router.io/peer.ips" - peerPasswordAnnotation = "kube-router.io/peer.passwords" - peerPortAnnotation = "kube-router.io/peer.ports" - rrClientAnnotation = "kube-router.io/rr.client" - rrServerAnnotation = "kube-router.io/rr.server" - svcLocalAnnotation = "kube-router.io/service.local" - bgpLocalAddressAnnotation = "kube-router.io/bgp-local-addresses" - LeaderElectionRecordAnnotationKey = "control-plane.alpha.kubernetes.io/leader" + nodeASNAnnotation = "kube-router.io/node.asn" + pathPrependASNAnnotation = "kube-router.io/path-prepend.as" + pathPrependRepeatNAnnotation = "kube-router.io/path-prepend.repeat-n" + peerASNAnnotation = "kube-router.io/peer.asns" + peerIPAnnotation = "kube-router.io/peer.ips" + peerPasswordAnnotation = "kube-router.io/peer.passwords" + peerPortAnnotation = "kube-router.io/peer.ports" + rrClientAnnotation = "kube-router.io/rr.client" + rrServerAnnotation = "kube-router.io/rr.server" + svcLocalAnnotation = "kube-router.io/service.local" + bgpLocalAddressAnnotation = "kube-router.io/bgp-local-addresses" + svcAdvertiseClusterAnnotation = "kube-router.io/service.advertise.clusterip" + svcAdvertiseExternalAnnotation = "kube-router.io/service.advertise.externalip" + svcAdvertiseLoadBalancerAnnotation = "kube-router.io/service.advertise.loadbalancerip" + LeaderElectionRecordAnnotationKey = "control-plane.alpha.kubernetes.io/leader" + + // Deprecated: use kube-router.io/service.advertise.loadbalancer instead + svcSkipLbIpsAnnotation = "kube-router.io/service.skiplbips" ) // NetworkRoutingController is struct to hold necessary information required by controller diff --git a/pkg/controllers/routing/network_routes_controller_test.go b/pkg/controllers/routing/network_routes_controller_test.go index a29830a2ba..225c54d758 100644 --- a/pkg/controllers/routing/network_routes_controller_test.go +++ b/pkg/controllers/routing/network_routes_controller_test.go @@ -446,7 +446,7 @@ func Test_advertiseExternalIPs(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "svc-1", Annotations: map[string]string{ - "kube-router.io/service.skiplbips": "true", + svcSkipLbIpsAnnotation: "true", }, }, Spec: v1core.ServiceSpec{ @@ -520,6 +520,352 @@ func Test_advertiseExternalIPs(t *testing.T) { } } +func Test_advertiseAnnotationOptOut(t *testing.T) { + testcases := []struct { + name string + nrc *NetworkRoutingController + existingServices []*v1core.Service + // the key is the subnet from the watch event + watchEvents map[string]bool + }{ + { + "add bgp paths for all service IPs", + &NetworkRoutingController{ + bgpServer: gobgp.NewBgpServer(), + }, + []*v1core.Service{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "svc-1", + }, + Spec: v1core.ServiceSpec{ + Type: "ClusterIP", + ClusterIP: "10.0.0.1", + ExternalIPs: []string{"1.1.1.1"}, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "svc-2", + }, + Spec: v1core.ServiceSpec{ + Type: "NodePort", + ClusterIP: "10.0.0.2", + ExternalIPs: []string{"2.2.2.2", "3.3.3.3"}, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "svc-3", + }, + Spec: v1core.ServiceSpec{ + Type: "LoadBalancer", + ClusterIP: "10.0.0.3", + // ignored since LoadBalancer services don't + // advertise external IPs. + ExternalIPs: []string{"4.4.4.4"}, + }, + Status: v1core.ServiceStatus{ + LoadBalancer: v1core.LoadBalancerStatus{ + Ingress: []v1core.LoadBalancerIngress{ + { + IP: "10.0.255.1", + }, + { + IP: "10.0.255.2", + }, + }, + }, + }, + }, + }, + map[string]bool{ + "10.0.0.1/32": true, + "10.0.0.2/32": true, + "10.0.0.3/32": true, + "1.1.1.1/32": true, + "2.2.2.2/32": true, + "3.3.3.3/32": true, + "10.0.255.1/32": true, + "10.0.255.2/32": true, + }, + }, + { + "opt out to advertise any IPs via annotations", + &NetworkRoutingController{ + bgpServer: gobgp.NewBgpServer(), + }, + []*v1core.Service{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "svc-1", + Annotations: map[string]string{ + svcAdvertiseClusterAnnotation: "false", + svcAdvertiseExternalAnnotation: "false", + svcAdvertiseLoadBalancerAnnotation: "false", + }, + }, + Spec: v1core.ServiceSpec{ + Type: "LoadBalancer", + ClusterIP: "10.0.0.1", + ExternalIPs: []string{"1.1.1.1", "2.2.2.2"}, + }, + Status: v1core.ServiceStatus{ + LoadBalancer: v1core.LoadBalancerStatus{ + Ingress: []v1core.LoadBalancerIngress{ + { + IP: "10.0.255.1", + }, + { + IP: "10.0.255.2", + }, + }, + }, + }, + }, + }, + map[string]bool{}, + }, + } + + for _, testcase := range testcases { + t.Run(testcase.name, func(t *testing.T) { + go testcase.nrc.bgpServer.Serve() + err := testcase.nrc.bgpServer.Start(&config.Global{ + Config: config.GlobalConfig{ + As: 1, + RouterId: "10.0.0.0", + Port: 10000, + }, + }) + if err != nil { + t.Fatalf("failed to start BGP server: %v", err) + } + defer testcase.nrc.bgpServer.Stop() + w := testcase.nrc.bgpServer.Watch(gobgp.WatchBestPath(false)) + + clientset := fake.NewSimpleClientset() + startInformersForRoutes(testcase.nrc, clientset) + + err = createServices(clientset, testcase.existingServices) + if err != nil { + t.Fatalf("failed to create existing services: %v", err) + } + + waitForListerWithTimeout(testcase.nrc.svcLister, time.Second*10, t) + + // By default advertise all IPs + testcase.nrc.advertiseClusterIP = true + testcase.nrc.advertiseExternalIP = true + testcase.nrc.advertiseLoadBalancerIP = true + + toAdvertise, toWithdraw, _ := testcase.nrc.getActiveVIPs() + testcase.nrc.advertiseVIPs(toAdvertise) + testcase.nrc.withdrawVIPs(toWithdraw) + + watchEvents := waitForBGPWatchEventWithTimeout(time.Second*10, len(testcase.watchEvents), w, t) + for _, watchEvent := range watchEvents { + for _, path := range watchEvent.PathList { + if _, ok := testcase.watchEvents[path.GetNlri().String()]; ok { + continue + } else { + t.Errorf("got unexpected path: %v", path.GetNlri().String()) + } + } + } + }) + } +} + +func Test_advertiseAnnotationOptIn(t *testing.T) { + testcases := []struct { + name string + nrc *NetworkRoutingController + existingServices []*v1core.Service + // the key is the subnet from the watch event + watchEvents map[string]bool + }{ + { + "no bgp paths for any service IPs", + &NetworkRoutingController{ + bgpServer: gobgp.NewBgpServer(), + }, + []*v1core.Service{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "svc-1", + }, + Spec: v1core.ServiceSpec{ + Type: "ClusterIP", + ClusterIP: "10.0.0.1", + ExternalIPs: []string{"1.1.1.1"}, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "svc-2", + }, + Spec: v1core.ServiceSpec{ + Type: "NodePort", + ClusterIP: "10.0.0.2", + ExternalIPs: []string{"2.2.2.2", "3.3.3.3"}, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "svc-3", + }, + Spec: v1core.ServiceSpec{ + Type: "LoadBalancer", + ClusterIP: "10.0.0.3", + // ignored since LoadBalancer services don't + // advertise external IPs. + ExternalIPs: []string{"4.4.4.4"}, + }, + Status: v1core.ServiceStatus{ + LoadBalancer: v1core.LoadBalancerStatus{ + Ingress: []v1core.LoadBalancerIngress{ + { + IP: "10.0.255.1", + }, + { + IP: "10.0.255.2", + }, + }, + }, + }, + }, + }, + map[string]bool{}, + }, + { + "opt in to advertise all IPs via annotations", + &NetworkRoutingController{ + bgpServer: gobgp.NewBgpServer(), + }, + []*v1core.Service{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "svc-1", + Annotations: map[string]string{ + svcAdvertiseClusterAnnotation: "true", + svcAdvertiseExternalAnnotation: "true", + svcAdvertiseLoadBalancerAnnotation: "true", + }, + }, + Spec: v1core.ServiceSpec{ + Type: "ClusterIP", + ClusterIP: "10.0.0.1", + ExternalIPs: []string{"1.1.1.1"}, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "svc-2", + Annotations: map[string]string{ + svcAdvertiseClusterAnnotation: "true", + svcAdvertiseExternalAnnotation: "true", + svcAdvertiseLoadBalancerAnnotation: "true", + }, + }, + Spec: v1core.ServiceSpec{ + Type: "NodePort", + ClusterIP: "10.0.0.2", + ExternalIPs: []string{"2.2.2.2", "3.3.3.3"}, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "svc-3", + Annotations: map[string]string{ + svcAdvertiseClusterAnnotation: "true", + svcAdvertiseExternalAnnotation: "true", + svcAdvertiseLoadBalancerAnnotation: "true", + }, + }, + Spec: v1core.ServiceSpec{ + Type: "LoadBalancer", + ClusterIP: "10.0.0.3", + // ignored since LoadBalancer services don't + // advertise external IPs. + ExternalIPs: []string{"4.4.4.4"}, + }, + Status: v1core.ServiceStatus{ + LoadBalancer: v1core.LoadBalancerStatus{ + Ingress: []v1core.LoadBalancerIngress{ + { + IP: "10.0.255.1", + }, + { + IP: "10.0.255.2", + }, + }, + }, + }, + }, + }, + map[string]bool{ + "10.0.0.1/32": true, + "10.0.0.2/32": true, + "10.0.0.3/32": true, + "1.1.1.1/32": true, + "2.2.2.2/32": true, + "3.3.3.3/32": true, + "10.0.255.1/32": true, + "10.0.255.2/32": true, + }, + }, + } + + for _, testcase := range testcases { + t.Run(testcase.name, func(t *testing.T) { + go testcase.nrc.bgpServer.Serve() + err := testcase.nrc.bgpServer.Start(&config.Global{ + Config: config.GlobalConfig{ + As: 1, + RouterId: "10.0.0.0", + Port: 10000, + }, + }) + if err != nil { + t.Fatalf("failed to start BGP server: %v", err) + } + defer testcase.nrc.bgpServer.Stop() + w := testcase.nrc.bgpServer.Watch(gobgp.WatchBestPath(false)) + + clientset := fake.NewSimpleClientset() + startInformersForRoutes(testcase.nrc, clientset) + + err = createServices(clientset, testcase.existingServices) + if err != nil { + t.Fatalf("failed to create existing services: %v", err) + } + + waitForListerWithTimeout(testcase.nrc.svcLister, time.Second*10, t) + + // By default do not advertise any IPs + testcase.nrc.advertiseClusterIP = false + testcase.nrc.advertiseExternalIP = false + testcase.nrc.advertiseLoadBalancerIP = false + + toAdvertise, toWithdraw, _ := testcase.nrc.getActiveVIPs() + testcase.nrc.advertiseVIPs(toAdvertise) + testcase.nrc.withdrawVIPs(toWithdraw) + + watchEvents := waitForBGPWatchEventWithTimeout(time.Second*10, len(testcase.watchEvents), w, t) + for _, watchEvent := range watchEvents { + for _, path := range watchEvent.PathList { + if _, ok := testcase.watchEvents[path.GetNlri().String()]; ok { + continue + } else { + t.Errorf("got unexpected path: %v", path.GetNlri().String()) + } + } + } + }) + } +} + func Test_nodeHasEndpointsForService(t *testing.T) { testcases := []struct { name string