implement per-service annotations to control IP advertisment

docs/user-guide.md

@@ -124,6 +124,39 @@ and if you want to move back to kube-proxy then clean up config done by kube-rou
 and run kube-proxy with the configuration you have.
 - [General Setup](/README.md#getting-started)
 
+
+## Advertising IPs
+
+kube-router can advertise Cluster, External and LoadBalancer IPs to BGP peers.
+It does this by:
+* locally adding the advertised IPs to the nodes' `kube-dummy-if` network interface
+* advertising the IPs to its BGP peers
+
+To set the default for all services use the `--advertise-cluster-ip`,
+`--advertise-external-ip` and `--advertise-loadbalancer-ip` flags.
+
+To selectively enable or disable this feature per-service use the
+`kube-router.io/service.advertise.cluster`, `kube-router.io/service.advertise.external`
+and `kube-router.io/service.advertise.loadbalancer` annotations.
+
+e.g.:
+`$ kubectl annotate service my-advertised-service "kube-router.io/service.advertise.cluster=true"`
+`$ kubectl annotate service my-advertised-service "kube-router.io/service.advertise.external=true"`
+`$ kubectl annotate service my-advertised-service "kube-router.io/service.advertise.loadbalancer=true"`
+
+`$ kubectl annotate service my-non-advertised-service "kube-router.io/service.advertise.cluster=false"`
+`$ kubectl annotate service my-non-advertised-service "kube-router.io/service.advertise.external=false"`
+`$ kubectl annotate service my-non-advertised-service "kube-router.io/service.advertise.loadbalancer=false"`
+
+By combining the flags with the per-service annotations you can choose either
+a opt-in or opt-out strategy for advertising IPs.
+
+Advertising LoadBalancer IPs works by inspecting the services
+`status.loadBalancer.ingress` IPs that are set by external LoadBalancers like
+for example MetalLb. This has been successfully tested together with
+[MetalLB](https://github.com/google/metallb) in ARP mode.
+
+
 ## Hairpin Mode
 
 Communication from a Pod that is behind a Service to its own ClusterIP:Port is
@@ -206,25 +239,6 @@ For destination hashing scheduling use:
 kubectl annotate service my-service "kube-router.io/service.scheduler=dh"
 ```
 
-## LoadBalancer IPs
-
-If you want to also advertise loadbalancer set IPs
-(`status.loadBalancer.ingress` IPs), e.g. when using it with MetalLb,
-add the `--advertise-loadbalancer-ip` flag (`false` by default).
-
-To selectively disable this behaviour per-service, you can use
-the `kube-router.io/service.skiplbips` annotation as e.g.:
-`$ kubectl annotate service my-external-service "kube-router.io/service.skiplbips=true"`
-
-In concrete, unless the Service is annotated as per above, the
-`--advertise-loadbalancer-ip` flag will make Service's Ingress IP(s)
-set by the LoadBalancer to:
-* be locally added to nodes' `kube-dummy-if` network interface
-* be advertised to BGP peers
-
-FYI Above has been successfully tested together with
-[MetalLB](https://github.com/google/metallb) in ARP mode.
-
 ## HostPort support
 
 If you would like to use `HostPort` functionality below changes are required in the manifest.

pkg/controllers/routing/ecmp_vip.go

@@ -268,12 +268,9 @@ func (nrc *NetworkRoutingController) getLoadBalancerIps(svc *v1core.Service) []s
 	if svc.Spec.Type == "LoadBalancer" {
 		// skip headless services
 		if svc.Spec.ClusterIP != "None" && svc.Spec.ClusterIP != "" {
-			_, skiplbips := svc.ObjectMeta.Annotations["kube-router.io/service.skiplbips"]
-			if !skiplbips {
-				for _, lbIngress := range svc.Status.LoadBalancer.Ingress {
-					if len(lbIngress.IP) > 0 {
-						loadBalancerIpList = append(loadBalancerIpList, lbIngress.IP)
-					}
+			for _, lbIngress := range svc.Status.LoadBalancer.Ingress {
+				if len(lbIngress.IP) > 0 {
+					loadBalancerIpList = append(loadBalancerIpList, lbIngress.IP)
 				}
 			}
 		}
@@ -313,6 +310,16 @@ func (nrc *NetworkRoutingController) getVIPs(onlyActiveEndpoints bool) ([]string
 	return toAdvertiseList, toWithdrawList, nil
 }
 
+func (nrc *NetworkRoutingController) shouldAdvertiseService(svc *v1core.Service, annotation string, defaultValue bool) bool {
+	returnValue := defaultValue
+	stringValue, exists := svc.Annotations[annotation]
+	if exists {
+		// Service annotations overrides defaults.
+		returnValue, _ = strconv.ParseBool(stringValue)
+	}
+	return returnValue
+}
+
 func (nrc *NetworkRoutingController) getVIPsForService(svc *v1core.Service, onlyActiveEndpoints bool) ([]string, []string, error) {
 	ipList := make([]string, 0)
 	var err error
@@ -328,16 +335,21 @@ func (nrc *NetworkRoutingController) getVIPsForService(svc *v1core.Service, only
 		}
 	}
 
-	if nrc.advertiseClusterIP {
+	if nrc.shouldAdvertiseService(svc, svcAdvertiseClusterAnnotation, nrc.advertiseClusterIP) {
 		clusterIp := nrc.getClusterIp(svc)
 		if clusterIp != "" {
 			ipList = append(ipList, clusterIp)
 		}
 	}
-	if nrc.advertiseExternalIP {
+
+	if nrc.shouldAdvertiseService(svc, svcAdvertiseExternalAnnotation, nrc.advertiseExternalIP) {
 		ipList = append(ipList, nrc.getExternalIps(svc)...)
 	}
-	if nrc.advertiseLoadBalancerIP {
+
+	// Deprecated: Use service.advertise.loadbalancer=false instead of service.skiplbips.
+	_, skiplbips := svc.Annotations[svcSkipLbIpsAnnotation]
+	advertiseLoadBalancer := nrc.shouldAdvertiseService(svc, svcAdvertiseLoadBalancerAnnotation, nrc.advertiseLoadBalancerIP)
+	if advertiseLoadBalancer && !skiplbips {
 		ipList = append(ipList, nrc.getLoadBalancerIps(svc)...)
 	}