Skip to content

Commit

Permalink
fix(ecmp_vip): handle ipv4 & ipv6 protocols
Browse files Browse the repository at this point in the history
  • Loading branch information
@aauren
aauren committed Jan 23, 2023
1 parent 437da14 commit 6a1bab2
Show file tree
Hide file tree
Showing 4 changed files with 77 additions and 29 deletions.
4 changes: 1 addition & 3 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@ module github.com/cloudnativelabs/kube-router

require (
github.com/aws/aws-sdk-go v1.44.171
github.com/containernetworking/cni v1.1.2
github.com/containernetworking/plugins v1.1.1
github.com/coreos/go-iptables v0.6.0
github.com/docker/docker v20.10.23+incompatible
github.com/moby/ipvs v1.1.0
Expand All @@ -15,6 +13,7 @@ require (
github.com/stretchr/testify v1.8.1
github.com/vishvananda/netlink v1.2.1-beta.2
github.com/vishvananda/netns v0.0.2
golang.org/x/exp v0.0.0-20221230185412-738e83a70c30
golang.org/x/net v0.5.0
google.golang.org/grpc v1.51.0
google.golang.org/protobuf v1.28.1
Expand Down Expand Up @@ -77,7 +76,6 @@ require (
github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/common v0.37.0 // indirect
github.com/prometheus/procfs v0.8.0 // indirect
github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1 // indirect
github.com/sirupsen/logrus v1.9.0 // indirect
github.com/spf13/afero v1.9.2 // indirect
github.com/spf13/cast v1.5.0 // indirect
Expand Down
13 changes: 2 additions & 11 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,10 +85,6 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX
github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/containernetworking/cni v1.1.2 h1:wtRGZVv7olUHMOqouPpn3cXJWpJgM6+EUl31EQbXALQ=
github.com/containernetworking/cni v1.1.2/go.mod h1:sDpYKmGVENF3s6uvMvGgldDWeG8dMxakj/u+i9ht9vw=
github.com/containernetworking/plugins v1.1.1 h1:+AGfFigZ5TiQH00vhR8qPeSatj53eNGz0C1d3wVYlHE=
github.com/containernetworking/plugins v1.1.1/go.mod h1:Sr5TH/eBsGLXK/h71HeLfX19sZPp3ry5uHSkI4LPxV8=
github.com/coreos/go-iptables v0.6.0 h1:is9qnZMPYjLd8LYqmm/qlE+wwEgJIkTYdhV3rfZo4jk=
github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
Expand Down Expand Up @@ -230,7 +226,6 @@ github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLe
github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
Expand Down Expand Up @@ -324,15 +319,12 @@ github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=
github.com/onsi/ginkgo/v2 v2.7.0 h1:/XxtEV3I3Eif/HobnVx9YmJgk8ENdRsuUmM+fLCFNow=
github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y=
github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
Expand Down Expand Up @@ -382,8 +374,6 @@ github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0ua
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1 h1:ZFfeKAhIQiiOrQaI3/znw0gOmYpO28Tcu1YaqMa/jtQ=
github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
Expand Down Expand Up @@ -460,6 +450,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
golang.org/x/exp v0.0.0-20221230185412-738e83a70c30 h1:m9O6OTJ627iFnN2JIWfdqlZCzneRO6EEBsHXI25P8ws=
golang.org/x/exp v0.0.0-20221230185412-738e83a70c30/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
Expand Down Expand Up @@ -525,7 +517,6 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
Expand Down
54 changes: 39 additions & 15 deletions pkg/controllers/routing/ecmp_vip.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import (
"fmt"
"strconv"

"golang.org/x/exp/maps"

"google.golang.org/protobuf/types/known/anypb"

"github.com/cloudnativelabs/kube-router/pkg/utils"
Expand All @@ -21,24 +23,27 @@ import (

// bgpAdvertiseVIP advertises the service vip (cluster ip or load balancer ip or external IP) the configured peers
func (nrc *NetworkRoutingController) bgpAdvertiseVIP(vip string) error {
subnet, nh, afiFamily, err := nrc.getBGPRouteInfoForVIP(vip)
if err != nil {
return fmt.Errorf("unable to advertise VIP because of: %v", err)
}

klog.V(2).Infof("Advertising route: '%s/%s via %s' to peers",
vip, strconv.Itoa(32), nrc.primaryIP.String())
klog.V(2).Infof("Advertising route: '%s/%d via %s' to peers", vip, subnet, nh)

a1, _ := anypb.New(&gobgpapi.OriginAttribute{
Origin: 0,
})
a2, _ := anypb.New(&gobgpapi.NextHopAttribute{
NextHop: nrc.primaryIP.String(),
NextHop: nh,
})
attrs := []*anypb.Any{a1, a2}
nlri1, _ := anypb.New(&gobgpapi.IPAddressPrefix{
Prefix: vip,
PrefixLen: 32,
PrefixLen: subnet,
})
_, err := nrc.bgpServer.AddPath(context.Background(), &gobgpapi.AddPathRequest{
_, err = nrc.bgpServer.AddPath(context.Background(), &gobgpapi.AddPathRequest{
Path: &gobgpapi.Path{
Family: &gobgpapi.Family{Afi: gobgpapi.Family_AFI_IP, Safi: gobgpapi.Family_SAFI_UNICAST},
Family: &gobgpapi.Family{Afi: afiFamily, Safi: gobgpapi.Family_SAFI_UNICAST},
Nlri: nlri1,
Pattrs: attrs,
},
Expand All @@ -47,28 +52,32 @@ func (nrc *NetworkRoutingController) bgpAdvertiseVIP(vip string) error {
return err
}

// bgpWithdrawVIP unadvertises the service vip
// bgpWithdrawVIP un-advertises the service vip
func (nrc *NetworkRoutingController) bgpWithdrawVIP(vip string) error {
klog.V(2).Infof("Withdrawing route: '%s/%s via %s' to peers",
vip, strconv.Itoa(32), nrc.primaryIP.String())
subnet, nh, afiFamily, err := nrc.getBGPRouteInfoForVIP(vip)
if err != nil {
return fmt.Errorf("unable to advertise VIP because of: %v", err)
}

klog.V(2).Infof("Withdrawing route: '%s/%d via %s' to peers", vip, subnet, nh)

a1, _ := anypb.New(&gobgpapi.OriginAttribute{
Origin: 0,
})
a2, _ := anypb.New(&gobgpapi.NextHopAttribute{
NextHop: nrc.primaryIP.String(),
NextHop: nh,
})
attrs := []*anypb.Any{a1, a2}
nlri, _ := anypb.New(&gobgpapi.IPAddressPrefix{
Prefix: vip,
PrefixLen: 32,
PrefixLen: subnet,
})
path := gobgpapi.Path{
Family: &gobgpapi.Family{Afi: gobgpapi.Family_AFI_IP, Safi: gobgpapi.Family_SAFI_UNICAST},
Family: &gobgpapi.Family{Afi: afiFamily, Safi: gobgpapi.Family_SAFI_UNICAST},
Nlri: nlri,
Pattrs: attrs,
}
err := nrc.bgpServer.DeletePath(context.Background(), &gobgpapi.DeletePathRequest{
err = nrc.bgpServer.DeletePath(context.Background(), &gobgpapi.DeletePathRequest{
TableType: gobgpapi.TableType_GLOBAL,
Path: &path,
})
Expand Down Expand Up @@ -515,15 +524,30 @@ func (nrc *NetworkRoutingController) nodeHasEndpointsForService(svc *v1core.Serv
return false, errors.New("failed to convert cache item to Endpoints type")
}

// Find all the IPs that this node has on it so that we can use it to compare it against endpoint IPs
allNodeIPs := make([]string, 0)
for _, ips := range maps.Values(nrc.nodeIPv4Addrs) {
for _, ip := range ips {
allNodeIPs = append(allNodeIPs, ip.String())
}
}
for _, ips := range maps.Values(nrc.nodeIPv6Addrs) {
for _, ip := range ips {
allNodeIPs = append(allNodeIPs, ip.String())
}
}

for _, subset := range ep.Subsets {
for _, address := range subset.Addresses {
if address.NodeName != nil {
if *address.NodeName == nrc.nodeName {
return true, nil
}
} else {
if address.IP == nrc.primaryIP.String() {
return true, nil
for _, nodeIP := range allNodeIPs {
if address.IP == nodeIP {
return true, nil
}
}
}
}
Expand Down
35 changes: 35 additions & 0 deletions pkg/controllers/routing/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"strconv"
"strings"

"github.com/cloudnativelabs/kube-router/pkg/utils"
gobgpapi "github.com/osrg/gobgp/v3/api"
"github.com/osrg/gobgp/v3/pkg/packet/bgp"
"github.com/vishvananda/netlink/nl"
Expand Down Expand Up @@ -258,3 +259,37 @@ func getPodCIDRsFromAllNodeSources(node *v1core.Node) (podCIDRs []string) {
// Finally, if all else fails, use the PodCIDRs on the node spec
return node.Spec.PodCIDRs
}

// getBGPRouteInfoForVIP attempt to automatically find the subnet, BGP AFI/SAFI Family, and nexthop for a given VIP
// based upon whether it is an IPv4 address or an IPv6 address. Returns slash notation subnet as uint32 suitable for
// sending to GoBGP and an error if it is unable to determine the subnet automatically
func (nrc *NetworkRoutingController) getBGPRouteInfoForVIP(vip string) (subnet uint32, nh string,
afiFamily gobgpapi.Family_Afi, err error) {
ip := net.ParseIP(vip)
if ip == nil {
err = fmt.Errorf("could not parse VIP: %s", vip)
return
}
if ip.To4() != nil {
subnet = 32
afiFamily = gobgpapi.Family_AFI_IP
nhIP := utils.FindBestIPv4NodeAddress(nrc.primaryIP, nrc.nodeIPv4Addrs)
if nhIP == nil {
err = fmt.Errorf("could not find an IPv4 address on node to set as nexthop for vip: %s", vip)
}
nh = nhIP.String()
return
}
if ip.To16() != nil {
subnet = 128
afiFamily = gobgpapi.Family_AFI_IP6
nhIP := utils.FindBestIPv6NodeAddress(nrc.primaryIP, nrc.nodeIPv6Addrs)
if nhIP == nil {
err = fmt.Errorf("could not find an IPv6 address on node to set as nexthop for vip: %s", vip)
}
nh = nhIP.String()
return
}
err = fmt.Errorf("could not convert IP to IPv4 or IPv6, unable to find subnet for: %s", vip)
return
}

0 comments on commit 6a1bab2

Please sign in to comment.