Skip to content

Commit

Permalink
Use ipset-go instead of calling the ipset binary
Browse files Browse the repository at this point in the history 
This reduces our requirements on the environment in which Submariner
components run.

The ipset binaries are kept in the container images, for debugging
purposes and because subctl gather relies on ipset in the route
agent.

Signed-off-by: Stephen Kitt <[email protected]>
  • Loading branch information
@skitt
skitt committed Apr 25, 2023
1 parent d688d53 commit 3535bee
Show file tree
Hide file tree
Showing 10 changed files with 220 additions and 697 deletions.
3 changes: 2 additions & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ require (
github.com/coreos/go-iptables v0.6.0
github.com/emirpasic/gods v1.18.1
github.com/kelseyhightower/envconfig v1.4.0
github.com/lrh3321/ipset-go v0.0.0-20230425010353-0d9880b1ecac
github.com/onsi/ginkgo/v2 v2.9.2
github.com/onsi/gomega v1.27.6
github.com/ovn-org/libovsdb v0.6.1-0.20220427123326-d7b273399db4
Expand Down Expand Up @@ -86,7 +87,7 @@ require (
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/testify v1.8.0 // indirect
github.com/urfave/cli/v2 v2.4.0 // indirect
github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae // indirect
github.com/vishvananda/netns v0.0.4 // indirect
golang.org/x/crypto v0.1.0 // indirect
golang.org/x/net v0.8.0 // indirect
golang.org/x/oauth2 v0.5.0 // indirect
Expand Down
7 changes: 6 additions & 1 deletion go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -321,6 +321,8 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/lib/pq v0.0.0-20180327071824-d34b9ff171c2/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lrh3321/ipset-go v0.0.0-20230425010353-0d9880b1ecac h1:cYSHLRAFFmEugFMDyup/b/b8nlI1rVvI3mXfpa9HWCw=
github.com/lrh3321/ipset-go v0.0.0-20230425010353-0d9880b1ecac/go.mod h1:t1CW3V84GWqI2sqZIlriuy8OLGLMikvFX9pWI19mPbc=
github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=
github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
Expand Down Expand Up @@ -525,8 +527,9 @@ github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYp
github.com/vishvananda/netlink v1.2.1-beta.2 h1:Llsql0lnQEbHj0I1OuKyp8otXp0r3q0mPkuhwHfStVs=
github.com/vishvananda/netlink v1.2.1-beta.2/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=
github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae h1:4hwBBUfQCFe3Cym0ZtKyq7L16eZUtYKs+BaHDN6mAns=
github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8=
github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
Expand Down Expand Up @@ -660,6 +663,8 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
Expand Down
6 changes: 4 additions & 2 deletions pkg/globalnet/controllers/egress_pod_watcher.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,9 @@ package controllers

import (
"fmt"
"net"

ipsetgo "github.com/lrh3321/ipset-go"
"github.com/pkg/errors"
"github.com/submariner-io/admiral/pkg/log"
"github.com/submariner-io/admiral/pkg/watcher"
Expand Down Expand Up @@ -95,7 +97,7 @@ func (w *egressPodWatcher) onCreateOrUpdate(obj runtime.Object, _ int) bool {

logger.V(log.DEBUG).Infof("Pod %q with IP %s created/updated", key, pod.Status.PodIP)

if err := w.namedIPSet.AddEntry(pod.Status.PodIP, true); err != nil {
if err := w.namedIPSet.AddEntry(&ipsetgo.Entry{IP: net.ParseIP(pod.Status.PodIP)}, true); err != nil {
logger.Errorf(err, "Error adding pod IP %q to IP set %q", pod.Status.PodIP, w.ipSetName)
return true
}
Expand All @@ -109,7 +111,7 @@ func (w *egressPodWatcher) onDelete(obj runtime.Object, _ int) bool {

logger.V(log.DEBUG).Infof("Pod %q removed", key)

if err := w.namedIPSet.DelEntry(pod.Status.PodIP); err != nil {
if err := w.namedIPSet.DelEntry(&ipsetgo.Entry{IP: net.ParseIP(pod.Status.PodIP)}); err != nil {
logger.Errorf(err, "Error deleting pod IP %q from IP set %q", pod.Status.PodIP, w.ipSetName)
return true
}
Expand Down
13 changes: 8 additions & 5 deletions pkg/globalnet/controllers/global_egressip_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"encoding/base32"
"fmt"

ipsetgo "github.com/lrh3321/ipset-go"
"github.com/pkg/errors"
"github.com/submariner-io/admiral/pkg/federate"
"github.com/submariner-io/admiral/pkg/syncer"
Expand All @@ -39,7 +40,6 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/tools/cache"
utilexec "k8s.io/utils/exec"
)

func NewGlobalEgressIPController(config *syncer.ResourceSyncerConfig, pool *ipam.IPPool) (Interface, error) {
Expand All @@ -63,7 +63,10 @@ func NewGlobalEgressIPController(config *syncer.ResourceSyncerConfig, pool *ipam
},
}

controller.ipSetIface = ipset.New(utilexec.New())
controller.ipSetIface, err = ipset.New()
if err != nil {
return nil, errors.WithMessage(err, "error creating the ipset handler")
}

_, gvr, err := util.ToUnstructuredResource(&submarinerv1.GlobalEgressIP{}, config.RestMapper)
if err != nil {
Expand Down Expand Up @@ -383,8 +386,8 @@ func (c *globalEgressIPController) flushGlobalEgressRulesAndReleaseIPs(key, ipSe
}

func (c *globalEgressIPController) newNamedIPSet(key string) ipset.Named {
return ipset.NewNamed(&ipset.IPSet{
Name: c.getIPSetName(key),
SetType: ipset.HashIP,
return ipset.NewNamed(&ipsetgo.Sets{
SetName: c.getIPSetName(key),
TypeName: ipsetgo.TypeHashIP,
}, c.ipSetIface)
}
6 changes: 4 additions & 2 deletions pkg/globalnet/controllers/uninstall.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@ import (
"k8s.io/client-go/kubernetes/scheme"
rest "k8s.io/client-go/rest"
"k8s.io/client-go/util/retry"
utilexec "k8s.io/utils/exec"
)

func UninstallDataPath() {
Expand Down Expand Up @@ -87,7 +86,10 @@ func UninstallDataPath() {
}
}

ipsetIface := ipset.New(utilexec.New())
ipsetIface, err := ipset.New()
if err != nil {
logger.FatalOnError(err, "Error initializing ipsets")
}

ipSetList, err := ipsetIface.ListSets()
if err != nil {
Expand Down
85 changes: 26 additions & 59 deletions pkg/ipset/fake/ipset.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"sync"
"time"

ipsetgo "github.com/lrh3321/ipset-go"
. "github.com/onsi/gomega"
"github.com/submariner-io/submariner/pkg/ipset"
"k8s.io/apimachinery/pkg/util/sets"
Expand All @@ -47,24 +48,24 @@ func New() *IPSet {
}
}

func (i *IPSet) CreateSet(set *ipset.IPSet, ignoreExistErr bool) error {
func (i *IPSet) CreateSet(set *ipsetgo.Sets, ignoreExistErr bool) error {
i.mutex.Lock()
defer i.mutex.Unlock()

err := matchForError(&i.failOnCreateSetMatchers, set.Name)
err := matchForError(&i.failOnCreateSetMatchers, set.SetName)
if err != nil {
return err
}

if i.sets[set.Name] != nil {
if i.sets[set.SetName] != nil {
if ignoreExistErr {
return nil
}

return fmt.Errorf("IP set %q already exists", set.Name)
return fmt.Errorf("IP set %q already exists", set.SetName)
}

i.sets[set.Name] = sets.New[string]()
i.sets[set.SetName] = sets.New[string]()

return nil
}
Expand Down Expand Up @@ -101,43 +102,47 @@ func (i *IPSet) DestroySet(set string) error {
return nil
}

func (i *IPSet) DestroyAllSets() error {
i.mutex.Lock()
defer i.mutex.Unlock()

i.sets = map[string]sets.Set[string]{}
func entryToString(entry *ipsetgo.Entry) string {
// hash IP or hash net only
if entry.CIDR > 0 {
return fmt.Sprintf("%s/%d", entry.IP.String(), entry.CIDR)
}

return nil
return entry.IP.String()
}

func (i *IPSet) AddEntry(entry string, set *ipset.IPSet, ignoreExistErr bool) error {
func (i *IPSet) AddEntry(entry *ipsetgo.Entry, set *ipsetgo.Sets, ignoreExistErr bool) error {
i.mutex.Lock()
defer i.mutex.Unlock()

err := matchForError(&i.failOnAddEntryMatchers, entry)
entryString := entryToString(entry)

err := matchForError(&i.failOnAddEntryMatchers, entryString)
if err != nil {
return err
}

entries := i.sets[set.Name]
entries := i.sets[set.SetName]
if entries == nil {
return fmt.Errorf("IP set %q does not exist", set.Name)
return fmt.Errorf("IP set %q does not exist", set.SetName)
}

if entries.Has(entry) && !ignoreExistErr {
return fmt.Errorf("entry %q already exists", entry)
if entries.Has(entryString) && !ignoreExistErr {
return fmt.Errorf("entry %q already exists", entryString)
}

entries.Insert(entry)
entries.Insert(entryString)

return nil
}

func (i *IPSet) DelEntry(entry, set string) error {
func (i *IPSet) DelEntry(entry *ipsetgo.Entry, set string) error {
i.mutex.Lock()
defer i.mutex.Unlock()

err := matchForError(&i.failOnDelEntryMatchers, entry)
entryString := entryToString(entry)

err := matchForError(&i.failOnDelEntryMatchers, entryString)
if err != nil {
return err
}
Expand All @@ -147,23 +152,11 @@ func (i *IPSet) DelEntry(entry, set string) error {
return nil
}

entries.Delete(entry)
entries.Delete(entryString)

return nil
}

func (i *IPSet) TestEntry(entry, set string) (bool, error) {
i.mutex.Lock()
defer i.mutex.Unlock()

entries := i.sets[set]
if entries == nil {
return false, fmt.Errorf("IP set %q does not exist", set)
}

return entries.Has(entry), nil
}

func (i *IPSet) ListEntries(set string) ([]string, error) {
i.mutex.Lock()
defer i.mutex.Unlock()
Expand All @@ -189,32 +182,6 @@ func (i *IPSet) ListSets() ([]string, error) {
return names, nil
}

func (i *IPSet) GetVersion() (string, error) {
return "v7.6", nil
}

func (i *IPSet) AddEntryWithOptions(entry *ipset.Entry, set *ipset.IPSet, _ bool) error {
i.mutex.Lock()
defer i.mutex.Unlock()

entries := i.sets[set.Name]
if entries == nil {
return fmt.Errorf("IP set %q does not exist", set)
}

entries.Insert(entry.String())

return nil
}

func (i *IPSet) DelEntryWithOptions(set, entry string, _ ...string) error {
return i.DelEntry(set, entry)
}

func (i *IPSet) ListAllSetInfo() (string, error) {
return "", nil
}

func (i *IPSet) AwaitSet(stringOrMatcher interface{}) {
Eventually(func() []string {
s, _ := i.ListSets()
Expand Down
Loading

0 comments on commit 3535bee

Please sign in to comment.