use ipvs library instead of ipvsadm-save & reduce collector json file…

… size

Signed-off-by: xiayu.lyt <[email protected]>

go.mod

@@ -21,6 +21,7 @@ require (
 	github.com/google/uuid v1.3.0
 	github.com/gorilla/mux v1.8.0
 	github.com/mdlayher/netlink v1.7.1
+	github.com/moby/ipvs v1.1.0
 	github.com/onsi/ginkgo/v2 v2.9.1
 	github.com/onsi/gomega v1.27.4
 	github.com/patrickmn/go-cache v2.1.0+incompatible

go.sum

@@ -786,6 +786,8 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A=
+github.com/moby/ipvs v1.1.0 h1:ONN4pGaZQgAx+1Scz5RvWV4Q7Gb+mvfRh3NsPS+1XQQ=
+github.com/moby/ipvs v1.1.0/go.mod h1:4VJMWuf098bsUMmZEiD4Tjk/O7mOn3l1PTD3s4OoYAs=
 github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg=
 github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc=
 github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8=

pkg/skoop/assertions/netstack.go

@@ -209,12 +209,6 @@ func (na *NetstackAssertion) AssertVEthOnBridge(index int, expectedBridgeName st
 	// todo: br_port_state
 }
 
-func (na *NetstackAssertion) AssertIPVSServiceExists(service, servicePort, protocol string) {
-	key := fmt.Sprintf("%s:%s:%s", protocol, service, servicePort)
-	AssertTrue(na, slices.Contains(na.netns.NetNSInfo.IPVSInfo, key), model.SuspicionLevelWarning,
-		fmt.Sprintf("ipvs has no service %s", key))
-}
-
 type RouteAssertion struct {
 	Dev      *string
 	Scope    *netstack.Scope
@@ -464,7 +458,7 @@ func (na *NetstackAssertion) AssertNetfilterServe(pktIn model.Packet, iif string
 func (na *NetstackAssertion) AssertIPVSServerExists(service string, servicePort uint16, protocol model.Protocol,
 	backend string, backendPort uint16) {
 	key := fmt.Sprintf("%s:%s:%d", protocol, service, servicePort)
-	_, ok := lo.Find(na.netns.NetNSInfo.IPVSInfo, func(i string) bool { return i == key })
+	_, ok := na.netns.NetNSInfo.IPVSInfo[key]
 	if !ok {
 		return
 	}

pkg/skoop/collector/manager/manager.go

@@ -172,12 +172,12 @@ func (m *simplePodCollectorManager) buildCache(nodeName string) error {
 	}
 
 	nodeInfo.Router = netstack.NewSimulateRouter(nodeInfo.NetNSInfo.RuleInfo, nodeInfo.NetNSInfo.RouteInfo, nodeInfo.NetNSInfo.Interfaces)
-	nodeInfo.IPVS, err = netstack.ParseIPVS(nodeInfo.NetNSInfo.IPVSInfo)
+	nodeInfo.IPVS = netstack.NewIPVS(nodeInfo.NetNSInfo.IPVSInfo)
 	nodeInfo.IPTables = netstack.ParseIPTables(nodeInfo.NetNSInfo.IptablesInfo)
 	if err != nil {
 		return err
 	}
-	nodeInfo.IPSetManager, err = netstack.ParseIPSet(nodeInfo.NetNSInfo.IpsetInfo)
+	nodeInfo.IPSetManager, err = netstack.NewIPSetManager(nodeInfo.NetNSInfo.IpsetInfo)
 	if err != nil {
 		return err
 	}
@@ -212,12 +212,12 @@ func (m *simplePodCollectorManager) buildCache(nodeName string) error {
 			podInfo.NetNSInfo = &podNetNS
 		}
 
-		podInfo.IPVS, err = netstack.ParseIPVS(podInfo.NetNSInfo.IPVSInfo)
+		podInfo.IPVS = netstack.NewIPVS(podInfo.NetNSInfo.IPVSInfo)
 		podInfo.IPTables = netstack.ParseIPTables(podInfo.NetNSInfo.IptablesInfo)
 		if err != nil {
 			return err
 		}
-		podInfo.IPSetManager, err = netstack.ParseIPSet(podInfo.NetNSInfo.IpsetInfo)
+		podInfo.IPSetManager, err = netstack.NewIPSetManager(podInfo.NetNSInfo.IpsetInfo)
 		if err != nil {
 			return err
 		}

pkg/skoop/collector/podcollector/collector.go

@@ -26,6 +26,7 @@ import (
 	"github.com/bastjan/netstat"
 	"github.com/containerd/containerd/pkg/cri/server"
 	"github.com/docker/docker/client"
+	"github.com/moby/ipvs"
 	"github.com/vishvananda/netlink"
 	"github.com/vishvananda/netns"
 	"golang.org/x/exp/slices"
@@ -265,7 +266,6 @@ func NSExec(args []string) (string, error) {
 	}
 	return string(output), nil
 }
-
 func namespaceCmd(pid uint32, cmd string) (string, error) {
 	cmdExec := exec.Command("nsenter", strconv.Itoa(int(pid)), "sh", "-c", cmd)
 	cmdExec.Path = "/proc/self/exe"
@@ -477,16 +477,52 @@ func iptablesCollector(sandboxInfo *netstack.NetNSInfo) error {
 
 func ipsetCollector(sandboxInfo *netstack.NetNSInfo) error {
 	var err error
-	sandboxInfo.IpsetInfo, err = namespaceCmd(sandboxInfo.PID, "ipset list -o xml")
+	info, err := namespaceCmd(sandboxInfo.PID, "ipset list -o xml")
+	if err != nil {
+		return err
+	}
+	sandboxInfo.IpsetInfo, err = netstack.ParseIPSet(info)
 	return err
 }
 
 func ipvsCollector(sandboxInfo *netstack.NetNSInfo) error {
-	ipvsStr, err := namespaceCmd(sandboxInfo.PID, "ipvsadm-save -n")
+	path := fmt.Sprintf("/proc/%d/ns/net", sandboxInfo.PID)
+	handler, err := ipvs.New(path)
+	if err != nil {
+		return err
+	}
+	services, err := handler.GetServices()
 	if err != nil {
 		return err
 	}
-	sandboxInfo.IPVSInfo = strings.Split(ipvsStr, "\n")
+
+	m := map[string]*netstack.IPVSService{}
+	for _, svc := range services {
+		i := &netstack.IPVSService{
+			Protocol:  intToProtocol(svc.Protocol),
+			IP:        svc.Address.String(),
+			Port:      svc.Port,
+			Scheduler: svc.SchedName,
+			RS:        nil,
+		}
+		dsts, err := handler.GetDestinations(svc)
+		if err != nil {
+			return err
+		}
+		for _, dst := range dsts {
+			rs := netstack.RealServer{
+				IP:         dst.Address.String(),
+				Port:       dst.Port,
+				Masquerade: dst.ConnectionFlags == ipvs.ConnectionFlagMasq,
+				Weight:     dst.Weight,
+			}
+			i.RS = append(i.RS, rs)
+		}
+
+		m[i.Service()] = i
+	}
+
+	sandboxInfo.IPVSInfo = m
 	return nil
 }
 
@@ -546,3 +582,13 @@ func sockCollector(sandboxInfo *netstack.NetNSInfo) error {
 	}
 	return nil
 }
+
+func intToProtocol(proto uint16) model.Protocol {
+	switch proto {
+	case unix.IPPROTO_TCP:
+		return model.TCP
+	case unix.IPPROTO_UDP:
+		return model.UDP
+	}
+	return "unknown"
+}

pkg/skoop/k8s/node.go

@@ -6,19 +6,19 @@ import (
 )
 
 type PodNetInfo struct {
-	ContainerID  string `json:"container_id"`
-	PodName      string `json:"pod_name"`
-	PodNamespace string `json:"pod_namespace"`
-	PodUID       string `json:"pod_uid"`
+	ContainerID  string `json:"id"`
+	PodName      string `json:"n"`
+	PodNamespace string `json:"ns"`
+	PodUID       string `json:"u"`
 	PID          uint32 `json:"pid"`
-	Netns        string `json:"netns"`
-	HostNetwork  bool   `json:"host_network"`
-	NetworkMode  string `json:"network_mode"`
+	Netns        string `json:"net"`
+	HostNetwork  bool   `json:"hn"`
+	NetworkMode  string `json:"nm"`
 }
 
 type NodeNetworkStackDump struct {
-	Pods  []PodNetInfo         `json:"pods"`
-	Netns []netstack.NetNSInfo `json:"netns"`
+	Pods  []PodNetInfo         `json:"p"`
+	Netns []netstack.NetNSInfo `json:"n"`
 }
 
 type NodeMeta struct {

pkg/skoop/netstack/ipset.go

@@ -14,19 +14,24 @@ func (m *IPSetManager) GetIPSet(name string) *IPSet {
 }
 
 type IPSet struct {
-	Name    string
-	Type    string
-	Members map[string]string
+	Name    string            `json:"n"`
+	Type    string            `json:"t"`
+	Members map[string]string `json:"m"`
 }
 
-func ParseIPSet(dump string) (*IPSetManager, error) {
+func NewIPSetManager(ipsets []*IPSet) (*IPSetManager, error) {
 	ret := &IPSetManager{
 		sets: make(map[string]*IPSet),
 	}
-	if dump == "" {
-		return ret, nil
+
+	for _, i := range ipsets {
+		ret.sets[i.Name] = i
 	}
+	return ret, nil
+}
 
+func ParseIPSet(dump string) ([]*IPSet, error) {
+	var ret []*IPSet
 	doc := etree.NewDocument()
 	if err := doc.ReadFromString(dump); err != nil {
 		return nil, err
@@ -36,7 +41,7 @@ func ParseIPSet(dump string) (*IPSetManager, error) {
 		if err != nil {
 			return nil, errors.Wrap(err, "error parse ipset")
 		}
-		ret.sets[ipset.Name] = ipset
+		ret = append(ret, ipset)
 	}
 	return ret, nil
 }

pkg/skoop/netstack/ipvs.go

@@ -12,19 +12,19 @@ import (
 )
 
 type RealServer struct {
-	Service    string
-	IP         string
-	Port       uint16
-	Masquerade bool
-	Weight     int
+	Service    string `json:"s,omitempty"`
+	IP         string `json:"ip"`
+	Port       uint16 `json:"p"`
+	Masquerade bool   `json:"m"`
+	Weight     int    `json:"w"`
 }
 
 type IPVSService struct {
-	Protocol  model.Protocol
-	IP        string
-	Port      uint16
-	Scheduler string
-	RS        []RealServer
+	Protocol  model.Protocol `json:"pro"`
+	IP        string         `json:"ip"`
+	Port      uint16         `json:"p"`
+	Scheduler string         `json:"s"`
+	RS        []RealServer   `json:"r"`
 }
 
 func (s *IPVSService) Service() string {
@@ -142,3 +142,9 @@ func ParseIPVS(dump []string) (*IPVS, error) {
 
 	return ipvs, nil
 }
+
+func NewIPVS(services map[string]*IPVSService) *IPVS {
+	return &IPVS{
+		services: services,
+	}
+}

pkg/skoop/netstack/link.go

@@ -34,17 +34,17 @@ var (
 )
 
 type Interface struct {
-	Name        string            `json:"name"`
-	Index       int               `json:"index"`
-	MTU         int               `json:"MTU"`
-	Driver      string            `json:"driver"`
-	Addrs       []Addr            `json:"addrs"`
-	State       int               `json:"state"`
-	DevSysctls  map[string]string `json:"dev_sysctls"`
-	NeighInfo   []Neigh           `json:"neigh_info"`
-	FdbInfo     []Neigh           `json:"fdb_info"`
-	PeerIndex   int               `json:"peer_index"`
-	MasterIndex int               `json:"master_index"`
+	Name        string            `json:"n"`
+	Index       int               `json:"i"`
+	MTU         int               `json:"m"`
+	Driver      string            `json:"d"`
+	Addrs       []Addr            `json:"a"`
+	State       int               `json:"st"`
+	DevSysctls  map[string]string `json:"s"`
+	NeighInfo   []Neigh           `json:"ne"`
+	FdbInfo     []Neigh           `json:"f"`
+	PeerIndex   int               `json:"p"`
+	MasterIndex int               `json:"mi"`
 }
 
 func GetDefaultIPv4(iface *Interface) (net.IP, net.IPMask) {

pkg/skoop/netstack/netns.go

@@ -13,16 +13,16 @@ type NetNS struct {
 
 // NetNSInfo raw data load from collector
 type NetNSInfo struct {
-	Netns        string            `json:"netns"`
-	NetnsID      string            `json:"netns_id"`
-	PID          uint32            `json:"pid"`
-	Key          string            `json:"key"`
-	Interfaces   []Interface       `json:"interfaces"`
-	SysctlInfo   map[string]string `json:"sysctl_info"`
-	RouteInfo    []Route           `json:"route_info"`
-	RuleInfo     []Rule            `json:"rule_info"`
-	IptablesInfo string            `json:"iptables_info"`
-	IpsetInfo    string            `json:"ipset_info"`
-	IPVSInfo     []string          `json:"ipvs_info"`
-	ConnStats    []ConnStat        `json:"conn_stats"`
+	Netns        string                  `json:"n"`
+	NetnsID      string                  `json:"i"`
+	PID          uint32                  `json:"p"`
+	Key          string                  `json:"k"`
+	Interfaces   []Interface             `json:"if"`
+	SysctlInfo   map[string]string       `json:"s"`
+	RouteInfo    []Route                 `json:"r"`
+	RuleInfo     []Rule                  `json:"ru"`
+	IptablesInfo string                  `json:"it"`
+	IpsetInfo    []*IPSet                `json:"is"`
+	IPVSInfo     map[string]*IPVSService `json:"vs"`
+	ConnStats    []ConnStat              `json:"c"`
 }

pkg/skoop/netstack/route.go

@@ -42,19 +42,19 @@ const (
 var ErrNoRouteToHost = errors.New("no route to host")
 
 type Route struct {
-	Family   int
-	OifName  string
-	IifName  string
-	Scope    Scope
-	Dst      *net.IPNet
-	Src      net.IP
-	Gw       net.IP
-	Protocol int
-	Priority int
-	Table    int
-	Type     int
-	Tos      int
-	Flags    int
+	Family   int        `json:"f"`
+	OifName  string     `json:"o"`
+	IifName  string     `json:"i"`
+	Scope    Scope      `json:"sc"`
+	Dst      *net.IPNet `json:"d"`
+	Src      net.IP     `json:"s"`
+	Gw       net.IP     `json:"g"`
+	Protocol int        `json:"p"`
+	Priority int        `json:"pr"`
+	Table    int        `json:"tb"`
+	Type     int        `json:"t"`
+	Tos      int        `json:"tos"`
+	Flags    int        `json:"fl"`
 }
 
 func (r Route) String() string {
@@ -143,19 +143,19 @@ func RouteProtocolToString(protocol int) string {
 }
 
 type Rule struct {
-	Priority int
-	Family   int
-	Table    int
-	Mark     int
-	Mask     int
-	Tos      uint
-	TunID    uint
-	Goto     int
-	Src      *net.IPNet
-	Dst      *net.IPNet
-	Flow     int
-	IifName  string
-	OifName  string
+	Priority int        `json:"p"`
+	Family   int        `json:"f"`
+	Table    int        `json:"tb"`
+	Mark     int        `json:"m"`
+	Mask     int        `json:"ma"`
+	Tos      uint       `json:"tos"`
+	TunID    uint       `json:"ti"`
+	Goto     int        `json:"gt"`
+	Src      *net.IPNet `json:"s"`
+	Dst      *net.IPNet `json:"d"`
+	Flow     int        `json:"fl"`
+	IifName  string     `json:"i"`
+	OifName  string     `json:"o"`
 }
 
 type Router interface {