From 381181888d45a1aed9b151f43e3fd55c9ba20231 Mon Sep 17 00:00:00 2001 From: "xiayu.lyt" Date: Wed, 24 May 2023 21:02:22 +0800 Subject: [PATCH] use ipvs library instead of ipvsadm-save & reduce collector json file size Signed-off-by: xiayu.lyt --- go.mod | 1 + go.sum | 2 + pkg/skoop/assertions/netstack.go | 8 +-- pkg/skoop/collector/manager/manager.go | 8 +-- pkg/skoop/collector/podcollector/collector.go | 54 +++++++++++++++++-- pkg/skoop/k8s/node.go | 18 +++---- pkg/skoop/netstack/ipset.go | 19 ++++--- pkg/skoop/netstack/ipvs.go | 26 +++++---- pkg/skoop/netstack/link.go | 22 ++++---- pkg/skoop/netstack/netns.go | 24 ++++----- pkg/skoop/netstack/route.go | 52 +++++++++--------- pkg/skoop/netstack/socket.go | 12 ++--- pkg/skoop/network/aliyun/node.go | 13 ++++- 13 files changed, 161 insertions(+), 98 deletions(-) diff --git a/go.mod b/go.mod index 74a4f974..ac2a874a 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( github.com/google/uuid v1.3.0 github.com/gorilla/mux v1.8.0 github.com/mdlayher/netlink v1.7.1 + github.com/moby/ipvs v1.1.0 github.com/onsi/ginkgo/v2 v2.9.1 github.com/onsi/gomega v1.27.4 github.com/patrickmn/go-cache v2.1.0+incompatible diff --git a/go.sum b/go.sum index c14dd31c..5c58f9e2 100644 --- a/go.sum +++ b/go.sum @@ -786,6 +786,8 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= +github.com/moby/ipvs v1.1.0 h1:ONN4pGaZQgAx+1Scz5RvWV4Q7Gb+mvfRh3NsPS+1XQQ= +github.com/moby/ipvs v1.1.0/go.mod h1:4VJMWuf098bsUMmZEiD4Tjk/O7mOn3l1PTD3s4OoYAs= github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= diff --git a/pkg/skoop/assertions/netstack.go b/pkg/skoop/assertions/netstack.go index cc1153a8..45b38e31 100644 --- a/pkg/skoop/assertions/netstack.go +++ b/pkg/skoop/assertions/netstack.go @@ -209,12 +209,6 @@ func (na *NetstackAssertion) AssertVEthOnBridge(index int, expectedBridgeName st // todo: br_port_state } -func (na *NetstackAssertion) AssertIPVSServiceExists(service, servicePort, protocol string) { - key := fmt.Sprintf("%s:%s:%s", protocol, service, servicePort) - AssertTrue(na, slices.Contains(na.netns.NetNSInfo.IPVSInfo, key), model.SuspicionLevelWarning, - fmt.Sprintf("ipvs has no service %s", key)) -} - type RouteAssertion struct { Dev *string Scope *netstack.Scope @@ -464,7 +458,7 @@ func (na *NetstackAssertion) AssertNetfilterServe(pktIn model.Packet, iif string func (na *NetstackAssertion) AssertIPVSServerExists(service string, servicePort uint16, protocol model.Protocol, backend string, backendPort uint16) { key := fmt.Sprintf("%s:%s:%d", protocol, service, servicePort) - _, ok := lo.Find(na.netns.NetNSInfo.IPVSInfo, func(i string) bool { return i == key }) + _, ok := na.netns.NetNSInfo.IPVSInfo[key] if !ok { return } diff --git a/pkg/skoop/collector/manager/manager.go b/pkg/skoop/collector/manager/manager.go index fb0fa713..54747901 100644 --- a/pkg/skoop/collector/manager/manager.go +++ b/pkg/skoop/collector/manager/manager.go @@ -172,12 +172,12 @@ func (m *simplePodCollectorManager) buildCache(nodeName string) error { } nodeInfo.Router = netstack.NewSimulateRouter(nodeInfo.NetNSInfo.RuleInfo, nodeInfo.NetNSInfo.RouteInfo, nodeInfo.NetNSInfo.Interfaces) - nodeInfo.IPVS, err = netstack.ParseIPVS(nodeInfo.NetNSInfo.IPVSInfo) + nodeInfo.IPVS = netstack.NewIPVS(nodeInfo.NetNSInfo.IPVSInfo) nodeInfo.IPTables = netstack.ParseIPTables(nodeInfo.NetNSInfo.IptablesInfo) if err != nil { return err } - nodeInfo.IPSetManager, err = netstack.ParseIPSet(nodeInfo.NetNSInfo.IpsetInfo) + nodeInfo.IPSetManager, err = netstack.NewIPSetManager(nodeInfo.NetNSInfo.IpsetInfo) if err != nil { return err } @@ -212,12 +212,12 @@ func (m *simplePodCollectorManager) buildCache(nodeName string) error { podInfo.NetNSInfo = &podNetNS } - podInfo.IPVS, err = netstack.ParseIPVS(podInfo.NetNSInfo.IPVSInfo) + podInfo.IPVS = netstack.NewIPVS(podInfo.NetNSInfo.IPVSInfo) podInfo.IPTables = netstack.ParseIPTables(podInfo.NetNSInfo.IptablesInfo) if err != nil { return err } - podInfo.IPSetManager, err = netstack.ParseIPSet(podInfo.NetNSInfo.IpsetInfo) + podInfo.IPSetManager, err = netstack.NewIPSetManager(podInfo.NetNSInfo.IpsetInfo) if err != nil { return err } diff --git a/pkg/skoop/collector/podcollector/collector.go b/pkg/skoop/collector/podcollector/collector.go index fda7258a..388b659e 100644 --- a/pkg/skoop/collector/podcollector/collector.go +++ b/pkg/skoop/collector/podcollector/collector.go @@ -26,6 +26,7 @@ import ( "github.com/bastjan/netstat" "github.com/containerd/containerd/pkg/cri/server" "github.com/docker/docker/client" + "github.com/moby/ipvs" "github.com/vishvananda/netlink" "github.com/vishvananda/netns" "golang.org/x/exp/slices" @@ -265,7 +266,6 @@ func NSExec(args []string) (string, error) { } return string(output), nil } - func namespaceCmd(pid uint32, cmd string) (string, error) { cmdExec := exec.Command("nsenter", strconv.Itoa(int(pid)), "sh", "-c", cmd) cmdExec.Path = "/proc/self/exe" @@ -477,16 +477,52 @@ func iptablesCollector(sandboxInfo *netstack.NetNSInfo) error { func ipsetCollector(sandboxInfo *netstack.NetNSInfo) error { var err error - sandboxInfo.IpsetInfo, err = namespaceCmd(sandboxInfo.PID, "ipset list -o xml") + info, err := namespaceCmd(sandboxInfo.PID, "ipset list -o xml") + if err != nil { + return err + } + sandboxInfo.IpsetInfo, err = netstack.ParseIPSet(info) return err } func ipvsCollector(sandboxInfo *netstack.NetNSInfo) error { - ipvsStr, err := namespaceCmd(sandboxInfo.PID, "ipvsadm-save -n") + path := fmt.Sprintf("/proc/%d/ns/net", sandboxInfo.PID) + handler, err := ipvs.New(path) + if err != nil { + return err + } + services, err := handler.GetServices() if err != nil { return err } - sandboxInfo.IPVSInfo = strings.Split(ipvsStr, "\n") + + m := map[string]*netstack.IPVSService{} + for _, svc := range services { + i := &netstack.IPVSService{ + Protocol: intToProtocol(svc.Protocol), + IP: svc.Address.String(), + Port: svc.Port, + Scheduler: svc.SchedName, + RS: nil, + } + dsts, err := handler.GetDestinations(svc) + if err != nil { + return err + } + for _, dst := range dsts { + rs := netstack.RealServer{ + IP: dst.Address.String(), + Port: dst.Port, + Masquerade: dst.ConnectionFlags == ipvs.ConnectionFlagMasq, + Weight: dst.Weight, + } + i.RS = append(i.RS, rs) + } + + m[i.Service()] = i + } + + sandboxInfo.IPVSInfo = m return nil } @@ -546,3 +582,13 @@ func sockCollector(sandboxInfo *netstack.NetNSInfo) error { } return nil } + +func intToProtocol(proto uint16) model.Protocol { + switch proto { + case unix.IPPROTO_TCP: + return model.TCP + case unix.IPPROTO_UDP: + return model.UDP + } + return "unknown" +} diff --git a/pkg/skoop/k8s/node.go b/pkg/skoop/k8s/node.go index 70a49d8f..fa02bf49 100644 --- a/pkg/skoop/k8s/node.go +++ b/pkg/skoop/k8s/node.go @@ -6,19 +6,19 @@ import ( ) type PodNetInfo struct { - ContainerID string `json:"container_id"` - PodName string `json:"pod_name"` - PodNamespace string `json:"pod_namespace"` - PodUID string `json:"pod_uid"` + ContainerID string `json:"id"` + PodName string `json:"n"` + PodNamespace string `json:"ns"` + PodUID string `json:"u"` PID uint32 `json:"pid"` - Netns string `json:"netns"` - HostNetwork bool `json:"host_network"` - NetworkMode string `json:"network_mode"` + Netns string `json:"net"` + HostNetwork bool `json:"hn"` + NetworkMode string `json:"nm"` } type NodeNetworkStackDump struct { - Pods []PodNetInfo `json:"pods"` - Netns []netstack.NetNSInfo `json:"netns"` + Pods []PodNetInfo `json:"p"` + Netns []netstack.NetNSInfo `json:"n"` } type NodeMeta struct { diff --git a/pkg/skoop/netstack/ipset.go b/pkg/skoop/netstack/ipset.go index 30e895e9..be720af0 100644 --- a/pkg/skoop/netstack/ipset.go +++ b/pkg/skoop/netstack/ipset.go @@ -14,19 +14,24 @@ func (m *IPSetManager) GetIPSet(name string) *IPSet { } type IPSet struct { - Name string - Type string - Members map[string]string + Name string `json:"n"` + Type string `json:"t"` + Members map[string]string `json:"m"` } -func ParseIPSet(dump string) (*IPSetManager, error) { +func NewIPSetManager(ipsets []*IPSet) (*IPSetManager, error) { ret := &IPSetManager{ sets: make(map[string]*IPSet), } - if dump == "" { - return ret, nil + + for _, i := range ipsets { + ret.sets[i.Name] = i } + return ret, nil +} +func ParseIPSet(dump string) ([]*IPSet, error) { + var ret []*IPSet doc := etree.NewDocument() if err := doc.ReadFromString(dump); err != nil { return nil, err @@ -36,7 +41,7 @@ func ParseIPSet(dump string) (*IPSetManager, error) { if err != nil { return nil, errors.Wrap(err, "error parse ipset") } - ret.sets[ipset.Name] = ipset + ret = append(ret, ipset) } return ret, nil } diff --git a/pkg/skoop/netstack/ipvs.go b/pkg/skoop/netstack/ipvs.go index 1591a8ee..b17fc6f1 100644 --- a/pkg/skoop/netstack/ipvs.go +++ b/pkg/skoop/netstack/ipvs.go @@ -12,19 +12,19 @@ import ( ) type RealServer struct { - Service string - IP string - Port uint16 - Masquerade bool - Weight int + Service string `json:"s,omitempty"` + IP string `json:"ip"` + Port uint16 `json:"p"` + Masquerade bool `json:"m"` + Weight int `json:"w"` } type IPVSService struct { - Protocol model.Protocol - IP string - Port uint16 - Scheduler string - RS []RealServer + Protocol model.Protocol `json:"pro"` + IP string `json:"ip"` + Port uint16 `json:"p"` + Scheduler string `json:"s"` + RS []RealServer `json:"r"` } func (s *IPVSService) Service() string { @@ -142,3 +142,9 @@ func ParseIPVS(dump []string) (*IPVS, error) { return ipvs, nil } + +func NewIPVS(services map[string]*IPVSService) *IPVS { + return &IPVS{ + services: services, + } +} diff --git a/pkg/skoop/netstack/link.go b/pkg/skoop/netstack/link.go index 53754ce0..66c8ce07 100644 --- a/pkg/skoop/netstack/link.go +++ b/pkg/skoop/netstack/link.go @@ -34,17 +34,17 @@ var ( ) type Interface struct { - Name string `json:"name"` - Index int `json:"index"` - MTU int `json:"MTU"` - Driver string `json:"driver"` - Addrs []Addr `json:"addrs"` - State int `json:"state"` - DevSysctls map[string]string `json:"dev_sysctls"` - NeighInfo []Neigh `json:"neigh_info"` - FdbInfo []Neigh `json:"fdb_info"` - PeerIndex int `json:"peer_index"` - MasterIndex int `json:"master_index"` + Name string `json:"n"` + Index int `json:"i"` + MTU int `json:"m"` + Driver string `json:"d"` + Addrs []Addr `json:"a"` + State int `json:"st"` + DevSysctls map[string]string `json:"s"` + NeighInfo []Neigh `json:"ne"` + FdbInfo []Neigh `json:"f"` + PeerIndex int `json:"p"` + MasterIndex int `json:"mi"` } func GetDefaultIPv4(iface *Interface) (net.IP, net.IPMask) { diff --git a/pkg/skoop/netstack/netns.go b/pkg/skoop/netstack/netns.go index a87f715f..60bdd80e 100644 --- a/pkg/skoop/netstack/netns.go +++ b/pkg/skoop/netstack/netns.go @@ -13,16 +13,16 @@ type NetNS struct { // NetNSInfo raw data load from collector type NetNSInfo struct { - Netns string `json:"netns"` - NetnsID string `json:"netns_id"` - PID uint32 `json:"pid"` - Key string `json:"key"` - Interfaces []Interface `json:"interfaces"` - SysctlInfo map[string]string `json:"sysctl_info"` - RouteInfo []Route `json:"route_info"` - RuleInfo []Rule `json:"rule_info"` - IptablesInfo string `json:"iptables_info"` - IpsetInfo string `json:"ipset_info"` - IPVSInfo []string `json:"ipvs_info"` - ConnStats []ConnStat `json:"conn_stats"` + Netns string `json:"n"` + NetnsID string `json:"i"` + PID uint32 `json:"p"` + Key string `json:"k"` + Interfaces []Interface `json:"if"` + SysctlInfo map[string]string `json:"s"` + RouteInfo []Route `json:"r"` + RuleInfo []Rule `json:"ru"` + IptablesInfo string `json:"it"` + IpsetInfo []*IPSet `json:"is"` + IPVSInfo map[string]*IPVSService `json:"vs"` + ConnStats []ConnStat `json:"c"` } diff --git a/pkg/skoop/netstack/route.go b/pkg/skoop/netstack/route.go index 8aa6c5f9..c1911170 100644 --- a/pkg/skoop/netstack/route.go +++ b/pkg/skoop/netstack/route.go @@ -42,19 +42,19 @@ const ( var ErrNoRouteToHost = errors.New("no route to host") type Route struct { - Family int - OifName string - IifName string - Scope Scope - Dst *net.IPNet - Src net.IP - Gw net.IP - Protocol int - Priority int - Table int - Type int - Tos int - Flags int + Family int `json:"f"` + OifName string `json:"o"` + IifName string `json:"i"` + Scope Scope `json:"sc"` + Dst *net.IPNet `json:"d"` + Src net.IP `json:"s"` + Gw net.IP `json:"g"` + Protocol int `json:"p"` + Priority int `json:"pr"` + Table int `json:"tb"` + Type int `json:"t"` + Tos int `json:"tos"` + Flags int `json:"fl"` } func (r Route) String() string { @@ -143,19 +143,19 @@ func RouteProtocolToString(protocol int) string { } type Rule struct { - Priority int - Family int - Table int - Mark int - Mask int - Tos uint - TunID uint - Goto int - Src *net.IPNet - Dst *net.IPNet - Flow int - IifName string - OifName string + Priority int `json:"p"` + Family int `json:"f"` + Table int `json:"tb"` + Mark int `json:"m"` + Mask int `json:"ma"` + Tos uint `json:"tos"` + TunID uint `json:"ti"` + Goto int `json:"gt"` + Src *net.IPNet `json:"s"` + Dst *net.IPNet `json:"d"` + Flow int `json:"fl"` + IifName string `json:"i"` + OifName string `json:"o"` } type Router interface { diff --git a/pkg/skoop/netstack/socket.go b/pkg/skoop/netstack/socket.go index 38f9ad66..17d5a5ed 100644 --- a/pkg/skoop/netstack/socket.go +++ b/pkg/skoop/netstack/socket.go @@ -13,10 +13,10 @@ const ( ) type ConnStat struct { - LocalIP string - LocalPort uint16 - RemoteIP string - RemotePort uint16 - Protocol model.Protocol - State SockStat + LocalIP string `json:"l"` + LocalPort uint16 `json:"lp"` + RemoteIP string `json:"r"` + RemotePort uint16 `json:"rp"` + Protocol model.Protocol `json:"p"` + State SockStat `json:"st"` } diff --git a/pkg/skoop/network/aliyun/node.go b/pkg/skoop/network/aliyun/node.go index 40963f43..7aae56e9 100644 --- a/pkg/skoop/network/aliyun/node.go +++ b/pkg/skoop/network/aliyun/node.go @@ -140,7 +140,7 @@ func (n *slbNode) Receive(upstream *model.Link) ([]model.Transmission, error) { if _, ok := nodeMap[node.Name]; ok { continue } - pkt.Dst = net.ParseIP(node.Status.Addresses[0].Address) + pkt.Dst = net.ParseIP(getNodeInternalIP(node)) pkt.Dport = nodePort nodeMap[node.Name] = struct{}{} } @@ -211,7 +211,7 @@ func (n *slbNode) getTransmissionsToNodePort(nodePort uint16, pkt *model.Packet) ID: node.Name, } - ip := node.Status.Addresses[0].Address + ip := getNodeInternalIP(node) pkt := pkt.DeepCopy() pkt.Dst = net.ParseIP(ip) pkt.Dport = nodePort @@ -363,3 +363,12 @@ func (n *externalNode) sendToLoadBalancer(dst model.Endpoint, protocol model.Pro func (n *externalNode) Receive(upstream *model.Link) ([]model.Transmission, error) { return n.genericNode.Receive(upstream) } + +func getNodeInternalIP(node *v1.Node) string { + for _, n := range node.Status.Addresses { + if n.Type == v1.NodeInternalIP { + return n.Address + } + } + return "" +}