From 3aec6df887eb4e18b3124ece946ff9d31726ff5d Mon Sep 17 00:00:00 2001 From: hzma Date: Fri, 19 Apr 2024 09:49:28 +0800 Subject: [PATCH] add metric for subnet info (#3932) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 马洪贞 --- pkg/controller/controller.go | 2 +- pkg/controller/exporter.go | 80 ++++++++++++++++++++++++++++++++--- pkg/controller/net_metrics.go | 42 ++++++++++++++++++ 3 files changed, 116 insertions(+), 8 deletions(-) diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index e0ee25e862b..108618c1597 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -1049,7 +1049,7 @@ func (c *Controller) startWorkers(ctx context.Context) { } go wait.Until(c.resyncProviderNetworkStatus, 30*time.Second, ctx.Done()) - go wait.Until(c.resyncSubnetMetrics, 30*time.Second, ctx.Done()) + go wait.Until(c.exportSubnetMetrics, 30*time.Second, ctx.Done()) go wait.Until(c.CheckGatewayReady, 5*time.Second, ctx.Done()) go wait.Until(c.runAddOvnEipWorker, time.Second, ctx.Done()) diff --git a/pkg/controller/exporter.go b/pkg/controller/exporter.go index a493fb5ed85..1e599459c20 100644 --- a/pkg/controller/exporter.go +++ b/pkg/controller/exporter.go @@ -2,12 +2,15 @@ package controller import ( "math" + "strconv" + "strings" "sync" "k8s.io/apimachinery/pkg/labels" "k8s.io/klog/v2" kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + "github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb" ) var registerMetricsOnce sync.Once @@ -19,23 +22,32 @@ func (c *Controller) registerSubnetMetrics() { }) } -// resyncSubnetMetrics start to update subnet metrics -func (c *Controller) resyncSubnetMetrics() { - c.exportSubnetMetrics() +func resetSubnetMetrics() { + metricSubnetAvailableIPs.Reset() + metricSubnetUsedIPs.Reset() + metricCentralSubnetInfo.Reset() + metricSubnetIPAMInfo.Reset() + metricSubnetIPAssignedInfo.Reset() } -func (c *Controller) exportSubnetMetrics() bool { +func (c *Controller) exportSubnetMetrics() { subnets, err := c.subnetsLister.List(labels.Everything()) if err != nil { klog.Errorf("failed to list subnet, %v", err) - return false + return } + + resetSubnetMetrics() for _, subnet := range subnets { c.exportSubnetAvailableIPsGauge(subnet) c.exportSubnetUsedIPsGauge(subnet) - } + c.exportSubnetIPAMInfo(subnet) + c.exportSubnetIPAssignedInfo(subnet) - return true + if subnet.Spec.GatewayType == kubeovnv1.GWCentralizedType { + c.exportCentralizedSubnetInfo(subnet) + } + } } func (c *Controller) exportSubnetAvailableIPsGauge(subnet *kubeovnv1.Subnet) { @@ -60,3 +72,57 @@ func (c *Controller) exportSubnetUsedIPsGauge(subnet *kubeovnv1.Subnet) { } metricSubnetUsedIPs.WithLabelValues(subnet.Name, subnet.Spec.Protocol, subnet.Spec.CIDRBlock).Set(usingIPs) } + +func (c *Controller) exportCentralizedSubnetInfo(subnet *kubeovnv1.Subnet) { + lrPolicyList, err := c.OVNNbClient.GetLogicalRouterPoliciesByExtID(c.config.ClusterRouter, "subnet", subnet.Name) + if err != nil { + klog.Errorf("failed to list lr policy for subnet %s: %v", subnet.Name, err) + return + } + + for _, lrPolicy := range lrPolicyList { + if lrPolicy.Action == ovnnb.LogicalRouterPolicyActionReroute { + metricCentralSubnetInfo.WithLabelValues(subnet.Name, strconv.FormatBool(subnet.Spec.EnableEcmp), subnet.Spec.GatewayNode, subnet.Status.ActivateGateway, lrPolicy.Match, strings.Join(lrPolicy.Nexthops, ",")).Set(1) + break + } + } +} + +func (c *Controller) exportSubnetIPAMInfo(subnet *kubeovnv1.Subnet) { + ipamSubnet, ok := c.ipam.Subnets[subnet.Name] + if !ok { + klog.Errorf("failed to get subnet %s in ipam", subnet.Name) + return + } + + switch subnet.Spec.Protocol { + case kubeovnv1.ProtocolIPv4: + metricSubnetIPAMInfo.WithLabelValues(subnet.Name, subnet.Spec.CIDRBlock, ipamSubnet.V4Free.String(), ipamSubnet.V4Reserved.String(), ipamSubnet.V4Available.String(), ipamSubnet.V4Using.String()).Set(1) + + case kubeovnv1.ProtocolIPv6: + metricSubnetIPAMInfo.WithLabelValues(subnet.Name, subnet.Spec.CIDRBlock, ipamSubnet.V6Free.String(), ipamSubnet.V6Reserved.String(), ipamSubnet.V6Available.String(), ipamSubnet.V6Using.String()).Set(1) + + case kubeovnv1.ProtocolDual: + metricSubnetIPAMInfo.WithLabelValues(subnet.Name, subnet.Spec.CIDRBlock, ipamSubnet.V4Free.String(), ipamSubnet.V4Reserved.String(), ipamSubnet.V4Available.String(), ipamSubnet.V4Using.String()).Set(1) + metricSubnetIPAMInfo.WithLabelValues(subnet.Name, subnet.Spec.CIDRBlock, ipamSubnet.V6Free.String(), ipamSubnet.V6Reserved.String(), ipamSubnet.V6Available.String(), ipamSubnet.V6Using.String()).Set(1) + } +} + +func (c *Controller) exportSubnetIPAssignedInfo(subnet *kubeovnv1.Subnet) { + ipamSubnet, ok := c.ipam.Subnets[subnet.Name] + if !ok { + klog.Errorf("failed to get subnet %s in ipam", subnet.Name) + return + } + + if subnet.Spec.Protocol == kubeovnv1.ProtocolIPv4 || subnet.Spec.Protocol == kubeovnv1.ProtocolDual { + for ip, pod := range ipamSubnet.V4IPToPod { + metricSubnetIPAssignedInfo.WithLabelValues(subnet.Name, ip, pod).Set(1) + } + } + if subnet.Spec.Protocol == kubeovnv1.ProtocolIPv6 || subnet.Spec.Protocol == kubeovnv1.ProtocolDual { + for ip, pod := range ipamSubnet.V6IPToPod { + metricSubnetIPAssignedInfo.WithLabelValues(subnet.Name, ip, pod).Set(1) + } + } +} diff --git a/pkg/controller/net_metrics.go b/pkg/controller/net_metrics.go index 2e2efdcf6d3..f0c24238909 100644 --- a/pkg/controller/net_metrics.go +++ b/pkg/controller/net_metrics.go @@ -24,9 +24,51 @@ var ( "protocol", "subnet_cidr", }) + + metricCentralSubnetInfo = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "centralized_subnet_info", + Help: "Provide information for centralized subnet.", + }, + []string{ + "subnet_name", + "enable_ecmp", + "gateway_node", + "active_gateway", + "match", + "nexthops", + }) + + metricSubnetIPAMInfo = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "subnet_ipam_info", + Help: "Provide information for subnet ip address management.", + }, + []string{ + "subnet_name", + "cidr", + "free", + "reserved", + "available", + "using", + }) + + metricSubnetIPAssignedInfo = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "subnet_ip_assign_info", + Help: "Provide information for subnet ip address assigned info.", + }, + []string{ + "subnet_name", + "ip", + "pod_name", + }) ) func registerMetrics() { prometheus.MustRegister(metricSubnetAvailableIPs) prometheus.MustRegister(metricSubnetUsedIPs) + prometheus.MustRegister(metricCentralSubnetInfo) + prometheus.MustRegister(metricSubnetIPAMInfo) + prometheus.MustRegister(metricSubnetIPAssignedInfo) }