add metrics about error keys

openyurtio · Jul 11, 2024 · eb9b2ef · eb9b2ef
1 parent fb377ba
commit eb9b2ef
Show file tree

Hide file tree

Showing 2 changed files with 40 additions and 0 deletions.
diff --git a/pkg/yurthub/cachemanager/error_keys.go b/pkg/yurthub/cachemanager/error_keys.go
@@ -31,6 +31,7 @@ import (
 	"k8s.io/klog/v2"
 
 	"github.com/openyurtio/openyurt/pkg/projectinfo"
+	"github.com/openyurtio/openyurt/pkg/yurthub/metrics"
 )
 
 var (
@@ -67,6 +68,7 @@ func NewErrorKeys() *errorKeys {
 	ek.file = file
 	go ek.sync()
 	go ek.compress()
+	metrics.Metrics.SetErrorKeysPersistencyStatus(1)
 	return ek
 }
 
@@ -87,6 +89,7 @@ func (ek *errorKeys) put(key string, val string) {
 	ek.Lock()
 	defer ek.Unlock()
 	ek.keys[key] = val
+	metrics.Metrics.IncErrorKeysCount()
 	ek.queue.AddRateLimited(operation{Operator: PUT, Key: key, Val: val})
 }
 
@@ -97,6 +100,7 @@ func (ek *errorKeys) del(key string) {
 		return
 	}
 	delete(ek.keys, key)
+	metrics.Metrics.DecErrorKeysCount()
 	ek.queue.AddRateLimited(operation{Operator: DEL, Key: key})
 }
 
@@ -203,6 +207,8 @@ func (ek *errorKeys) rewrite() {
 	}
 	file, err = os.OpenFile(filepath.Join(AOFPrefix, "aof"), os.O_RDWR, 0644)
 	if err != nil {
+		klog.ErrorS(err, "failed to open file", "name", filepath.Join(AOFPrefix, "aof"))
+		metrics.Metrics.SetErrorKeysPersistencyStatus(0)
 		ek.queue.ShutDown()
 		return
 	}

diff --git a/pkg/yurthub/metrics/metrics.go b/pkg/yurthub/metrics/metrics.go
@@ -54,6 +54,8 @@ type HubMetrics struct {
 	yurtCoordinatorYurthubRoleCollector   *prometheus.GaugeVec
 	yurtCoordinatorHealthyStatusCollector *prometheus.GaugeVec
 	yurtCoordinatorReadyStatusCollector   *prometheus.GaugeVec
+	errorKeysPersistencyStatusCollector   prometheus.Gauge
+	errorKeysCountCollector               prometheus.Gauge
 }
 
 func newHubMetrics() *HubMetrics {
@@ -135,6 +137,20 @@ func newHubMetrics() *HubMetrics {
 			Help:      "yurt coordinator ready status 1: ready, 0: notReady",
 		},
 		[]string{})
+	errorKeysPersistencyStatusCollector := prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: subsystem,
+			Name:      "error_keys_persistency_status",
+			Help:      "error keys persistency status 1: ready, 0: notReady",
+		})
+	errorKeysCountCollector := prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: subsystem,
+			Name:      "error_keys_count",
+			Help:      "error keys count",
+		})
 	prometheus.MustRegister(serversHealthyCollector)
 	prometheus.MustRegister(inFlightRequestsCollector)
 	prometheus.MustRegister(inFlightRequestsGauge)
@@ -145,6 +161,8 @@ func newHubMetrics() *HubMetrics {
 	prometheus.MustRegister(yurtCoordinatorYurthubRoleCollector)
 	prometheus.MustRegister(yurtCoordinatorHealthyStatusCollector)
 	prometheus.MustRegister(yurtCoordinatorReadyStatusCollector)
+	prometheus.MustRegister(errorKeysPersistencyStatusCollector)
+	prometheus.MustRegister(errorKeysCountCollector)
 	return &HubMetrics{
 		serversHealthyCollector:               serversHealthyCollector,
 		inFlightRequestsCollector:             inFlightRequestsCollector,
@@ -156,6 +174,8 @@ func newHubMetrics() *HubMetrics {
 		yurtCoordinatorHealthyStatusCollector: yurtCoordinatorHealthyStatusCollector,
 		yurtCoordinatorReadyStatusCollector:   yurtCoordinatorReadyStatusCollector,
 		yurtCoordinatorYurthubRoleCollector:   yurtCoordinatorYurthubRoleCollector,
+		errorKeysPersistencyStatusCollector:   errorKeysPersistencyStatusCollector,
+		errorKeysCountCollector:               errorKeysCountCollector,
 	}
 }
 
@@ -166,6 +186,8 @@ func (hm *HubMetrics) Reset() {
 	hm.closableConnsCollector.Reset()
 	hm.proxyTrafficCollector.Reset()
 	hm.proxyLatencyCollector.Reset()
+	hm.errorKeysPersistencyStatusCollector.Set(float64(0))
+	hm.errorKeysCountCollector.Set(float64(0))
 }
 
 func (hm *HubMetrics) ObserveServerHealthy(server string, status int) {
@@ -219,3 +241,15 @@ func (hm *HubMetrics) AddProxyTrafficCollector(client, verb, resource, subresour
 func (hm *HubMetrics) SetProxyLatencyCollector(client, verb, resource, subresource string, latencyType LatencyType, duration int64) {
 	hm.proxyLatencyCollector.WithLabelValues(client, verb, resource, subresource, string(latencyType)).Set(float64(duration))
 }
+
+func (hm *HubMetrics) SetErrorKeysPersistencyStatus(status int) {
+	hm.errorKeysPersistencyStatusCollector.Set(float64(status))
+}
+
+func (hm *HubMetrics) IncErrorKeysCount() {
+	hm.errorKeysCountCollector.Inc()
+}
+
+func (hm *HubMetrics) DecErrorKeysCount() {
+	hm.errorKeysCountCollector.Dec()
+}