metrics: make counter inc when operator finished rather than operator add (#2962)

server/schedule/operator/operator.go

@@ -49,17 +49,18 @@ type Cluster interface {
 
 // Operator contains execution steps generated by scheduler.
 type Operator struct {
-	desc        string
-	brief       string
-	regionID    uint64
-	regionEpoch *metapb.RegionEpoch
-	kind        OpKind
-	steps       []OpStep
-	currentStep int32
-	status      OpStatusTracker
-	stepTime    int64
-	level       core.PriorityLevel
-	Counters    []prometheus.Counter
+	desc             string
+	brief            string
+	regionID         uint64
+	regionEpoch      *metapb.RegionEpoch
+	kind             OpKind
+	steps            []OpStep
+	currentStep      int32
+	status           OpStatusTracker
+	stepTime         int64
+	level            core.PriorityLevel
+	Counters         []prometheus.Counter
+	FinishedCounters []prometheus.Counter
 }
 
 // NewOperator creates a new operator.

server/schedule/operator_controller.go

@@ -539,6 +539,9 @@ func (oc *OperatorController) buryOperator(op *operator.Operator, extraFileds ..
 			zap.Reflect("operator", op))
 		operatorCounter.WithLabelValues(op.Desc(), "finish").Inc()
 		operatorDuration.WithLabelValues(op.Desc()).Observe(op.RunningTime().Seconds())
+		for _, counter := range op.FinishedCounters {
+			counter.Inc()
+		}
 	case operator.REPLACED:
 		log.Info("replace old operator",
 			zap.Uint64("region-id", op.RegionID()),

server/schedulers/balance_leader.go

@@ -287,9 +287,11 @@ func (l *balanceLeaderScheduler) createOperator(cluster opt.Cluster, region *cor
 	targetLabel := strconv.FormatUint(targetID, 10)
 	op.Counters = append(op.Counters,
 		schedulerCounter.WithLabelValues(l.GetName(), "new-operator"),
+		balanceDirectionCounter.WithLabelValues(l.GetName(), sourceLabel, targetLabel),
+	)
+	op.FinishedCounters = append(op.FinishedCounters,
 		l.counter.WithLabelValues("move-leader", source.GetAddress()+"-out", sourceLabel),
 		l.counter.WithLabelValues("move-leader", target.GetAddress()+"-in", targetLabel),
-		balanceDirectionCounter.WithLabelValues(l.GetName(), sourceLabel, targetLabel),
 	)
 	return []*operator.Operator{op}
 }

server/schedulers/balance_region.go

@@ -242,9 +242,11 @@ func (s *balanceRegionScheduler) transferPeer(cluster opt.Cluster, region *core.
 		sourceLabel := strconv.FormatUint(sourceID, 10)
 		targetLabel := strconv.FormatUint(targetID, 10)
 		op.Counters = append(op.Counters,
+			balanceDirectionCounter.WithLabelValues(s.GetName(), sourceLabel, targetLabel),
+		)
+		op.FinishedCounters = append(op.FinishedCounters,
 			s.counter.WithLabelValues("move-peer", source.GetAddress()+"-out", sourceLabel),
 			s.counter.WithLabelValues("move-peer", target.GetAddress()+"-in", targetLabel),
-			balanceDirectionCounter.WithLabelValues(s.GetName(), sourceLabel, targetLabel),
 		)
 		return op
 	}

server/schedulers/balance_test.go

@@ -454,7 +454,8 @@ func (s *testBalanceLeaderRangeSchedulerSuite) TestSingleRangeBalance(c *C) {
 	ops := lb.Schedule(s.tc)
 	c.Assert(ops, NotNil)
 	c.Assert(ops, HasLen, 1)
-	c.Assert(ops[0].Counters, HasLen, 5)
+	c.Assert(ops[0].Counters, HasLen, 3)
+	c.Assert(ops[0].FinishedCounters, HasLen, 2)
 	lb, err = schedule.CreateScheduler(BalanceLeaderType, s.oc, core.NewStorage(kv.NewMemoryKV()), schedule.ConfigSliceDecoder(BalanceLeaderType, []string{"h", "n"}))
 	c.Assert(err, IsNil)
 	c.Assert(lb.Schedule(s.tc), IsNil)