mcs, tso: fix logging metrics issues (tikv#95)

* metrics: add tso events metrics (tikv#6501) close tikv#6502 Signed-off-by: bufferflies <[email protected]> * keyspace: add benchmarks for keyspace assignment patrol (tikv#6507) ref tikv#5895 Add benchmarks for keyspace assignment patrol. Signed-off-by: JmPotato <[email protected]> Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> * Print keyspace-group-id zap field in the tso log conditionally (tikv#6514) ref tikv#5895 To keep the logging info in on-premises clean, we only print keyspace-group-id zap field for the non-default keyspace group id. Signed-off-by: Bin Shi <[email protected]> Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> * Improve logging when tso keyspace group meta is updated. (tikv#6513) close tikv#6512 Improve logging when tso keyspace group meta is updated. Signed-off-by: Bin Shi <[email protected]> Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> * tso: log tso service discovery info on the client side only when the primary is changed (tikv#6511) close tikv#6508 Skip logging of the tso service discovery info when the secondar list is changed, because tso servers currently don't have consistent view of the member list due to remote etcd being used by tso service, which results in changing member list when the client queries the tso servers in round-robin. We need to improve the server side so that all tso servers can return the global consistent view of the keyspace groups' serving or membership info. Signed-off-by: Bin Shi <[email protected]> Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --------- Signed-off-by: bufferflies <[email protected]> Co-authored-by: buffer <[email protected]> Co-authored-by: JmPotato <[email protected]> Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com>
rleungx · May 25, 2023 · 69e47f6 · 69e47f6
1 parent 0f0d9c7
commit 69e47f6
Show file tree

Hide file tree

Showing 8 changed files with 215 additions and 39 deletions.
diff --git a/client/tso_service_discovery.go b/client/tso_service_discovery.go
@@ -488,17 +488,15 @@ func (c *tsoServiceDiscovery) updateMember() error {
 		}
 	}
 
-	oldPrimary, primarySwitched, secondaryChanged :=
+	oldPrimary, primarySwitched, _ :=
 		c.keyspaceGroupSD.update(keyspaceGroup, primaryAddr, secondaryAddrs, addrs)
 	if primarySwitched {
+		log.Info("[tso] updated keyspace group service discovery info",
+			zap.String("keyspace-group-service", keyspaceGroup.String()))
 		if err := c.afterPrimarySwitched(oldPrimary, primaryAddr); err != nil {
 			return err
 		}
 	}
-	if primarySwitched || secondaryChanged {
-		log.Info("[tso] updated keyspace group service discovery info",
-			zap.String("keyspace-group-service", keyspaceGroup.String()))
-	}
 
 	// Even if the primary address is empty, we still updated other returned info above, including the
 	// keyspace group info and the secondary addresses.

diff --git a/metrics/grafana/pd.json b/metrics/grafana/pd.json
@@ -10035,6 +10035,16 @@
               "legendFormat": "tso request/secs",
               "refId": "A",
               "step": 2
+            },
+            {
+              "expr": "sum(rate(pd_tso_events{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type,dc)",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{type}}-{{dc}}",
+              "hide": true,
+              "refId": "B",
+              "step": 2
             }
           ],
           "thresholds": [],

diff --git a/pkg/keyspace/keyspace.go b/pkg/keyspace/keyspace.go
@@ -647,13 +647,27 @@ func (manager *Manager) allocID() (uint32, error) {
 // PatrolKeyspaceAssignment is used to patrol all keyspaces and assign them to the keyspace groups.
 func (manager *Manager) PatrolKeyspaceAssignment() error {
 	var (
+		// Some statistics info.
+		start                  = time.Now()
+		patrolledKeyspaceCount uint64
+		assignedKeyspaceCount  uint64
 		// The current start ID of the patrol, used for logging.
 		currentStartID = manager.nextPatrolStartID
 		// The next start ID of the patrol, used for the next patrol.
 		nextStartID  = currentStartID
 		moreToPatrol = true
 		err          error
 	)
+	defer func() {
+		log.Debug("[keyspace] patrol keyspace assignment finished",
+			zap.Duration("cost", time.Since(start)),
+			zap.Uint64("patrolled-keyspace-count", patrolledKeyspaceCount),
+			zap.Uint64("assigned-keyspace-count", assignedKeyspaceCount),
+			zap.Int("batch-size", keyspacePatrolBatchSize),
+			zap.Uint32("current-start-id", currentStartID),
+			zap.Uint32("next-start-id", nextStartID),
+		)
+	}()
 	for moreToPatrol {
 		err = manager.store.RunInTxn(manager.ctx, func(txn kv.Txn) error {
 			defaultKeyspaceGroup, err := manager.kgm.store.LoadKeyspaceGroup(txn, utils.DefaultKeyspaceGroupID)
@@ -692,6 +706,7 @@ func (manager *Manager) PatrolKeyspaceAssignment() error {
 				if ks == nil {
 					continue
 				}
+				patrolledKeyspaceCount++
 				manager.metaLock.Lock(ks.Id)
 				if ks.Config == nil {
 					ks.Config = make(map[string]string, 1)
@@ -718,6 +733,7 @@ func (manager *Manager) PatrolKeyspaceAssignment() error {
 						zap.Uint32("keyspace-id", ks.Id), zap.Error(err))
 					return err
 				}
+				assignedKeyspaceCount++
 			}
 			if assigned {
 				err = manager.kgm.store.SaveKeyspaceGroup(txn, defaultKeyspaceGroup)

diff --git a/pkg/keyspace/keyspace_test.go b/pkg/keyspace/keyspace_test.go
@@ -434,3 +434,47 @@ func (suite *keyspaceTestSuite) TestPatrolKeyspaceAssignmentInBatch() {
 		re.Contains(defaultKeyspaceGroup.Keyspaces, uint32(i))
 	}
 }
+
+// Benchmark the keyspace assignment patrol.
+func BenchmarkPatrolKeyspaceAssignment1000(b *testing.B) {
+	benchmarkPatrolKeyspaceAssignmentN(1000, b)
+}
+
+func BenchmarkPatrolKeyspaceAssignment10000(b *testing.B) {
+	benchmarkPatrolKeyspaceAssignmentN(10000, b)
+}
+
+func BenchmarkPatrolKeyspaceAssignment100000(b *testing.B) {
+	benchmarkPatrolKeyspaceAssignmentN(100000, b)
+}
+
+func benchmarkPatrolKeyspaceAssignmentN(
+	n int, b *testing.B,
+) {
+	suite := new(keyspaceTestSuite)
+	suite.SetT(&testing.T{})
+	suite.SetupSuite()
+	suite.SetupTest()
+	re := suite.Require()
+	// Create some keyspaces without any keyspace group.
+	for i := 1; i <= n; i++ {
+		now := time.Now().Unix()
+		err := suite.manager.saveNewKeyspace(&keyspacepb.KeyspaceMeta{
+			Id:             uint32(i),
+			Name:           strconv.Itoa(i),
+			State:          keyspacepb.KeyspaceState_ENABLED,
+			CreatedAt:      now,
+			StateChangedAt: now,
+		})
+		re.NoError(err)
+	}
+	// Benchmark the keyspace assignment patrol.
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := suite.manager.PatrolKeyspaceAssignment()
+		re.NoError(err)
+	}
+	b.StopTimer()
+	suite.TearDownTest()
+	suite.TearDownSuite()
+}