infoschema, util/stmtsummary: enhance statements_summary.

infoschema/tables.go

@@ -1131,8 +1131,8 @@ var tableStatementsSummaryCols = []columnInfo{
 	{name: "SUMMARY_BEGIN_TIME", tp: mysql.TypeTimestamp, size: 26, flag: mysql.NotNullFlag, comment: "Begin time of this summary"},
 	{name: "SUMMARY_END_TIME", tp: mysql.TypeTimestamp, size: 26, flag: mysql.NotNullFlag, comment: "End time of this summary"},
 	{name: "STMT_TYPE", tp: mysql.TypeVarchar, size: 64, flag: mysql.NotNullFlag, comment: "Statement type"},
-	{name: "SCHEMA_NAME", tp: mysql.TypeVarchar, size: 64, flag: mysql.NotNullFlag, comment: "Current schema"},
-	{name: "DIGEST", tp: mysql.TypeVarchar, size: 64, flag: mysql.NotNullFlag},
+	{name: "SCHEMA_NAME", tp: mysql.TypeVarchar, size: 64, comment: "Current schema"},
+	{name: "DIGEST", tp: mysql.TypeVarchar, size: 64},
 	{name: "DIGEST_TEXT", tp: mysql.TypeBlob, size: types.UnspecifiedLength, flag: mysql.NotNullFlag, comment: "Normalized statement"},
 	{name: "TABLE_NAMES", tp: mysql.TypeBlob, size: types.UnspecifiedLength, comment: "Involved tables"},
 	{name: "INDEX_NAMES", tp: mysql.TypeBlob, size: types.UnspecifiedLength, comment: "Used indices"},

infoschema/tables_test.go

@@ -1401,6 +1401,71 @@ func (s *testTableSuite) TestSimpleStmtSummaryEvictedCount(c *C) {
 	tk.MustExec("set global tidb_stmt_summary_refresh_interval = 1800")
 }
 
+func (s *testTableSuite) TestStmtSummaryTableOther(c *C) {
+	interval := int64(1800)
+	tk := s.newTestKitWithRoot(c)
+	tk.MustExec(fmt.Sprintf("set global tidb_stmt_summary_refresh_interval=%v", interval))
+	tk.MustExec("set global tidb_enable_stmt_summary=0")
+	tk.MustExec("set global tidb_enable_stmt_summary=1")
+	// set stmt size to 1
+	// first sql
+	tk.MustExec("set global tidb_stmt_summary_max_stmt_count=1")
+	defer tk.MustExec("set global tidb_stmt_summary_max_stmt_count=100")
+	// second sql, evict first sql from stmt_summary
+	tk.MustExec("show databases;")
+	// third sql, evict second sql from stmt_summary
+	tk.MustQuery("SELECT DIGEST_TEXT, DIGEST FROM `INFORMATION_SCHEMA`.`STATEMENTS_SUMMARY`;").
+		Check(testkit.Rows(
+			// digest in cache
+			// "show databases ;"
+			"show databases ; dcd020298c5f79e8dc9d63b3098083601614a04a52db458738347d15ea5712a1",
+			// digest evicted
+			" <nil>",
+		))
+	// forth sql, evict third sql from stmt_summary
+	tk.MustQuery("SELECT SCHEMA_NAME FROM `INFORMATION_SCHEMA`.`STATEMENTS_SUMMARY`;").
+		Check(testkit.Rows(
+			// digest in cache
+			"test", // select xx from yy;
+			// digest evicted
+			"<nil>",
+		))
+}
+
+func (s *testTableSuite) TestStmtSummaryHistoryTableOther(c *C) {
+	tk := s.newTestKitWithRoot(c)
+	// disable refreshing summary
+	interval := int64(9999)
+	tk.MustExec("set global tidb_stmt_summary_max_stmt_count = 1")
+	defer tk.MustExec("set global tidb_stmt_summary_max_stmt_count = 100")
+	tk.MustExec(fmt.Sprintf("set global tidb_stmt_summary_refresh_interval = %v", interval))
+	defer tk.MustExec(fmt.Sprintf("set global tidb_stmt_summary_refresh_interval = %v", 1800))
+
+	tk.MustExec("set global tidb_enable_stmt_summary = 0")
+	tk.MustExec("set global tidb_enable_stmt_summary = 1")
+	// first sql
+	tk.MustExec("set global tidb_stmt_summary_max_stmt_count=1")
+	// second sql, evict first sql from stmt_summary
+	tk.MustExec("show databases;")
+	// third sql, evict second sql from stmt_summary
+	tk.MustQuery("SELECT DIGEST_TEXT, DIGEST FROM `INFORMATION_SCHEMA`.`STATEMENTS_SUMMARY_HISTORY`;").
+		Check(testkit.Rows(
+			// digest in cache
+			// "show databases ;"
+			"show databases ; dcd020298c5f79e8dc9d63b3098083601614a04a52db458738347d15ea5712a1",
+			// digest evicted
+			" <nil>",
+		))
+	// forth sql, evict third sql from stmt_summary
+	tk.MustQuery("SELECT SCHEMA_NAME FROM `INFORMATION_SCHEMA`.`STATEMENTS_SUMMARY_HISTORY`;").
+		Check(testkit.Rows(
+			// digest in cache
+			"test", // select xx from yy;
+			// digest evicted
+			"<nil>",
+		))
+}
+
 func (s *testTableSuite) TestPerformanceSchemaforPlanCache(c *C) {
 	orgEnable := plannercore.PreparedPlanCacheEnabled()
 	defer func() {

util/stmtsummary/evicted.go

@@ -15,6 +15,8 @@ package stmtsummary
 
 import (
 	"container/list"
+	"math"
+	"sync"
 	"time"
 
 	"github.com/pingcap/parser/mysql"
@@ -23,6 +25,7 @@ import (
 
 // stmtSummaryByDigestEvicted contents digests evicted from stmtSummaryByDigestMap
 type stmtSummaryByDigestEvicted struct {
+	sync.Mutex
 	// record evicted data in intervals
 	// latest history data is Back()
 	history *list.List
@@ -34,8 +37,10 @@ type stmtSummaryByDigestEvictedElement struct {
 	beginTime int64
 	// endTime is the end time of current interval
 	endTime int64
-	// *Kinds* of digest being evicted
+	// digestKeyMap contains *Kinds* of digest being evicted
 	digestKeyMap map[string]struct{}
+	// otherSummary contains summed up information of evicted elements
+	otherSummary *stmtSummaryByDigestElement
 }
 
 // spawn a new pointer to stmtSummaryByDigestEvicted
@@ -51,6 +56,14 @@ func newStmtSummaryByDigestEvictedElement(beginTime int64, endTime int64) *stmtS
 		beginTime:    beginTime,
 		endTime:      endTime,
 		digestKeyMap: make(map[string]struct{}),
+		otherSummary: &stmtSummaryByDigestElement{
+			beginTime:    beginTime,
+			endTime:      endTime,
+			authUsers:    make(map[string]struct{}),
+			minLatency:   time.Duration(math.MaxInt64),
+			backoffTypes: make(map[string]int),
+			firstSeen:    time.Unix(endTime, 0),
+		},
 	}
 }
 
@@ -63,6 +76,9 @@ func (ssbde *stmtSummaryByDigestEvicted) AddEvicted(evictedKey *stmtSummaryByDig
 	evictedValue.Lock()
 	defer evictedValue.Unlock()
 
+	ssbde.Lock()
+	defer ssbde.Unlock()
+
 	if evictedValue.history == nil {
 		return
 	}
@@ -130,13 +146,16 @@ func (ssbde *stmtSummaryByDigestEvicted) AddEvicted(evictedKey *stmtSummaryByDig
 
 // Clear up all records in stmtSummaryByDigestEvicted
 func (ssbde *stmtSummaryByDigestEvicted) Clear() {
+	ssbde.Lock()
+	defer ssbde.Unlock()
 	ssbde.history.Init()
 }
 
 // add an evicted record to stmtSummaryByDigestEvictedElement
 func (seElement *stmtSummaryByDigestEvictedElement) addEvicted(digestKey *stmtSummaryByDigestKey, digestValue *stmtSummaryByDigestElement) {
 	if digestKey != nil {
 		seElement.digestKeyMap[string(digestKey.Hash())] = struct{}{}
+		addInfo(seElement.otherSummary, digestValue)
 	}
 }
 
@@ -190,3 +209,211 @@ func (seElement *stmtSummaryByDigestEvictedElement) toEvictedCountDatum() []type
 func (ssMap *stmtSummaryByDigestMap) ToEvictedCountDatum() [][]types.Datum {
 	return ssMap.other.ToEvictedCountDatum()
 }
+
+func (ssbde *stmtSummaryByDigestEvicted) toCurrentDatum() []types.Datum {
+	var seElement *stmtSummaryByDigestEvictedElement
+
+	ssbde.Lock()
+	if ssbde.history.Len() > 0 {
+		seElement = ssbde.history.Back().Value.(*stmtSummaryByDigestEvictedElement)
+	}
+	ssbde.Unlock()
+
+	if seElement == nil {
+		return nil
+	}
+
+	return seElement.toDatum()
+}
+
+func (ssbde *stmtSummaryByDigestEvicted) toHistoryDatum(historySize int) [][]types.Datum {
+	// Collect all history summaries to an array.
+	ssbde.Lock()
+	seElements := ssbde.collectHistorySummaries(historySize)
+	ssbde.Unlock()
+	rows := make([][]types.Datum, 0, len(seElements))
+
+	for _, seElement := range seElements {
+		rows = append(rows, seElement.toDatum())
+	}
+	return rows
+}
+
+func (ssbde *stmtSummaryByDigestEvicted) collectHistorySummaries(historySize int) []*stmtSummaryByDigestEvictedElement {
+	lst := make([]*stmtSummaryByDigestEvictedElement, 0, ssbde.history.Len())
+	for element := ssbde.history.Front(); element != nil && len(lst) < historySize; element = element.Next() {
+		seElement := element.Value.(*stmtSummaryByDigestEvictedElement)
+		lst = append(lst, seElement)
+	}
+	return lst
+}
+
+func (seElement *stmtSummaryByDigestEvictedElement) toDatum() []types.Datum {
+	return seElement.otherSummary.toDatum(new(stmtSummaryByDigest))
+}
+
+// addInfo adds information in addWith into addTo.
+func addInfo(addTo *stmtSummaryByDigestElement, addWith *stmtSummaryByDigestElement) {
+	addTo.Lock()
+	defer addTo.Unlock()
+
+	// user
+	for user := range addWith.authUsers {
+		addTo.authUsers[user] = struct{}{}
+	}
+
+	// execCount and sumWarnings
+	addTo.execCount += addWith.execCount
+	addTo.sumWarnings += addWith.sumWarnings
+
+	// latency
+	addTo.sumLatency += addWith.sumLatency
+	if addTo.maxLatency < addWith.maxLatency {
+		addTo.maxLatency = addWith.maxLatency
+	}
+	if addTo.minLatency > addWith.minLatency {
+		addTo.minLatency = addWith.minLatency
+	}
+	addTo.sumParseLatency += addWith.sumParseLatency
+	if addTo.maxParseLatency < addWith.maxParseLatency {
+		addTo.maxParseLatency = addWith.maxParseLatency
+	}
+	addTo.sumCompileLatency += addWith.sumCompileLatency
+	if addTo.maxCompileLatency < addWith.maxCompileLatency {
+		addTo.maxCompileLatency = addWith.maxCompileLatency
+	}
+
+	// coprocessor
+	addTo.sumNumCopTasks += addWith.sumNumCopTasks
+	if addTo.maxCopProcessTime < addWith.maxCopProcessTime {
+		addTo.maxCopProcessTime = addWith.maxCopProcessTime
+		addTo.maxCopProcessAddress = addWith.maxCopProcessAddress
+	}
+	if addTo.maxCopWaitTime < addWith.maxCopWaitTime {
+		addTo.maxCopWaitTime = addWith.maxCopWaitTime
+		addTo.maxCopWaitAddress = addWith.maxCopWaitAddress
+	}
+
+	// TiKV
+	addTo.sumProcessTime += addWith.sumProcessTime
+	if addTo.maxProcessTime < addWith.maxProcessTime {
+		addTo.maxProcessTime = addWith.maxProcessTime
+	}
+	addTo.sumWaitTime += addWith.sumWaitTime
+	if addTo.maxWaitTime < addWith.maxWaitTime {
+		addTo.maxWaitTime = addWith.maxWaitTime
+	}
+	addTo.sumBackoffTime += addWith.sumBackoffTime
+	if addTo.maxBackoffTime < addWith.maxBackoffTime {
+		addTo.maxBackoffTime = addWith.maxBackoffTime
+	}
+
+	addTo.sumTotalKeys += addWith.sumTotalKeys
+	if addTo.maxTotalKeys < addWith.maxTotalKeys {
+		addTo.maxTotalKeys = addWith.maxTotalKeys
+	}
+	addTo.sumProcessedKeys += addWith.sumProcessedKeys
+	if addTo.maxProcessedKeys < addWith.maxProcessedKeys {
+		addTo.maxProcessedKeys = addWith.maxProcessedKeys
+	}
+	addTo.sumRocksdbDeleteSkippedCount += addWith.sumRocksdbDeleteSkippedCount
+	if addTo.maxRocksdbDeleteSkippedCount < addWith.maxRocksdbDeleteSkippedCount {
+		addTo.maxRocksdbDeleteSkippedCount = addWith.maxRocksdbDeleteSkippedCount
+	}
+	addTo.sumRocksdbKeySkippedCount += addWith.sumRocksdbKeySkippedCount
+	if addTo.maxRocksdbKeySkippedCount < addWith.maxRocksdbKeySkippedCount {
+		addTo.maxRocksdbKeySkippedCount = addWith.maxRocksdbKeySkippedCount
+	}
+	addTo.sumRocksdbBlockCacheHitCount += addWith.sumRocksdbBlockCacheHitCount
+	if addTo.maxRocksdbBlockCacheHitCount < addWith.maxRocksdbBlockCacheHitCount {
+		addTo.maxRocksdbBlockCacheHitCount = addWith.maxRocksdbBlockCacheHitCount
+	}
+	addTo.sumRocksdbBlockReadCount += addWith.sumRocksdbBlockReadCount
+	if addTo.maxRocksdbBlockReadCount < addWith.maxRocksdbBlockReadCount {
+		addTo.maxRocksdbBlockReadCount = addWith.maxRocksdbBlockReadCount
+	}
+	addTo.sumRocksdbBlockReadByte += addWith.sumRocksdbBlockReadByte
+	if addTo.maxRocksdbBlockReadByte < addWith.maxRocksdbBlockReadByte {
+		addTo.maxRocksdbBlockReadByte = addWith.maxRocksdbBlockReadByte
+	}
+
+	// txn
+	addTo.commitCount += addWith.commitCount
+	addTo.sumPrewriteTime += addWith.sumPrewriteTime
+	if addTo.maxPrewriteTime < addWith.maxPrewriteTime {
+		addTo.maxPrewriteTime = addWith.maxPrewriteTime
+	}
+	addTo.sumCommitTime += addWith.sumCommitTime
+	if addTo.maxCommitTime < addWith.maxCommitTime {
+		addTo.maxCommitTime = addWith.maxCommitTime
+	}
+	addTo.sumGetCommitTsTime += addWith.sumGetCommitTsTime
+	if addTo.maxGetCommitTsTime < addWith.maxGetCommitTsTime {
+		addTo.maxGetCommitTsTime = addWith.maxGetCommitTsTime
+	}
+	addTo.sumCommitBackoffTime += addWith.sumCommitBackoffTime
+	if addTo.maxCommitBackoffTime < addWith.maxCommitBackoffTime {
+		addTo.maxCommitBackoffTime = addWith.maxCommitBackoffTime
+	}
+	addTo.sumResolveLockTime += addWith.sumResolveLockTime
+	if addTo.maxResolveLockTime < addWith.maxResolveLockTime {
+		addTo.maxResolveLockTime = addWith.maxResolveLockTime
+	}
+	addTo.sumLocalLatchTime += addWith.sumLocalLatchTime
+	if addTo.maxLocalLatchTime < addWith.maxLocalLatchTime {
+		addTo.maxLocalLatchTime = addWith.maxLocalLatchTime
+	}
+	addTo.sumWriteKeys += addWith.sumWriteKeys
+	if addTo.maxWriteKeys < addWith.maxWriteKeys {
+		addTo.maxWriteKeys = addWith.maxWriteKeys
+	}
+	addTo.sumWriteSize += addWith.sumWriteSize
+	if addTo.maxWriteSize < addWith.maxWriteSize {
+		addTo.maxWriteSize = addWith.maxWriteSize
+	}
+	addTo.sumPrewriteRegionNum += addWith.sumPrewriteRegionNum
+	if addTo.maxPrewriteRegionNum < addWith.maxPrewriteRegionNum {
+		addTo.maxPrewriteRegionNum = addWith.maxPrewriteRegionNum
+	}
+	addTo.sumTxnRetry += addWith.sumTxnRetry
+	if addTo.maxTxnRetry < addWith.maxTxnRetry {
+		addTo.maxTxnRetry = addWith.maxTxnRetry
+	}
+	addTo.sumBackoffTimes += addWith.sumBackoffTimes
+	for backoffType, backoffValue := range addWith.backoffTypes {
+		_, ok := addTo.backoffTypes[backoffType]
+		if ok {
+			addTo.backoffTypes[backoffType] += backoffValue
+		} else {
+			addTo.backoffTypes[backoffType] = backoffValue
+		}
+	}
+
+	// plan cache
+	addTo.planCacheHits += addWith.planCacheHits
+
+	// other
+	addTo.sumAffectedRows += addWith.sumAffectedRows
+	addTo.sumMem += addWith.sumMem
+	if addTo.maxMem < addWith.maxMem {
+		addTo.maxMem = addWith.maxMem
+	}
+	addTo.sumDisk += addWith.sumDisk
+	if addTo.maxDisk < addWith.maxDisk {
+		addTo.maxDisk = addWith.maxDisk
+	}
+	if addTo.firstSeen.After(addWith.firstSeen) {
+		addTo.firstSeen = addWith.firstSeen
+	}
+	if addTo.lastSeen.Before(addWith.lastSeen) {
+		addTo.lastSeen = addWith.lastSeen
+	}
+	addTo.execRetryCount += addWith.execRetryCount
+	addTo.execRetryTime += addWith.execRetryTime
+	addTo.sumKVTotal += addWith.sumKVTotal
+	addTo.sumPDTotal += addWith.sumPDTotal
+	addTo.sumBackoffTotal += addWith.sumBackoffTotal
+	addTo.sumWriteSQLRespTotal += addWith.sumWriteSQLRespTotal
+
+	addTo.sumErrors += addWith.sumErrors
+}