util/stmtsummary: discard the plan if it is too long and enlarge the …

…tidb_stmt_summary_max_stmt_count value to 3000 (#25843) (#25872)

config/config.go

@@ -687,7 +687,7 @@ var defaultConf = Config{
 	StmtSummary: StmtSummary{
 		Enable:              true,
 		EnableInternalQuery: false,
-		MaxStmtCount:        200,
+		MaxStmtCount:        3000,
 		MaxSQLLength:        4096,
 		RefreshInterval:     1800,
 		HistorySize:         24,

config/config.toml.example

@@ -430,7 +430,7 @@ enable = true
 enable-internal-query = false
 
 # max number of statements kept in memory.
-max-stmt-count = 200
+max-stmt-count = 3000
 
 # max length of displayed normalized sql and sample sql.
 max-sql-length = 4096

util/plancodec/codec.go

@@ -40,6 +40,12 @@ const (
 	separatorStr   = "\t"
 )
 
+var (
+	// PlanDiscardedEncoded indicates the discard plan because it is too long
+	PlanDiscardedEncoded = "[discard]"
+	planDiscardedDecoded = "(plan discarded because too long)"
+)
+
 var decoderPool = sync.Pool{
 	New: func() interface{} {
 		return &planDecoder{}
@@ -87,6 +93,9 @@ type planInfo struct {
 func (pd *planDecoder) decode(planString string) (string, error) {
 	str, err := decompress(planString)
 	if err != nil {
+		if planString == PlanDiscardedEncoded {
+			return planDiscardedDecoded, nil
+		}
 		return "", err
 	}
 	return pd.buildPlanTree(str)

util/plancodec/codec_test.go

@@ -50,3 +50,9 @@ func (s *testPlanCodecSuite) TestEncodeTaskType(c *C) {
 	_, err = decodeTaskType("1_x")
 	c.Assert(err, NotNil)
 }
+
+func (s *testPlanCodecSuite) TestDecodeDiscardPlan(c *C) {
+	plan, err := DecodePlan(PlanDiscardedEncoded)
+	c.Assert(err, IsNil)
+	c.Assert(plan, DeepEquals, planDiscardedDecoded)
+}

util/stmtsummary/statement_summary.go

@@ -607,10 +607,15 @@ func (ssbd *stmtSummaryByDigest) collectHistorySummaries(historySize int) []*stm
 	return ssElements
 }
 
+var maxEncodedPlanSizeInBytes = 1024 * 1024
+
 func newStmtSummaryByDigestElement(sei *StmtExecInfo, beginTime int64, intervalSeconds int64) *stmtSummaryByDigestElement {
 	// sampleSQL / authUsers(sampleUser) / samplePlan / prevSQL / indexNames store the values shown at the first time,
 	// because it compacts performance to update every time.
 	samplePlan, planHint := sei.PlanGenerator()
+	if len(samplePlan) > maxEncodedPlanSizeInBytes {
+		samplePlan = plancodec.PlanDiscardedEncoded
+	}
 	ssElement := &stmtSummaryByDigestElement{
 		beginTime: beginTime,
 		sampleSQL: formatSQL(sei.OriginalSQL),

util/stmtsummary/statement_summary_test.go

@@ -30,6 +30,7 @@ import (
 	"github.com/pingcap/tidb/store/tikv"
 	"github.com/pingcap/tidb/types"
 	"github.com/pingcap/tidb/util/execdetails"
+	"github.com/pingcap/tidb/util/plancodec"
 )
 
 var _ = Suite(&testStmtSummarySuite{})
@@ -410,6 +411,31 @@ func (s *testStmtSummarySuite) TestAddStatement(c *C) {
 	c.Assert(s.ssMap.summaryMap.Size(), Equals, 4)
 	_, ok = s.ssMap.summaryMap.Get(key)
 	c.Assert(ok, IsTrue)
+
+	// Test for plan too large
+	stmtExecInfo7 := stmtExecInfo1
+	stmtExecInfo7.PlanDigest = "plan_digest7"
+	stmtExecInfo7.PlanGenerator = func() (string, string) {
+		buf := make([]byte, maxEncodedPlanSizeInBytes+1)
+		for i := range buf {
+			buf[i] = 'a'
+		}
+		return string(buf), ""
+	}
+	key = &stmtSummaryByDigestKey{
+		schemaName: stmtExecInfo7.SchemaName,
+		digest:     stmtExecInfo7.Digest,
+		planDigest: stmtExecInfo7.PlanDigest,
+	}
+	s.ssMap.AddStatement(stmtExecInfo7)
+	c.Assert(s.ssMap.summaryMap.Size(), Equals, 5)
+	v, ok := s.ssMap.summaryMap.Get(key)
+	c.Assert(ok, IsTrue)
+	stmt := v.(*stmtSummaryByDigest)
+	c.Assert(stmt.digest, DeepEquals, key.digest)
+	e := stmt.history.Back()
+	ssElement := e.Value.(*stmtSummaryByDigestElement)
+	c.Assert(ssElement.samplePlan, Equals, plancodec.PlanDiscardedEncoded)
 }
 
 func matchStmtSummaryByDigest(first, second *stmtSummaryByDigest) bool {