From 50f3f321f3c1220791c06c32c1d885d87c9afa92 Mon Sep 17 00:00:00 2001 From: crazycs Date: Thu, 1 Jul 2021 18:31:26 +0800 Subject: [PATCH 1/2] cherry pick #25843 to release-5.0 Signed-off-by: ti-srebot --- config/config.go | 2 +- config/config.toml.example | 2 +- util/plancodec/codec.go | 9 +++++++ util/plancodec/codec_test.go | 6 +++++ util/stmtsummary/statement_summary.go | 5 ++++ util/stmtsummary/statement_summary_test.go | 30 ++++++++++++++++++++++ 6 files changed, 52 insertions(+), 2 deletions(-) diff --git a/config/config.go b/config/config.go index 93ca18fccfdc4..b1b7a0b0586fc 100644 --- a/config/config.go +++ b/config/config.go @@ -646,7 +646,7 @@ var defaultConf = Config{ StmtSummary: StmtSummary{ Enable: true, EnableInternalQuery: false, - MaxStmtCount: 200, + MaxStmtCount: 3000, MaxSQLLength: 4096, RefreshInterval: 1800, HistorySize: 24, diff --git a/config/config.toml.example b/config/config.toml.example index 6db0b5c517277..6ac986bb95807 100644 --- a/config/config.toml.example +++ b/config/config.toml.example @@ -448,7 +448,7 @@ enable = true enable-internal-query = false # max number of statements kept in memory. -max-stmt-count = 200 +max-stmt-count = 3000 # max length of displayed normalized sql and sample sql. max-sql-length = 4096 diff --git a/util/plancodec/codec.go b/util/plancodec/codec.go index e7568493fed94..65fd2c64b899f 100644 --- a/util/plancodec/codec.go +++ b/util/plancodec/codec.go @@ -40,6 +40,12 @@ const ( separatorStr = "\t" ) +var ( + // PlanDiscardedEncoded indicates the discard plan because it is too long + PlanDiscardedEncoded = "[discard]" + planDiscardedDecoded = "(plan discarded because too long)" +) + var decoderPool = sync.Pool{ New: func() interface{} { return &planDecoder{} @@ -87,6 +93,9 @@ type planInfo struct { func (pd *planDecoder) decode(planString string) (string, error) { str, err := decompress(planString) if err != nil { + if planString == PlanDiscardedEncoded { + return planDiscardedDecoded, nil + } return "", err } return pd.buildPlanTree(str) diff --git a/util/plancodec/codec_test.go b/util/plancodec/codec_test.go index 1f98adda4cf99..a3375673c95d4 100644 --- a/util/plancodec/codec_test.go +++ b/util/plancodec/codec_test.go @@ -50,3 +50,9 @@ func (s *testPlanCodecSuite) TestEncodeTaskType(c *C) { _, err = decodeTaskType("1_x") c.Assert(err, NotNil) } + +func (s *testPlanCodecSuite) TestDecodeDiscardPlan(c *C) { + plan, err := DecodePlan(PlanDiscardedEncoded) + c.Assert(err, IsNil) + c.Assert(plan, DeepEquals, planDiscardedDecoded) +} diff --git a/util/stmtsummary/statement_summary.go b/util/stmtsummary/statement_summary.go index 35c93c4e926d2..4d1d4839c745c 100644 --- a/util/stmtsummary/statement_summary.go +++ b/util/stmtsummary/statement_summary.go @@ -615,10 +615,15 @@ func (ssbd *stmtSummaryByDigest) collectHistorySummaries(historySize int) []*stm return ssElements } +var maxEncodedPlanSizeInBytes = 1024 * 1024 + func newStmtSummaryByDigestElement(sei *StmtExecInfo, beginTime int64, intervalSeconds int64) *stmtSummaryByDigestElement { // sampleSQL / authUsers(sampleUser) / samplePlan / prevSQL / indexNames store the values shown at the first time, // because it compacts performance to update every time. samplePlan, planHint := sei.PlanGenerator() + if len(samplePlan) > maxEncodedPlanSizeInBytes { + samplePlan = plancodec.PlanDiscardedEncoded + } ssElement := &stmtSummaryByDigestElement{ beginTime: beginTime, sampleSQL: formatSQL(sei.OriginalSQL), diff --git a/util/stmtsummary/statement_summary_test.go b/util/stmtsummary/statement_summary_test.go index d81527494dbd1..83fa8f7b7bc67 100644 --- a/util/stmtsummary/statement_summary_test.go +++ b/util/stmtsummary/statement_summary_test.go @@ -30,6 +30,11 @@ import ( "github.com/pingcap/tidb/store/tikv" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/execdetails" +<<<<<<< HEAD +======= + "github.com/pingcap/tidb/util/plancodec" + "github.com/tikv/client-go/v2/util" +>>>>>>> c4424c292... util/stmtsummary: discard the plan if it is too long and enlarge the tidb_stmt_summary_max_stmt_count value to 3000 (#25843) ) var _ = Suite(&testStmtSummarySuite{}) @@ -423,6 +428,31 @@ func (s *testStmtSummarySuite) TestAddStatement(c *C) { c.Assert(s.ssMap.summaryMap.Size(), Equals, 4) _, ok = s.ssMap.summaryMap.Get(key) c.Assert(ok, IsTrue) + + // Test for plan too large + stmtExecInfo7 := stmtExecInfo1 + stmtExecInfo7.PlanDigest = "plan_digest7" + stmtExecInfo7.PlanGenerator = func() (string, string) { + buf := make([]byte, maxEncodedPlanSizeInBytes+1) + for i := range buf { + buf[i] = 'a' + } + return string(buf), "" + } + key = &stmtSummaryByDigestKey{ + schemaName: stmtExecInfo7.SchemaName, + digest: stmtExecInfo7.Digest, + planDigest: stmtExecInfo7.PlanDigest, + } + s.ssMap.AddStatement(stmtExecInfo7) + c.Assert(s.ssMap.summaryMap.Size(), Equals, 5) + v, ok := s.ssMap.summaryMap.Get(key) + c.Assert(ok, IsTrue) + stmt := v.(*stmtSummaryByDigest) + c.Assert(stmt.digest, DeepEquals, key.digest) + e := stmt.history.Back() + ssElement := e.Value.(*stmtSummaryByDigestElement) + c.Assert(ssElement.samplePlan, Equals, plancodec.PlanDiscardedEncoded) } func matchStmtSummaryByDigest(first, second *stmtSummaryByDigest) bool { From 61b83c0986679d2b53cd88831d14b660b35da1f5 Mon Sep 17 00:00:00 2001 From: crazycs Date: Mon, 5 Jul 2021 13:05:16 +0800 Subject: [PATCH 2/2] resolve conflict Signed-off-by: crazycs --- util/stmtsummary/statement_summary_test.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/util/stmtsummary/statement_summary_test.go b/util/stmtsummary/statement_summary_test.go index 83fa8f7b7bc67..f1ef5b6599b70 100644 --- a/util/stmtsummary/statement_summary_test.go +++ b/util/stmtsummary/statement_summary_test.go @@ -30,11 +30,7 @@ import ( "github.com/pingcap/tidb/store/tikv" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/execdetails" -<<<<<<< HEAD -======= "github.com/pingcap/tidb/util/plancodec" - "github.com/tikv/client-go/v2/util" ->>>>>>> c4424c292... util/stmtsummary: discard the plan if it is too long and enlarge the tidb_stmt_summary_max_stmt_count value to 3000 (#25843) ) var _ = Suite(&testStmtSummarySuite{})