From fac17a254ebe2898b5ebf671a8007f0697d1f94c Mon Sep 17 00:00:00 2001 From: crazycs Date: Tue, 29 Jun 2021 20:35:25 +0800 Subject: [PATCH 01/16] topsql: use new cache policy for top-n SQL (#25744) --- sessionctx/variable/tidb_vars.go | 2 +- util/topsql/reporter/client.go | 2 +- util/topsql/reporter/reporter.go | 121 ++++++++++++++++---------- util/topsql/reporter/reporter_test.go | 72 +++++++++++++++ 4 files changed, 149 insertions(+), 48 deletions(-) diff --git a/sessionctx/variable/tidb_vars.go b/sessionctx/variable/tidb_vars.go index 3a6ae0a067576..66c7ae9c18380 100644 --- a/sessionctx/variable/tidb_vars.go +++ b/sessionctx/variable/tidb_vars.go @@ -712,7 +712,7 @@ const ( DefTiDBTopSQLEnable = false DefTiDBTopSQLAgentAddress = "" DefTiDBTopSQLPrecisionSeconds = 1 - DefTiDBTopSQLMaxStatementCount = 2000 + DefTiDBTopSQLMaxStatementCount = 200 DefTiDBTopSQLMaxCollect = 10000 DefTiDBTopSQLReportIntervalSeconds = 60 DefTiDBEnableGlobalTemporaryTable = false diff --git a/util/topsql/reporter/client.go b/util/topsql/reporter/client.go index 41d71400c3959..300c055dd13c5 100644 --- a/util/topsql/reporter/client.go +++ b/util/topsql/reporter/client.go @@ -99,7 +99,7 @@ func (r *GRPCReportClient) Close() { } // sendBatchCPUTimeRecord sends a batch of TopSQL records by stream. -func (r *GRPCReportClient) sendBatchCPUTimeRecord(ctx context.Context, records map[string]*dataPoints) error { +func (r *GRPCReportClient) sendBatchCPUTimeRecord(ctx context.Context, records []*dataPoints) error { if len(records) == 0 { return nil } diff --git a/util/topsql/reporter/reporter.go b/util/topsql/reporter/reporter.go index 34163383f160e..97f79c8395221 100644 --- a/util/topsql/reporter/reporter.go +++ b/util/topsql/reporter/reporter.go @@ -61,25 +61,31 @@ type dataPoints struct { CPUTimeMsTotal uint64 } -// cpuTimeSort is used to sort TopSQL records by total CPU time -type cpuTimeSort struct { - Key string - SQLDigest []byte - PlanDigest []byte - CPUTimeMsTotal uint64 // The sorting field +type dataPointsOrderByCPUTime []*dataPoints + +func (t dataPointsOrderByCPUTime) Len() int { + return len(t) } -type cpuTimeSortSlice []cpuTimeSort +func (t dataPointsOrderByCPUTime) Less(i, j int) bool { + // We need find the kth largest value, so here should use > + return t[i].CPUTimeMsTotal > t[j].CPUTimeMsTotal +} +func (t dataPointsOrderByCPUTime) Swap(i, j int) { + t[i], t[j] = t[j], t[i] +} -func (t cpuTimeSortSlice) Len() int { +type sqlCPUTimeRecordSlice []tracecpu.SQLCPUTimeRecord + +func (t sqlCPUTimeRecordSlice) Len() int { return len(t) } -func (t cpuTimeSortSlice) Less(i, j int) bool { +func (t sqlCPUTimeRecordSlice) Less(i, j int) bool { // We need find the kth largest value, so here should use > - return t[i].CPUTimeMsTotal > t[j].CPUTimeMsTotal + return t[i].CPUTimeMs > t[j].CPUTimeMs } -func (t cpuTimeSortSlice) Swap(i, j int) { +func (t sqlCPUTimeRecordSlice) Swap(i, j int) { t[i], t[j] = t[j], t[i] } @@ -215,16 +221,45 @@ func encodeKey(buf *bytes.Buffer, sqlDigest, planDigest []byte) string { return buf.String() } -// doCollect uses a hashmap to store records in every second, and evict when necessary. +func getTopNRecords(records []tracecpu.SQLCPUTimeRecord) (topN, shouldEvict []tracecpu.SQLCPUTimeRecord) { + maxStmt := int(variable.TopSQLVariable.MaxStatementCount.Load()) + if len(records) <= maxStmt { + return records, nil + } + if err := quickselect.QuickSelect(sqlCPUTimeRecordSlice(records), maxStmt); err != nil { + // skip eviction + return records, nil + } + return records[:maxStmt], records[maxStmt:] +} + +func getTopNDataPoints(records []*dataPoints) (topN, shouldEvict []*dataPoints) { + maxStmt := int(variable.TopSQLVariable.MaxStatementCount.Load()) + if len(records) <= maxStmt { + return records, nil + } + if err := quickselect.QuickSelect(dataPointsOrderByCPUTime(records), maxStmt); err != nil { + // skip eviction + return records, nil + } + return records[:maxStmt], records[maxStmt:] +} + +// doCollect collects top N records of each round into collectTarget, and evict the data that is not in top N. func (tsr *RemoteTopSQLReporter) doCollect( collectTarget map[string]*dataPoints, timestamp uint64, records []tracecpu.SQLCPUTimeRecord) { defer util.Recover("top-sql", "doCollect", nil, false) + // Get top N records of each round records. + var evicted []tracecpu.SQLCPUTimeRecord + records, evicted = getTopNRecords(records) + keyBuf := bytes.NewBuffer(make([]byte, 0, 64)) listCapacity := int(variable.TopSQLVariable.ReportIntervalSeconds.Load()/variable.TopSQLVariable.PrecisionSeconds.Load() + 1) if listCapacity < 1 { listCapacity = 1 } + // Collect the top N records to collectTarget for each round. for _, record := range records { key := encodeKey(keyBuf, record.SQLDigest, record.PlanDigest) entry, exist := collectTarget[key] @@ -245,37 +280,15 @@ func (tsr *RemoteTopSQLReporter) doCollect( entry.CPUTimeMsTotal += uint64(record.CPUTimeMs) } - // evict records according to `MaxStatementCount` variable. - // TODO: Better to pass in the variable in the constructor, instead of referencing directly. - maxStmt := int(variable.TopSQLVariable.MaxStatementCount.Load()) - if len(collectTarget) <= maxStmt { - return - } - - // find the max CPUTimeMsTotal that should be evicted - digestCPUTimeList := make([]cpuTimeSort, len(collectTarget)) - idx := 0 - for key, value := range collectTarget { - digestCPUTimeList[idx] = cpuTimeSort{ - Key: key, - SQLDigest: value.SQLDigest, - PlanDigest: value.PlanDigest, - CPUTimeMsTotal: value.CPUTimeMsTotal, - } - idx++ - } - - // QuickSelect will only return error when the second parameter is out of range - if err := quickselect.QuickSelect(cpuTimeSortSlice(digestCPUTimeList), maxStmt); err != nil { - // skip eviction - return - } - - itemsToEvict := digestCPUTimeList[maxStmt:] + // Evict redundant data. normalizedSQLMap := tsr.normalizedSQLMap.Load().(*sync.Map) normalizedPlanMap := tsr.normalizedPlanMap.Load().(*sync.Map) - for _, evict := range itemsToEvict { - delete(collectTarget, evict.Key) + for _, evict := range evicted { + key := encodeKey(keyBuf, evict.SQLDigest, evict.PlanDigest) + _, ok := collectTarget[key] + if ok { + continue + } _, loaded := normalizedSQLMap.LoadAndDelete(string(evict.SQLDigest)) if loaded { tsr.sqlMapLength.Add(-1) @@ -290,11 +303,13 @@ func (tsr *RemoteTopSQLReporter) doCollect( // takeDataAndSendToReportChan takes out (resets) collected data. These data will be send to a report channel // for reporting later. func (tsr *RemoteTopSQLReporter) takeDataAndSendToReportChan(collectedDataPtr *map[string]*dataPoints) { - data := reportData{ - collectedData: *collectedDataPtr, - normalizedSQLMap: tsr.normalizedSQLMap.Load().(*sync.Map), - normalizedPlanMap: tsr.normalizedPlanMap.Load().(*sync.Map), + // Fetch TopN dataPoints. + records := make([]*dataPoints, 0, len(*collectedDataPtr)) + for _, v := range *collectedDataPtr { + records = append(records, v) } + normalizedSQLMap := tsr.normalizedSQLMap.Load().(*sync.Map) + normalizedPlanMap := tsr.normalizedPlanMap.Load().(*sync.Map) // Reset data for next report. *collectedDataPtr = make(map[string]*dataPoints) @@ -303,6 +318,20 @@ func (tsr *RemoteTopSQLReporter) takeDataAndSendToReportChan(collectedDataPtr *m tsr.sqlMapLength.Store(0) tsr.planMapLength.Store(0) + // Evict redundant data. + var evicted []*dataPoints + records, evicted = getTopNDataPoints(records) + for _, evict := range evicted { + normalizedSQLMap.LoadAndDelete(string(evict.SQLDigest)) + normalizedPlanMap.LoadAndDelete(string(evict.PlanDigest)) + } + + data := reportData{ + collectedData: records, + normalizedSQLMap: normalizedSQLMap, + normalizedPlanMap: normalizedPlanMap, + } + // Send to report channel. When channel is full, data will be dropped. select { case tsr.reportDataChan <- data: @@ -312,7 +341,7 @@ func (tsr *RemoteTopSQLReporter) takeDataAndSendToReportChan(collectedDataPtr *m // reportData contains data that reporter sends to the agent type reportData struct { - collectedData map[string]*dataPoints + collectedData []*dataPoints normalizedSQLMap *sync.Map normalizedPlanMap *sync.Map } diff --git a/util/topsql/reporter/reporter_test.go b/util/topsql/reporter/reporter_test.go index 8daa6997cdb2d..8728f992d80ae 100644 --- a/util/topsql/reporter/reporter_test.go +++ b/util/topsql/reporter/reporter_test.go @@ -169,6 +169,78 @@ func (s *testTopSQLReporter) TestCollectAndEvicted(c *C) { } } +func (s *testTopSQLReporter) newSQLCPUTimeRecord(tsr *RemoteTopSQLReporter, sqlID int, cpuTimeMs uint32) tracecpu.SQLCPUTimeRecord { + key := []byte("sqlDigest" + strconv.Itoa(sqlID)) + value := "sqlNormalized" + strconv.Itoa(sqlID) + tsr.RegisterSQL(key, value) + + key = []byte("planDigest" + strconv.Itoa(sqlID)) + value = "planNormalized" + strconv.Itoa(sqlID) + tsr.RegisterPlan(key, value) + + return tracecpu.SQLCPUTimeRecord{ + SQLDigest: []byte("sqlDigest" + strconv.Itoa(sqlID)), + PlanDigest: []byte("planDigest" + strconv.Itoa(sqlID)), + CPUTimeMs: cpuTimeMs, + } +} + +func (s *testTopSQLReporter) collectAndWait(tsr *RemoteTopSQLReporter, timestamp uint64, records []tracecpu.SQLCPUTimeRecord) { + tsr.Collect(timestamp, records) + time.Sleep(time.Millisecond * 100) +} + +func (s *testTopSQLReporter) TestCollectAndTopN(c *C) { + agentServer, err := mock.StartMockAgentServer() + c.Assert(err, IsNil) + defer agentServer.Stop() + + tsr := setupRemoteTopSQLReporter(2, 1, agentServer.Address()) + defer tsr.Close() + + records := []tracecpu.SQLCPUTimeRecord{ + s.newSQLCPUTimeRecord(tsr, 1, 1), + s.newSQLCPUTimeRecord(tsr, 2, 2), + } + s.collectAndWait(tsr, 1, records) + + records = []tracecpu.SQLCPUTimeRecord{ + s.newSQLCPUTimeRecord(tsr, 3, 3), + s.newSQLCPUTimeRecord(tsr, 1, 1), + } + s.collectAndWait(tsr, 2, records) + + records = []tracecpu.SQLCPUTimeRecord{ + s.newSQLCPUTimeRecord(tsr, 4, 1), + s.newSQLCPUTimeRecord(tsr, 1, 1), + } + s.collectAndWait(tsr, 3, records) + + // Wait agent server collect finish. + agentServer.WaitCollectCnt(1, time.Second*10) + + // check for equality of server received batch and the original data + results := agentServer.GetLatestRecords() + c.Assert(results, HasLen, 2) + for _, req := range results { + id := 0 + prefix := "sqlDigest" + if strings.HasPrefix(string(req.SqlDigest), prefix) { + n, err := strconv.Atoi(string(req.SqlDigest)[len(prefix):]) + c.Assert(err, IsNil) + id = n + } + if id != 1 && id != 3 { + c.Fatalf("the id should be 1 or 3, got: %v", id) + } + total := uint32(0) + for _, v := range req.RecordListCpuTimeMs { + total += v + } + c.Assert(total, Equals, uint32(3)) + } +} + func (s *testTopSQLReporter) TestCollectCapacity(c *C) { tsr := setupRemoteTopSQLReporter(maxSQLNum, 60, "") defer tsr.Close() From 7e53276296de14adca0c8e37542a66392ea5af21 Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Tue, 29 Jun 2021 23:21:25 +0800 Subject: [PATCH 02/16] domain,session: fix annoying log when running session bench test (#25796) --- domain/domain.go | 5 +++- executor/executor_test.go | 2 +- executor/index_lookup_hash_join.go | 2 +- session/bench_test.go | 39 +++++++++++++++++------------- 4 files changed, 28 insertions(+), 20 deletions(-) diff --git a/domain/domain.go b/domain/domain.go index 6f46efe08b9c9..fe82fd8f65222 100644 --- a/domain/domain.go +++ b/domain/domain.go @@ -622,10 +622,13 @@ func (do *Domain) Close() { terror.Log(errors.Trace(do.etcdClient.Close())) } - do.sysSessionPool.Close() do.slowQuery.Close() + do.cancel() do.wg.Wait() + + do.sysSessionPool.Close() + logutil.BgLogger().Info("domain closed", zap.Duration("take time", time.Since(startTime))) } diff --git a/executor/executor_test.go b/executor/executor_test.go index 4d4d19a07baef..3c7c7a0054652 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -5733,8 +5733,8 @@ func (s *testRecoverTable) SetUpSuite(c *C) { } func (s *testRecoverTable) TearDownSuite(c *C) { - s.store.Close() s.dom.Close() + s.store.Close() } func (s *testRecoverTable) mockGC(tk *testkit.TestKit) (string, string, string, func()) { diff --git a/executor/index_lookup_hash_join.go b/executor/index_lookup_hash_join.go index 62fa460c39111..ed7da54baf3f5 100644 --- a/executor/index_lookup_hash_join.go +++ b/executor/index_lookup_hash_join.go @@ -598,10 +598,10 @@ func (iw *indexHashJoinInnerWorker) handleTask(ctx context.Context, task *indexH // TODO(XuHuaiyu): we may always use the smaller side to build the hashtable. go util.WithRecovery(func() { iw.buildHashTableForOuterResult(ctx, task, h) }, iw.handleHashJoinInnerWorkerPanic) err := iw.fetchInnerResults(ctx, task.lookUpJoinTask) + iw.wg.Wait() if err != nil { return err } - iw.wg.Wait() joinStartTime = time.Now() if !task.keepOuterOrder { diff --git a/session/bench_test.go b/session/bench_test.go index 02d1889f9b73d..590cab932891b 100644 --- a/session/bench_test.go +++ b/session/bench_test.go @@ -23,6 +23,7 @@ import ( "time" "github.com/pingcap/log" + "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/store/mockstore" @@ -36,6 +37,10 @@ var smallCount = 100 var bigCount = 10000 func prepareBenchSession() (Session, *domain.Domain, kv.Storage) { + config.UpdateGlobal(func(cfg *config.Config) { + cfg.Log.EnableSlowLog = false + }) + store, err := mockstore.NewMockStore() if err != nil { logutil.BgLogger().Fatal(err.Error()) @@ -108,8 +113,8 @@ func BenchmarkBasic(b *testing.B) { se, do, st := prepareBenchSession() defer func() { se.Close() - st.Close() do.Close() + st.Close() }() b.ResetTimer() for i := 0; i < b.N; i++ { @@ -127,8 +132,8 @@ func BenchmarkTableScan(b *testing.B) { se, do, st := prepareBenchSession() defer func() { se.Close() - st.Close() do.Close() + st.Close() }() prepareBenchData(se, "int", "%v", smallCount) b.ResetTimer() @@ -147,8 +152,8 @@ func BenchmarkExplainTableScan(b *testing.B) { se, do, st := prepareBenchSession() defer func() { se.Close() - st.Close() do.Close() + st.Close() }() prepareBenchData(se, "int", "%v", 0) b.ResetTimer() @@ -167,8 +172,8 @@ func BenchmarkTableLookup(b *testing.B) { se, do, st := prepareBenchSession() defer func() { se.Close() - st.Close() do.Close() + st.Close() }() prepareBenchData(se, "int", "%d", smallCount) b.ResetTimer() @@ -187,8 +192,8 @@ func BenchmarkExplainTableLookup(b *testing.B) { se, do, st := prepareBenchSession() defer func() { se.Close() - st.Close() do.Close() + st.Close() }() prepareBenchData(se, "int", "%d", 0) b.ResetTimer() @@ -207,8 +212,8 @@ func BenchmarkStringIndexScan(b *testing.B) { se, do, st := prepareBenchSession() defer func() { se.Close() - st.Close() do.Close() + st.Close() }() prepareBenchData(se, "varchar(255)", "'hello %d'", smallCount) b.ResetTimer() @@ -227,8 +232,8 @@ func BenchmarkExplainStringIndexScan(b *testing.B) { se, do, st := prepareBenchSession() defer func() { se.Close() - st.Close() do.Close() + st.Close() }() prepareBenchData(se, "varchar(255)", "'hello %d'", 0) b.ResetTimer() @@ -247,8 +252,8 @@ func BenchmarkStringIndexLookup(b *testing.B) { se, do, st := prepareBenchSession() defer func() { se.Close() - st.Close() do.Close() + st.Close() }() prepareBenchData(se, "varchar(255)", "'hello %d'", smallCount) b.ResetTimer() @@ -267,8 +272,8 @@ func BenchmarkIntegerIndexScan(b *testing.B) { se, do, st := prepareBenchSession() defer func() { se.Close() - st.Close() do.Close() + st.Close() }() prepareBenchData(se, "int", "%v", smallCount) b.ResetTimer() @@ -287,8 +292,8 @@ func BenchmarkIntegerIndexLookup(b *testing.B) { se, do, st := prepareBenchSession() defer func() { se.Close() - st.Close() do.Close() + st.Close() }() prepareBenchData(se, "int", "%v", smallCount) b.ResetTimer() @@ -307,8 +312,8 @@ func BenchmarkDecimalIndexScan(b *testing.B) { se, do, st := prepareBenchSession() defer func() { se.Close() - st.Close() do.Close() + st.Close() }() prepareBenchData(se, "decimal(32,6)", "%v.1234", smallCount) b.ResetTimer() @@ -327,8 +332,8 @@ func BenchmarkDecimalIndexLookup(b *testing.B) { se, do, st := prepareBenchSession() defer func() { se.Close() - st.Close() do.Close() + st.Close() }() prepareBenchData(se, "decimal(32,6)", "%v.1234", smallCount) b.ResetTimer() @@ -346,8 +351,8 @@ func BenchmarkInsertWithIndex(b *testing.B) { se, do, st := prepareBenchSession() defer func() { se.Close() - st.Close() do.Close() + st.Close() }() mustExecute(se, "drop table if exists t") mustExecute(se, "create table t (pk int primary key, col int, index idx (col))") @@ -362,8 +367,8 @@ func BenchmarkInsertNoIndex(b *testing.B) { se, do, st := prepareBenchSession() defer func() { se.Close() - st.Close() do.Close() + st.Close() }() mustExecute(se, "drop table if exists t") mustExecute(se, "create table t (pk int primary key, col int)") @@ -379,8 +384,8 @@ func BenchmarkSort(b *testing.B) { se, do, st := prepareBenchSession() defer func() { se.Close() - st.Close() do.Close() + st.Close() }() prepareSortBenchData(se, "int", "%v", bigCount) b.ResetTimer() @@ -399,8 +404,8 @@ func BenchmarkJoin(b *testing.B) { se, do, st := prepareBenchSession() defer func() { se.Close() - st.Close() do.Close() + st.Close() }() prepareJoinBenchData(se, "int", "%v", smallCount) b.ResetTimer() @@ -419,8 +424,8 @@ func BenchmarkJoinLimit(b *testing.B) { se, do, st := prepareBenchSession() defer func() { se.Close() - st.Close() do.Close() + st.Close() }() prepareJoinBenchData(se, "int", "%v", smallCount) b.ResetTimer() From 0e1420db12d419352b41a19085b974e8f4936dd4 Mon Sep 17 00:00:00 2001 From: tison Date: Wed, 30 Jun 2021 09:35:25 +0800 Subject: [PATCH 03/16] .github: remove CODEOWNERS (#25784) --- .github/CODEOWNERS | 23 ----------------------- 1 file changed, 23 deletions(-) delete mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS deleted file mode 100644 index 23d773eca2680..0000000000000 --- a/.github/CODEOWNERS +++ /dev/null @@ -1,23 +0,0 @@ -/distsql @pingcap/exec-reviewers -/executor @pingcap/exec-reviewers -/expression @pingcap/exec-reviewers -/types @pingcap/exec-reviewers -/util/chunk @pingcap/exec-reviewers -/util/disk @pingcap/exec-reviewers -/util/execdetails @pingcap/exec-reviewers -/util/expensivequery @pingcap/exec-reviewers -/util/filesort @pingcap/exec-reviewers -/util/memory @pingcap/exec-reviewers -/util/sqlexec @pingcap/exec-reviewers - -/planner @pingcap/planner-reviewers -/statistics @pingcap/planner-reviewers -/util/ranger @pingcap/planner-reviewers -/util/plancodec @pingcap/planner-reviewers -/bindinfo @pingcap/planner-reviewers - -/ddl @pingcap/co-ddl -/domain @pingcap/co-ddl -/infoschema @pingcap/co-ddl -/meta @pingcap/co-ddl -/owner @pingcap/co-ddl From 7f18c4a5d408eeffff1099e38562ad0ebacc1fa2 Mon Sep 17 00:00:00 2001 From: Chengqi Deng Date: Wed, 30 Jun 2021 13:55:25 +0800 Subject: [PATCH 04/16] planner: fix incorrect usage of UNION and INTO (#24913) --- planner/core/preprocess.go | 4 ++++ planner/core/preprocess_test.go | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/planner/core/preprocess.go b/planner/core/preprocess.go index a07757b7a91f8..b9e2b421c2cfe 100644 --- a/planner/core/preprocess.go +++ b/planner/core/preprocess.go @@ -632,6 +632,10 @@ func (p *preprocessor) checkSetOprSelectList(stmt *ast.SetOprSelectList) { for _, sel := range stmt.Selects[:len(stmt.Selects)-1] { switch s := sel.(type) { case *ast.SelectStmt: + if s.SelectIntoOpt != nil { + p.err = ErrWrongUsage.GenWithStackByArgs("UNION", "INTO") + return + } if s.IsInBraces { continue } diff --git a/planner/core/preprocess_test.go b/planner/core/preprocess_test.go index e32148c572728..91ee8afe52b9d 100644 --- a/planner/core/preprocess_test.go +++ b/planner/core/preprocess_test.go @@ -236,6 +236,10 @@ func (s *testValidatorSuite) TestValidator(c *C) { {"CREATE TABLE t IGNORE SELECT * FROM u UNION SELECT * from v", false, errors.New("'CREATE TABLE ... SELECT' is not implemented yet")}, {"CREATE TABLE t (m int) REPLACE AS (SELECT * FROM u) UNION (SELECT * FROM v)", false, errors.New("'CREATE TABLE ... SELECT' is not implemented yet")}, + // issue 24309 + {"SELECT * FROM t INTO OUTFILE 'ttt' UNION SELECT * FROM u", false, core.ErrWrongUsage.GenWithStackByArgs("UNION", "INTO")}, + {"(SELECT * FROM t INTO OUTFILE 'ttt') UNION SELECT * FROM u", false, core.ErrWrongUsage.GenWithStackByArgs("UNION", "INTO")}, + {"select * from ( select 1 ) a, (select 2) a;", false, core.ErrNonUniqTable}, {"select * from ( select 1 ) a, (select 2) b, (select 3) a;", false, core.ErrNonUniqTable}, {"select * from ( select 1 ) a, (select 2) b, (select 3) A;", false, core.ErrNonUniqTable}, From 6063386a9d164399924ef046de76e8fa4b3dd91d Mon Sep 17 00:00:00 2001 From: baishen Date: Wed, 30 Jun 2021 01:03:25 -0500 Subject: [PATCH 05/16] expressions: Support `bin-to-uuid` and `uuid-to-bin` (#20140) --- executor/show_test.go | 2 +- expression/bench_test.go | 22 +++ expression/builtin.go | 2 + expression/builtin_miscellaneous.go | 157 +++++++++++++++++ expression/builtin_miscellaneous_test.go | 168 +++++++++++++++++++ expression/builtin_miscellaneous_vec.go | 114 +++++++++++++ expression/builtin_miscellaneous_vec_test.go | 8 + expression/errors.go | 1 + 8 files changed, 473 insertions(+), 1 deletion(-) diff --git a/executor/show_test.go b/executor/show_test.go index 95074baaa44c0..9a18fdfe348b8 100644 --- a/executor/show_test.go +++ b/executor/show_test.go @@ -1130,7 +1130,7 @@ func (s *testSuite5) TestShowBuiltin(c *C) { res := tk.MustQuery("show builtins;") c.Assert(res, NotNil) rows := res.Rows() - const builtinFuncNum = 269 + const builtinFuncNum = 271 c.Assert(builtinFuncNum, Equals, len(rows)) c.Assert("abs", Equals, rows[0][0].(string)) c.Assert("yearweek", Equals, rows[builtinFuncNum-1][0].(string)) diff --git a/expression/bench_test.go b/expression/bench_test.go index 51d001a8b07d8..1d926fd6cb1e6 100644 --- a/expression/bench_test.go +++ b/expression/bench_test.go @@ -25,6 +25,7 @@ import ( "testing" "time" + "github.com/google/uuid" . "github.com/pingcap/check" "github.com/pingcap/parser/ast" "github.com/pingcap/parser/auth" @@ -634,6 +635,27 @@ func (g *ipv4MappedByteGener) gen() interface{} { return string(ip[:net.IPv6len]) } +// uuidStrGener is used to generate uuid strings. +type uuidStrGener struct { + randGen *defaultRandGen +} + +func (g *uuidStrGener) gen() interface{} { + u, _ := uuid.NewUUID() + return u.String() +} + +// uuidBinGener is used to generate uuid binarys. +type uuidBinGener struct { + randGen *defaultRandGen +} + +func (g *uuidBinGener) gen() interface{} { + u, _ := uuid.NewUUID() + bin, _ := u.MarshalBinary() + return string(bin) +} + // randLenStrGener is used to generate strings whose lengths are in [lenBegin, lenEnd). type randLenStrGener struct { lenBegin int diff --git a/expression/builtin.go b/expression/builtin.go index 8da5528b6a975..275b82d28c797 100644 --- a/expression/builtin.go +++ b/expression/builtin.go @@ -792,6 +792,8 @@ var funcs = map[string]functionClass{ ast.UUID: &uuidFunctionClass{baseFunctionClass{ast.UUID, 0, 0}}, ast.UUIDShort: &uuidShortFunctionClass{baseFunctionClass{ast.UUIDShort, 0, 0}}, ast.VitessHash: &vitessHashFunctionClass{baseFunctionClass{ast.VitessHash, 1, 1}}, + ast.UUIDToBin: &uuidToBinFunctionClass{baseFunctionClass{ast.UUIDToBin, 1, 2}}, + ast.BinToUUID: &binToUUIDFunctionClass{baseFunctionClass{ast.BinToUUID, 1, 2}}, // get_lock() and release_lock() are parsed but do nothing. // It is used for preventing error in Ruby's activerecord migrations. diff --git a/expression/builtin_miscellaneous.go b/expression/builtin_miscellaneous.go index 6d4f95a59cea2..a354d18a666ab 100644 --- a/expression/builtin_miscellaneous.go +++ b/expression/builtin_miscellaneous.go @@ -53,6 +53,8 @@ var ( _ functionClass = &uuidFunctionClass{} _ functionClass = &uuidShortFunctionClass{} _ functionClass = &vitessHashFunctionClass{} + _ functionClass = &uuidToBinFunctionClass{} + _ functionClass = &binToUUIDFunctionClass{} ) var ( @@ -76,6 +78,8 @@ var ( _ builtinFunc = &builtinIsIPv6Sig{} _ builtinFunc = &builtinUUIDSig{} _ builtinFunc = &builtinVitessHashSig{} + _ builtinFunc = &builtinUUIDToBinSig{} + _ builtinFunc = &builtinBinToUUIDSig{} _ builtinFunc = &builtinNameConstIntSig{} _ builtinFunc = &builtinNameConstRealSig{} @@ -1094,3 +1098,156 @@ func (b *builtinVitessHashSig) evalInt(row chunk.Row) (int64, bool, error) { } return int64(hashed), false, nil } + +type uuidToBinFunctionClass struct { + baseFunctionClass +} + +func (c *uuidToBinFunctionClass) getFunction(ctx sessionctx.Context, args []Expression) (builtinFunc, error) { + if err := c.verifyArgs(args); err != nil { + return nil, err + } + argTps := []types.EvalType{types.ETString} + if len(args) == 2 { + argTps = append(argTps, types.ETInt) + } + bf, err := newBaseBuiltinFuncWithTp(ctx, c.funcName, args, types.ETString, argTps...) + if err != nil { + return nil, err + } + + bf.tp.Flen = 16 + types.SetBinChsClnFlag(bf.tp) + bf.tp.Decimal = 0 + sig := &builtinUUIDToBinSig{bf} + return sig, nil +} + +type builtinUUIDToBinSig struct { + baseBuiltinFunc +} + +func (b *builtinUUIDToBinSig) Clone() builtinFunc { + newSig := &builtinUUIDToBinSig{} + newSig.cloneFrom(&b.baseBuiltinFunc) + return newSig +} + +// evalString evals UUID_TO_BIN(string_uuid, swap_flag). +// See https://dev.mysql.com/doc/refman/8.0/en/miscellaneous-functions.html#function_uuid-to-bin +func (b *builtinUUIDToBinSig) evalString(row chunk.Row) (string, bool, error) { + val, isNull, err := b.args[0].EvalString(b.ctx, row) + if isNull || err != nil { + return "", isNull, err + } + + u, err := uuid.Parse(val) + if err != nil { + return "", false, errWrongValueForType.GenWithStackByArgs("string", val, "uuid_to_bin") + } + bin, err := u.MarshalBinary() + if err != nil { + return "", false, errWrongValueForType.GenWithStackByArgs("string", val, "uuid_to_bin") + } + + flag := int64(0) + if len(b.args) == 2 { + flag, isNull, err = b.args[1].EvalInt(b.ctx, row) + if isNull { + flag = 0 + } + if err != nil { + return "", false, err + } + } + if flag != 0 { + return swapBinaryUUID(bin), false, nil + } + return string(bin), false, nil +} + +type binToUUIDFunctionClass struct { + baseFunctionClass +} + +func (c *binToUUIDFunctionClass) getFunction(ctx sessionctx.Context, args []Expression) (builtinFunc, error) { + if err := c.verifyArgs(args); err != nil { + return nil, err + } + argTps := []types.EvalType{types.ETString} + if len(args) == 2 { + argTps = append(argTps, types.ETInt) + } + bf, err := newBaseBuiltinFuncWithTp(ctx, c.funcName, args, types.ETString, argTps...) + if err != nil { + return nil, err + } + + bf.tp.Charset, bf.tp.Collate = ctx.GetSessionVars().GetCharsetInfo() + bf.tp.Flen = 32 + bf.tp.Decimal = 0 + sig := &builtinBinToUUIDSig{bf} + return sig, nil +} + +type builtinBinToUUIDSig struct { + baseBuiltinFunc +} + +func (b *builtinBinToUUIDSig) Clone() builtinFunc { + newSig := &builtinBinToUUIDSig{} + newSig.cloneFrom(&b.baseBuiltinFunc) + return newSig +} + +// evalString evals BIN_TO_UUID(binary_uuid, swap_flag). +// See https://dev.mysql.com/doc/refman/8.0/en/miscellaneous-functions.html#function_bin-to-uuid +func (b *builtinBinToUUIDSig) evalString(row chunk.Row) (string, bool, error) { + val, isNull, err := b.args[0].EvalString(b.ctx, row) + if isNull || err != nil { + return "", isNull, err + } + + var u uuid.UUID + err = u.UnmarshalBinary([]byte(val)) + if err != nil { + return "", false, errWrongValueForType.GenWithStackByArgs("string", val, "bin_to_uuid") + } + + str := u.String() + flag := int64(0) + if len(b.args) == 2 { + flag, isNull, err = b.args[1].EvalInt(b.ctx, row) + if isNull { + flag = 0 + } + if err != nil { + return "", false, err + } + } + if flag != 0 { + return swapStringUUID(str), false, nil + } + return str, false, nil +} + +func swapBinaryUUID(bin []byte) string { + buf := make([]byte, len(bin)) + copy(buf[0:2], bin[6:8]) + copy(buf[2:4], bin[4:6]) + copy(buf[4:8], bin[0:4]) + copy(buf[8:], bin[8:]) + return string(buf) +} + +func swapStringUUID(str string) string { + buf := make([]byte, len(str)) + copy(buf[0:4], str[9:13]) + copy(buf[4:8], str[14:18]) + copy(buf[8:9], str[8:9]) + copy(buf[9:13], str[4:8]) + copy(buf[13:14], str[13:14]) + copy(buf[14:18], str[0:4]) + copy(buf[18:], str[18:]) + return string(buf) +} diff --git a/expression/builtin_miscellaneous_test.go b/expression/builtin_miscellaneous_test.go index 0a23283c1dbaa..6fc25f6d563b8 100644 --- a/expression/builtin_miscellaneous_test.go +++ b/expression/builtin_miscellaneous_test.go @@ -358,3 +358,171 @@ func (s *testEvaluatorSuite) TestNameConst(c *C) { t.asserts(d) } } + +func (s *testEvaluatorSuite) TestUUIDToBin(c *C) { + tests := []struct { + args []interface{} + expect interface{} + isNil bool + getWarning bool + getError bool + }{ + { + []interface{}{"6ccd780c-baba-1026-9564-5b8c656024db"}, + []byte{0x6C, 0xCD, 0x78, 0x0C, 0xBA, 0xBA, 0x10, 0x26, 0x95, 0x64, 0x5B, 0x8C, 0x65, 0x60, 0x24, 0xDB}, + false, + false, + false, + }, + { + []interface{}{"6CCD780C-BABA-1026-9564-5B8C656024DB"}, + []byte{0x6C, 0xCD, 0x78, 0x0C, 0xBA, 0xBA, 0x10, 0x26, 0x95, 0x64, 0x5B, 0x8C, 0x65, 0x60, 0x24, 0xDB}, + false, + false, + false, + }, + { + []interface{}{"6ccd780cbaba102695645b8c656024db"}, + []byte{0x6C, 0xCD, 0x78, 0x0C, 0xBA, 0xBA, 0x10, 0x26, 0x95, 0x64, 0x5B, 0x8C, 0x65, 0x60, 0x24, 0xDB}, + false, + false, + false, + }, + { + []interface{}{"{6ccd780c-baba-1026-9564-5b8c656024db}"}, + []byte{0x6C, 0xCD, 0x78, 0x0C, 0xBA, 0xBA, 0x10, 0x26, 0x95, 0x64, 0x5B, 0x8C, 0x65, 0x60, 0x24, 0xDB}, + false, + false, + false, + }, + { + []interface{}{"6ccd780c-baba-1026-9564-5b8c656024db", 0}, + []byte{0x6C, 0xCD, 0x78, 0x0C, 0xBA, 0xBA, 0x10, 0x26, 0x95, 0x64, 0x5B, 0x8C, 0x65, 0x60, 0x24, 0xDB}, + false, + false, + false, + }, + { + []interface{}{"6ccd780c-baba-1026-9564-5b8c656024db", 1}, + []byte{0x10, 0x26, 0xBA, 0xBA, 0x6C, 0xCD, 0x78, 0x0C, 0x95, 0x64, 0x5B, 0x8C, 0x65, 0x60, 0x24, 0xDB}, + false, + false, + false, + }, + { + []interface{}{"6ccd780c-baba-1026-9564-5b8c656024db", "a"}, + []byte{0x6C, 0xCD, 0x78, 0x0C, 0xBA, 0xBA, 0x10, 0x26, 0x95, 0x64, 0x5B, 0x8C, 0x65, 0x60, 0x24, 0xDB}, + false, + true, + false, + }, + { + []interface{}{"6ccd780c-baba-1026-9564-5b8c6560"}, + []byte{}, + false, + false, + true, + }, + { + []interface{}{nil}, + []byte{}, + true, + false, + false, + }, + } + + for _, test := range tests { + preWarningCnt := s.ctx.GetSessionVars().StmtCtx.WarningCount() + f, err := newFunctionForTest(s.ctx, ast.UUIDToBin, s.primitiveValsToConstants(test.args)...) + c.Assert(err, IsNil) + + result, err := f.Eval(chunk.Row{}) + if test.getError { + c.Assert(err, NotNil) + } else if test.getWarning { + c.Assert(err, IsNil) + c.Assert(s.ctx.GetSessionVars().StmtCtx.WarningCount(), Equals, preWarningCnt+1) + } else { + c.Assert(err, IsNil) + if test.isNil { + c.Assert(result.Kind(), Equals, types.KindNull) + } else { + c.Assert(result, testutil.DatumEquals, types.NewDatum(test.expect)) + } + } + } + + _, err := funcs[ast.UUIDToBin].getFunction(s.ctx, []Expression{NewZero()}) + c.Assert(err, IsNil) +} + +func (s *testEvaluatorSuite) TestBinToUUID(c *C) { + tests := []struct { + args []interface{} + expect string + isNil bool + getWarning bool + getError bool + }{ + { + []interface{}{[]byte{0x6C, 0xCD, 0x78, 0x0C, 0xBA, 0xBA, 0x10, 0x26, 0x95, 0x64, 0x5B, 0x8C, 0x65, 0x60, 0x24, 0xDB}}, + "6ccd780c-baba-1026-9564-5b8c656024db", + false, + false, + false, + }, + { + []interface{}{[]byte{0x6C, 0xCD, 0x78, 0x0C, 0xBA, 0xBA, 0x10, 0x26, 0x95, 0x64, 0x5B, 0x8C, 0x65, 0x60, 0x24, 0xDB}, 1}, + "baba1026-780c-6ccd-9564-5b8c656024db", + false, + false, + false, + }, + { + []interface{}{[]byte{0x6C, 0xCD, 0x78, 0x0C, 0xBA, 0xBA, 0x10, 0x26, 0x95, 0x64, 0x5B, 0x8C, 0x65, 0x60, 0x24, 0xDB}, "a"}, + "6ccd780c-baba-1026-9564-5b8c656024db", + false, + true, + false, + }, + { + []interface{}{[]byte{0x6C, 0xCD, 0x78, 0x0C, 0xBA, 0xBA, 0x10, 0x26, 0x95, 0x64, 0x5B, 0x8C, 0x65, 0x60}}, + "", + false, + false, + true, + }, + { + []interface{}{nil}, + "", + true, + false, + false, + }, + } + + for _, test := range tests { + preWarningCnt := s.ctx.GetSessionVars().StmtCtx.WarningCount() + f, err := newFunctionForTest(s.ctx, ast.BinToUUID, s.primitiveValsToConstants(test.args)...) + c.Assert(err, IsNil) + + result, err := f.Eval(chunk.Row{}) + if test.getError { + c.Assert(err, NotNil) + } else if test.getWarning { + c.Assert(err, IsNil) + c.Assert(s.ctx.GetSessionVars().StmtCtx.WarningCount(), Equals, preWarningCnt+1) + } else { + c.Assert(err, IsNil) + if test.isNil { + c.Assert(result.Kind(), Equals, types.KindNull) + } else { + c.Assert(result.GetString(), Equals, test.expect) + } + } + } + + _, err := funcs[ast.BinToUUID].getFunction(s.ctx, []Expression{NewZero()}) + c.Assert(err, IsNil) +} diff --git a/expression/builtin_miscellaneous_vec.go b/expression/builtin_miscellaneous_vec.go index 8e0689be3aab8..e087bdbb50bf3 100644 --- a/expression/builtin_miscellaneous_vec.go +++ b/expression/builtin_miscellaneous_vec.go @@ -653,3 +653,117 @@ func (b *builtinVitessHashSig) vecEvalInt(input *chunk.Chunk, result *chunk.Colu return nil } + +func (b *builtinUUIDToBinSig) vectorized() bool { + return true +} + +// evalString evals UUID_TO_BIN(string_uuid, swap_flag). +// See https://dev.mysql.com/doc/refman/8.0/en/miscellaneous-functions.html#function_uuid-to-bin +func (b *builtinUUIDToBinSig) vecEvalString(input *chunk.Chunk, result *chunk.Column) error { + n := input.NumRows() + valBuf, err := b.bufAllocator.get(types.ETString, n) + if err != nil { + return err + } + defer b.bufAllocator.put(valBuf) + if err := b.args[0].VecEvalString(b.ctx, input, valBuf); err != nil { + return err + } + + var flagBuf *chunk.Column + i64s := make([]int64, n) + if len(b.args) == 2 { + flagBuf, err = b.bufAllocator.get(types.ETInt, n) + if err != nil { + return err + } + defer b.bufAllocator.put(flagBuf) + if err := b.args[1].VecEvalInt(b.ctx, input, flagBuf); err != nil { + return err + } + i64s = flagBuf.Int64s() + } + result.ReserveString(n) + for i := 0; i < n; i++ { + if valBuf.IsNull(i) { + result.AppendNull() + continue + } + val := valBuf.GetString(i) + u, err := uuid.Parse(val) + if err != nil { + return errWrongValueForType.GenWithStackByArgs("string", val, "uuid_to_bin") + } + bin, err := u.MarshalBinary() + if err != nil { + return errWrongValueForType.GenWithStackByArgs("string", val, "uuid_to_bin") + } + if len(b.args) == 2 && flagBuf.IsNull(i) { + result.AppendString(string(bin)) + continue + } + if i64s[i] != 0 { + result.AppendString(swapBinaryUUID(bin)) + } else { + result.AppendString(string(bin)) + } + } + return nil +} + +func (b *builtinBinToUUIDSig) vectorized() bool { + return true +} + +// evalString evals BIN_TO_UUID(binary_uuid, swap_flag). +// See https://dev.mysql.com/doc/refman/8.0/en/miscellaneous-functions.html#function_bin-to-uuid +func (b *builtinBinToUUIDSig) vecEvalString(input *chunk.Chunk, result *chunk.Column) error { + n := input.NumRows() + valBuf, err := b.bufAllocator.get(types.ETString, n) + if err != nil { + return err + } + defer b.bufAllocator.put(valBuf) + if err := b.args[0].VecEvalString(b.ctx, input, valBuf); err != nil { + return err + } + + var flagBuf *chunk.Column + i64s := make([]int64, n) + if len(b.args) == 2 { + flagBuf, err = b.bufAllocator.get(types.ETInt, n) + if err != nil { + return err + } + defer b.bufAllocator.put(flagBuf) + if err := b.args[1].VecEvalInt(b.ctx, input, flagBuf); err != nil { + return err + } + i64s = flagBuf.Int64s() + } + result.ReserveString(n) + for i := 0; i < n; i++ { + if valBuf.IsNull(i) { + result.AppendNull() + continue + } + val := valBuf.GetString(i) + var u uuid.UUID + err = u.UnmarshalBinary([]byte(val)) + if err != nil { + return errWrongValueForType.GenWithStackByArgs("string", val, "bin_to_uuid") + } + str := u.String() + if len(b.args) == 2 && flagBuf.IsNull(i) { + result.AppendString(str) + continue + } + if i64s[i] != 0 { + result.AppendString(swapStringUUID(str)) + } else { + result.AppendString(str) + } + } + return nil +} diff --git a/expression/builtin_miscellaneous_vec_test.go b/expression/builtin_miscellaneous_vec_test.go index 85a329b8f41b8..4644811458681 100644 --- a/expression/builtin_miscellaneous_vec_test.go +++ b/expression/builtin_miscellaneous_vec_test.go @@ -99,6 +99,14 @@ var vecBuiltinMiscellaneousCases = map[string][]vecExprBenchCase{ {retEvalType: types.ETJson, childrenTypes: []types.EvalType{types.ETString, types.ETJson}}, {retEvalType: types.ETTimestamp, childrenTypes: []types.EvalType{types.ETString, types.ETTimestamp}}, }, + ast.UUIDToBin: { + {retEvalType: types.ETString, childrenTypes: []types.EvalType{types.ETString}, geners: []dataGenerator{&uuidStrGener{newDefaultRandGen()}}}, + {retEvalType: types.ETString, childrenTypes: []types.EvalType{types.ETString, types.ETInt}, geners: []dataGenerator{&uuidStrGener{newDefaultRandGen()}}}, + }, + ast.BinToUUID: { + {retEvalType: types.ETString, childrenTypes: []types.EvalType{types.ETString}, geners: []dataGenerator{&uuidBinGener{newDefaultRandGen()}}}, + {retEvalType: types.ETString, childrenTypes: []types.EvalType{types.ETString, types.ETInt}, geners: []dataGenerator{&uuidBinGener{newDefaultRandGen()}}}, + }, } func (s *testEvaluatorSuite) TestVectorizedBuiltinMiscellaneousEvalOneVec(c *C) { diff --git a/expression/errors.go b/expression/errors.go index 9efd5b8cc518b..ad0f49e64653c 100644 --- a/expression/errors.go +++ b/expression/errors.go @@ -47,6 +47,7 @@ var ( errTruncatedWrongValue = dbterror.ClassExpression.NewStd(mysql.ErrTruncatedWrongValue) errUnknownLocale = dbterror.ClassExpression.NewStd(mysql.ErrUnknownLocale) errNonUniq = dbterror.ClassExpression.NewStd(mysql.ErrNonUniq) + errWrongValueForType = dbterror.ClassExpression.NewStd(mysql.ErrWrongValueForType) // Sequence usage privilege check. errSequenceAccessDenied = dbterror.ClassExpression.NewStd(mysql.ErrTableaccessDenied) From 1249a118534ef18f6f7c57117f74ea563e4ec6aa Mon Sep 17 00:00:00 2001 From: disksing Date: Wed, 30 Jun 2021 14:17:25 +0800 Subject: [PATCH 06/16] *: update client-go and cleanup retry dependency (#25841) --- go.mod | 2 +- go.sum | 4 ++-- store/copr/batch_request_sender.go | 3 +-- store/gcworker/gc_worker_test.go | 3 +-- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 4ea0fc22e4617..9daec0e6d9064 100644 --- a/go.mod +++ b/go.mod @@ -55,7 +55,7 @@ require ( github.com/shirou/gopsutil v3.21.2+incompatible github.com/soheilhy/cmux v0.1.4 github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 - github.com/tikv/client-go/v2 v2.0.0-alpha.0.20210628064110-666340265aac + github.com/tikv/client-go/v2 v2.0.0-alpha.0.20210630040115-58b6783d1b56 github.com/tikv/pd v1.1.0-beta.0.20210323121136-78679e5e209d github.com/twmb/murmur3 v1.1.3 github.com/uber-go/atomic v1.4.0 diff --git a/go.sum b/go.sum index 66bf5f1f3a557..f8823b5413c4b 100644 --- a/go.sum +++ b/go.sum @@ -558,8 +558,8 @@ github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2/go.mod h1:2PfK github.com/tidwall/gjson v1.3.5/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tikv/client-go/v2 v2.0.0-alpha.0.20210628064110-666340265aac h1:iqrha2f7pXKhgoa3+Lriat13RM2dOtJ6430XpVJ8izk= -github.com/tikv/client-go/v2 v2.0.0-alpha.0.20210628064110-666340265aac/go.mod h1:crzTwbliZf57xC5ZSzmQx4iMZCLCGhA364to+E2JAPU= +github.com/tikv/client-go/v2 v2.0.0-alpha.0.20210630040115-58b6783d1b56 h1:R1jC5I6wJmmfrYGNxAPR+ABbo1T2KMWcEWPr+UN5Bec= +github.com/tikv/client-go/v2 v2.0.0-alpha.0.20210630040115-58b6783d1b56/go.mod h1:crzTwbliZf57xC5ZSzmQx4iMZCLCGhA364to+E2JAPU= github.com/tikv/pd v1.1.0-beta.0.20210323121136-78679e5e209d h1:K0XnvsnT6ofLDuM8Rt3PuFQO4p8bNraeHYstspD316g= github.com/tikv/pd v1.1.0-beta.0.20210323121136-78679e5e209d/go.mod h1:Jw9KG11C/23Rr7DW4XWQ7H5xOgGZo6DFL1OKAF4+Igw= github.com/tklauser/go-sysconf v0.3.4 h1:HT8SVixZd3IzLdfs/xlpq0jeSfTX57g1v6wB1EuzV7M= diff --git a/store/copr/batch_request_sender.go b/store/copr/batch_request_sender.go index 69a02ac22e0a2..4d2a9cda36909 100644 --- a/store/copr/batch_request_sender.go +++ b/store/copr/batch_request_sender.go @@ -20,7 +20,6 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/metapb" tikverr "github.com/tikv/client-go/v2/error" - "github.com/tikv/client-go/v2/retry" "github.com/tikv/client-go/v2/tikv" "github.com/tikv/client-go/v2/tikvrpc" "google.golang.org/grpc/codes" @@ -95,6 +94,6 @@ func (ss *RegionBatchRequestSender) onSendFailForBatchRegions(bo *Backoffer, ctx // When a store is not available, the leader of related region should be elected quickly. // TODO: the number of retry time should be limited:since region may be unavailable // when some unrecoverable disaster happened. - err = bo.Backoff(retry.BoTiFlashRPC, errors.Errorf("send request error: %v, ctx: %v, regionInfos: %v", err, ctx, regionInfos)) + err = bo.Backoff(tikv.BoTiFlashRPC(), errors.Errorf("send request error: %v, ctx: %v, regionInfos: %v", err, ctx, regionInfos)) return errors.Trace(err) } diff --git a/store/gcworker/gc_worker_test.go b/store/gcworker/gc_worker_test.go index 22ac9344e2fc6..5cb17c0a57f37 100644 --- a/store/gcworker/gc_worker_test.go +++ b/store/gcworker/gc_worker_test.go @@ -42,7 +42,6 @@ import ( "github.com/tikv/client-go/v2/mockstore/mocktikv" "github.com/tikv/client-go/v2/oracle" "github.com/tikv/client-go/v2/oracle/oracles" - "github.com/tikv/client-go/v2/retry" "github.com/tikv/client-go/v2/tikv" "github.com/tikv/client-go/v2/tikvrpc" pd "github.com/tikv/pd/client" @@ -945,7 +944,7 @@ func (s *testGCWorkerSuite) TestResolveLockRangeMeetRegionEnlargeCausedByRegionM mCluster.Merge(s.initRegion.regionID, region2) regionMeta, _ := mCluster.GetRegion(s.initRegion.regionID) _, err := s.tikvStore.GetRegionCache().OnRegionEpochNotMatch( - retry.NewNoopBackoff(context.Background()), + tikv.NewNoopBackoff(context.Background()), &tikv.RPCContext{Region: regionID, Store: &tikv.Store{}}, []*metapb.Region{regionMeta}) c.Assert(err, IsNil) From 3ad894da97d99d6ec4286afeab6f8f30d628be1c Mon Sep 17 00:00:00 2001 From: Han Fei Date: Wed, 30 Jun 2021 14:55:26 +0800 Subject: [PATCH 07/16] planner/core: thoroughly push down count-distinct agg in the MPP mode. (#25662) --- planner/core/exhaust_physical_plans.go | 6 ++++- planner/core/fragment.go | 7 ++++++ planner/core/physical_plans.go | 2 ++ planner/core/task.go | 23 +++++++++++++++-- .../testdata/integration_serial_suite_in.json | 4 ++- .../integration_serial_suite_out.json | 25 +++++++++++++++++++ 6 files changed, 63 insertions(+), 4 deletions(-) diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index b4d3198fba7b1..fb06f570fdce7 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -2451,7 +2451,11 @@ func (la *LogicalAggregation) tryToGetMppHashAggs(prop *property.PhysicalPropert childProp := &property.PhysicalProperty{TaskTp: property.MppTaskType, ExpectedCnt: math.MaxFloat64} agg := NewPhysicalHashAgg(la, la.stats.ScaleByExpectCnt(prop.ExpectedCnt), childProp) agg.SetSchema(la.schema.Clone()) - agg.MppRunMode = MppTiDB + if la.HasDistinct() { + agg.MppRunMode = MppScalar + } else { + agg.MppRunMode = MppTiDB + } hashAggs = append(hashAggs, agg) } return diff --git a/planner/core/fragment.go b/planner/core/fragment.go index 7315da176e6b8..c4464c1d19f9b 100644 --- a/planner/core/fragment.go +++ b/planner/core/fragment.go @@ -26,6 +26,7 @@ import ( "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/logutil" + "github.com/pingcap/tipb/go-tipb" "go.uber.org/zap" ) @@ -40,6 +41,8 @@ type Fragment struct { ExchangeSender *PhysicalExchangeSender // data exporter IsRoot bool + + singleton bool // indicates if this is a task running on a single node. } type tasksAndFrags struct { @@ -121,6 +124,7 @@ func (f *Fragment) init(p PhysicalPlan) error { } f.TableScan = x case *PhysicalExchangeReceiver: + f.singleton = x.children[0].(*PhysicalExchangeSender).ExchangeType == tipb.ExchangeType_PassThrough f.ExchangeReceivers = append(f.ExchangeReceivers, x) case *PhysicalUnionAll: return errors.New("unexpected union all detected") @@ -246,6 +250,9 @@ func (e *mppTaskGenerator) generateMPPTasksForFragment(f *Fragment) (tasks []*kv for _, r := range f.ExchangeReceivers { childrenTasks = append(childrenTasks, r.Tasks...) } + if f.singleton { + childrenTasks = childrenTasks[0:1] + } tasks = e.constructMPPTasksByChildrenTasks(childrenTasks) } if err != nil { diff --git a/planner/core/physical_plans.go b/planner/core/physical_plans.go index 4e5c6cbfd58e5..e58aa1f760823 100644 --- a/planner/core/physical_plans.go +++ b/planner/core/physical_plans.go @@ -1008,6 +1008,8 @@ const ( Mpp2Phase // MppTiDB runs agg on TiDB (and a partial agg on TiFlash if in 2 phase agg) MppTiDB + // MppScalar also has 2 phases. The second phase runs in a single task. + MppScalar ) type basePhysicalAgg struct { diff --git a/planner/core/task.go b/planner/core/task.go index d346f3d8a46aa..bb5bad84d4cb7 100644 --- a/planner/core/task.go +++ b/planner/core/task.go @@ -1939,9 +1939,9 @@ func (p *PhysicalHashAgg) attach2TaskForMpp(tasks ...task) task { } // TODO: how to set 2-phase cost? newMpp.addCost(p.GetCost(inputRows, false, true)) - finalAgg.SetCost(mpp.cost()) + finalAgg.SetCost(newMpp.cost()) if proj != nil { - proj.SetCost(mpp.cost()) + proj.SetCost(newMpp.cost()) } return newMpp case MppTiDB: @@ -1959,6 +1959,25 @@ func (p *PhysicalHashAgg) attach2TaskForMpp(tasks ...task) task { t.addCost(p.GetCost(inputRows, true, false)) finalAgg.SetCost(t.cost()) return t + case MppScalar: + proj := p.convertAvgForMPP() + partialAgg, finalAgg := p.newPartialAggregate(kv.TiFlash, true) + if partialAgg == nil || finalAgg == nil { + return invalidTask + } + attachPlan2Task(partialAgg, mpp) + prop := &property.PhysicalProperty{TaskTp: property.MppTaskType, ExpectedCnt: math.MaxFloat64, MPPPartitionTp: property.AnyType} + newMpp := mpp.enforceExchangerImpl(prop) + attachPlan2Task(finalAgg, newMpp) + if proj != nil { + attachPlan2Task(proj, newMpp) + } + newMpp.addCost(p.GetCost(inputRows, false, true)) + finalAgg.SetCost(newMpp.cost()) + if proj != nil { + proj.SetCost(newMpp.cost()) + } + return newMpp default: return invalidTask } diff --git a/planner/core/testdata/integration_serial_suite_in.json b/planner/core/testdata/integration_serial_suite_in.json index 20384808c7d28..a6e109e36b797 100644 --- a/planner/core/testdata/integration_serial_suite_in.json +++ b/planner/core/testdata/integration_serial_suite_in.json @@ -277,7 +277,9 @@ "desc format = 'brief' select * from t join ( select count(distinct value), id from t group by id) as A on A.id = t.id", "desc format = 'brief' select * from t join ( select count(1/value), id from t group by id) as A on A.id = t.id", "desc format = 'brief' select /*+hash_agg()*/ sum(id) from (select value, id from t where id > value group by id, value)A group by value /*the exchange should have only one partition column: test.t.value*/", - "desc format = 'brief' select /*+hash_agg()*/ sum(B.value) from t as B where B.id+1 > (select count(*) from t where t.id= B.id and t.value=B.value) group by B.id /*the exchange should have only one partition column: test.t.id*/" + "desc format = 'brief' select /*+hash_agg()*/ sum(B.value) from t as B where B.id+1 > (select count(*) from t where t.id= B.id and t.value=B.value) group by B.id /*the exchange should have only one partition column: test.t.id*/", + "desc format = 'brief' select count(distinct value) from t", + "desc format = 'brief' select count(distinct value), count(value), avg(value) from t" ] }, { diff --git a/planner/core/testdata/integration_serial_suite_out.json b/planner/core/testdata/integration_serial_suite_out.json index 88b13f5660bbe..6fb81b30bf590 100644 --- a/planner/core/testdata/integration_serial_suite_out.json +++ b/planner/core/testdata/integration_serial_suite_out.json @@ -2359,6 +2359,31 @@ " └─ExchangeSender 10000.00 batchCop[tiflash] ExchangeType: HashPartition, Hash Cols: test.t.id", " └─TableFullScan 10000.00 batchCop[tiflash] table:B keep order:false, stats:pseudo" ] + }, + { + "SQL": "desc format = 'brief' select count(distinct value) from t", + "Plan": [ + "TableReader 1.00 root data:ExchangeSender", + "└─ExchangeSender 1.00 batchCop[tiflash] ExchangeType: PassThrough", + " └─HashAgg 1.00 batchCop[tiflash] funcs:count(distinct test.t.value)->Column#4", + " └─ExchangeReceiver 1.00 batchCop[tiflash] ", + " └─ExchangeSender 1.00 batchCop[tiflash] ExchangeType: PassThrough", + " └─HashAgg 1.00 batchCop[tiflash] group by:test.t.value, ", + " └─TableFullScan 10000.00 batchCop[tiflash] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc format = 'brief' select count(distinct value), count(value), avg(value) from t", + "Plan": [ + "TableReader 1.00 root data:ExchangeSender", + "└─ExchangeSender 1.00 batchCop[tiflash] ExchangeType: PassThrough", + " └─Projection 1.00 batchCop[tiflash] Column#4, Column#5, div(Column#6, cast(case(eq(Column#7, 0), 1, Column#7), decimal(20,0) BINARY))->Column#6", + " └─HashAgg 1.00 batchCop[tiflash] funcs:count(distinct test.t.value)->Column#4, funcs:sum(Column#8)->Column#5, funcs:sum(Column#9)->Column#7, funcs:sum(Column#10)->Column#6", + " └─ExchangeReceiver 1.00 batchCop[tiflash] ", + " └─ExchangeSender 1.00 batchCop[tiflash] ExchangeType: PassThrough", + " └─HashAgg 1.00 batchCop[tiflash] group by:test.t.value, funcs:count(test.t.value)->Column#8, funcs:count(test.t.value)->Column#9, funcs:sum(test.t.value)->Column#10", + " └─TableFullScan 10000.00 batchCop[tiflash] table:t keep order:false, stats:pseudo" + ] } ] }, From e8af6eb0adecfb61e0806d6b73a660e4ade42733 Mon Sep 17 00:00:00 2001 From: disksing Date: Wed, 30 Jun 2021 16:43:25 +0800 Subject: [PATCH 08/16] store/driver: update unionstore import path (#25823) --- store/driver/txn/unionstore_driver.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/store/driver/txn/unionstore_driver.go b/store/driver/txn/unionstore_driver.go index e508997d68e19..d58eca5fd552d 100644 --- a/store/driver/txn/unionstore_driver.go +++ b/store/driver/txn/unionstore_driver.go @@ -19,15 +19,15 @@ import ( "github.com/pingcap/tidb/kv" derr "github.com/pingcap/tidb/store/driver/error" tikvstore "github.com/tikv/client-go/v2/kv" - "github.com/tikv/client-go/v2/unionstore" + "github.com/tikv/client-go/v2/tikv" ) -// memBuffer wraps unionstore.MemDB as kv.MemBuffer. +// memBuffer wraps tikv.MemDB as kv.MemBuffer. type memBuffer struct { - *unionstore.MemDB + *tikv.MemDB } -func newMemBuffer(m *unionstore.MemDB) kv.MemBuffer { +func newMemBuffer(m *tikv.MemDB) kv.MemBuffer { if m == nil { return nil } @@ -112,10 +112,10 @@ func (m *memBuffer) SnapshotGetter() kv.Getter { } type tikvGetter struct { - unionstore.Getter + tikv.Getter } -func newKVGetter(getter unionstore.Getter) kv.Getter { +func newKVGetter(getter tikv.Getter) kv.Getter { return &tikvGetter{Getter: getter} } @@ -124,12 +124,12 @@ func (g *tikvGetter) Get(_ context.Context, k kv.Key) ([]byte, error) { return data, derr.ToTiDBErr(err) } -// tikvIterator wraps unionstore.Iterator as kv.Iterator +// tikvIterator wraps tikv.Iterator as kv.Iterator type tikvIterator struct { - unionstore.Iterator + tikv.Iterator } -func newKVIterator(it unionstore.Iterator) kv.Iterator { +func newKVIterator(it tikv.Iterator) kv.Iterator { if it == nil { return nil } From e13fe350487e90356351abdecf287ff969cfc380 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E8=B6=85?= Date: Wed, 30 Jun 2021 17:25:25 +0800 Subject: [PATCH 09/16] *: add TestLocalTemporaryTableInfoSchema to store local temporary table schemas (#25518) --- infoschema/infoschema.go | 155 +++++++++++++++++++++ infoschema/infoschema_test.go | 251 ++++++++++++++++++++++++++++++++++ 2 files changed, 406 insertions(+) diff --git a/infoschema/infoschema.go b/infoschema/infoschema.go index 2494e89b4d57f..ebdc8d567bd9b 100644 --- a/infoschema/infoschema.go +++ b/infoschema/infoschema.go @@ -403,3 +403,158 @@ func GetBundle(h InfoSchema, ids []int64) *placement.Bundle { } return &placement.Bundle{ID: placement.GroupID(id), Rules: newRules} } + +type schemaLocalTempSchemaTables struct { + tables map[string]table.Table +} + +// LocalTemporaryTables store local temporary tables +type LocalTemporaryTables struct { + schemaMap map[string]*schemaLocalTempSchemaTables + idx2table map[int64]table.Table +} + +// NewLocalTemporaryTableInfoSchema creates a new LocalTemporaryTableInfoSchema object +func NewLocalTemporaryTableInfoSchema() *LocalTemporaryTables { + return &LocalTemporaryTables{ + schemaMap: make(map[string]*schemaLocalTempSchemaTables), + idx2table: make(map[int64]table.Table), + } +} + +// TableByName get table by name +func (is *LocalTemporaryTables) TableByName(schema, table model.CIStr) (table.Table, bool) { + if tbNames, ok := is.schemaMap[schema.L]; ok { + if t, ok := tbNames.tables[table.L]; ok { + return t, true + } + } + return nil, false +} + +// TableExists check if table with the name exists +func (is *LocalTemporaryTables) TableExists(schema, table model.CIStr) (ok bool) { + _, ok = is.TableByName(schema, table) + return +} + +// TableByID get table by table id +func (is *LocalTemporaryTables) TableByID(id int64) (tbl table.Table, ok bool) { + tbl, ok = is.idx2table[id] + return +} + +// AddTable add a table +func (is *LocalTemporaryTables) AddTable(schema *model.DBInfo, tbl table.Table) error { + schemaTables := is.ensureSchema(schema.Name) + + tblMeta := tbl.Meta() + if _, ok := schemaTables.tables[tblMeta.Name.L]; ok { + return ErrTableExists.GenWithStackByArgs(tblMeta.Name) + } + + if _, ok := is.idx2table[tblMeta.ID]; ok { + return ErrTableExists.GenWithStackByArgs(tblMeta.Name) + } + + schemaTables.tables[tblMeta.Name.L] = tbl + is.idx2table[tblMeta.ID] = tbl + + return nil +} + +// RemoveTable remove a table +func (is *LocalTemporaryTables) RemoveTable(schema, table model.CIStr) (exist bool) { + tbls := is.schemaTables(schema) + if tbls == nil { + return false + } + + oldTable, exist := tbls.tables[table.L] + if !exist { + return false + } + + delete(tbls.tables, table.L) + delete(is.idx2table, oldTable.Meta().ID) + return true +} + +// SchemaByTable get a table's schema name +func (is *LocalTemporaryTables) SchemaByTable(tableInfo *model.TableInfo) (string, bool) { + if tableInfo == nil { + return "", false + } + + for schema, v := range is.schemaMap { + if tbl, ok := v.tables[tableInfo.Name.L]; ok { + if tbl.Meta().ID == tableInfo.ID { + return schema, true + } + } + } + + return "", false +} + +func (is *LocalTemporaryTables) ensureSchema(schema model.CIStr) *schemaLocalTempSchemaTables { + if tbls, ok := is.schemaMap[schema.L]; ok { + return tbls + } + + tbls := &schemaLocalTempSchemaTables{tables: make(map[string]table.Table)} + is.schemaMap[schema.L] = tbls + return tbls +} + +func (is *LocalTemporaryTables) schemaTables(schema model.CIStr) *schemaLocalTempSchemaTables { + if is.schemaMap == nil { + return nil + } + + if tbls, ok := is.schemaMap[schema.L]; ok { + return tbls + } + + return nil +} + +// TemporaryTableAttachedInfoSchema implements InfoSchema +// Local temporary table has a loose relationship with database. +// So when a database is dropped, its temporary tables still exist and can be return by TableByName/TableByID. +// However SchemaByTable will return nil if database is dropped. +type TemporaryTableAttachedInfoSchema struct { + InfoSchema + LocalTemporaryTables *LocalTemporaryTables +} + +// TableByName implements InfoSchema.TableByName +func (ts *TemporaryTableAttachedInfoSchema) TableByName(schema, table model.CIStr) (table.Table, error) { + if tbl, ok := ts.LocalTemporaryTables.TableByName(schema, table); ok { + return tbl, nil + } + + return ts.InfoSchema.TableByName(schema, table) +} + +// TableByID implements InfoSchema.TableByID +func (ts *TemporaryTableAttachedInfoSchema) TableByID(id int64) (table.Table, bool) { + if tbl, ok := ts.LocalTemporaryTables.TableByID(id); ok { + return tbl, true + } + + return ts.InfoSchema.TableByID(id) +} + +// SchemaByTable implements InfoSchema.SchemaByTable +func (ts *TemporaryTableAttachedInfoSchema) SchemaByTable(tableInfo *model.TableInfo) (*model.DBInfo, bool) { + if tableInfo == nil { + return nil, false + } + + if schemaName, ok := ts.LocalTemporaryTables.SchemaByTable(tableInfo); ok { + return ts.SchemaByName(model.NewCIStr(schemaName)) + } + + return ts.InfoSchema.SchemaByTable(tableInfo) +} diff --git a/infoschema/infoschema_test.go b/infoschema/infoschema_test.go index 61f34032942fe..faa35b7f37cd6 100644 --- a/infoschema/infoschema_test.go +++ b/infoschema/infoschema_test.go @@ -15,8 +15,12 @@ package infoschema_test import ( "context" + "strings" "testing" + "github.com/pingcap/tidb/meta/autoid" + "github.com/pingcap/tidb/table" + . "github.com/pingcap/check" "github.com/pingcap/errors" "github.com/pingcap/parser/model" @@ -392,3 +396,250 @@ func (*testSuite) TestGetBundle(c *C) { b.ID = "test" c.Assert(bundle.ID, Equals, ptID) } + +func (*testSuite) TestLocalTemporaryTables(c *C) { + store, err := mockstore.NewMockStore() + c.Assert(err, IsNil) + + createNewSchemaInfo := func(schemaName string) *model.DBInfo { + schemaID, err := genGlobalID(store) + c.Assert(err, IsNil) + return &model.DBInfo{ + ID: schemaID, + Name: model.NewCIStr(schemaName), + State: model.StatePublic, + } + } + + createNewTable := func(schemaID int64, tbName string, tempType model.TempTableType) table.Table { + colID, err := genGlobalID(store) + c.Assert(err, IsNil) + + colInfo := &model.ColumnInfo{ + ID: colID, + Name: model.NewCIStr("col1"), + Offset: 0, + FieldType: *types.NewFieldType(mysql.TypeLonglong), + State: model.StatePublic, + } + + tbID, err := genGlobalID(store) + c.Assert(err, IsNil) + + tblInfo := &model.TableInfo{ + ID: tbID, + Name: model.NewCIStr(tbName), + Columns: []*model.ColumnInfo{colInfo}, + Indices: []*model.IndexInfo{}, + State: model.StatePublic, + } + + allocs := autoid.NewAllocatorsFromTblInfo(store, schemaID, tblInfo) + tbl, err := table.TableFromMeta(allocs, tblInfo) + c.Assert(err, IsNil) + + return tbl + } + + assertTableByName := func(sc *infoschema.LocalTemporaryTables, schemaName, tableName string, schema *model.DBInfo, tb table.Table) { + got, ok := sc.TableByName(model.NewCIStr(schemaName), model.NewCIStr(tableName)) + if tb == nil { + c.Assert(schema, IsNil) + c.Assert(ok, IsFalse) + c.Assert(got, IsNil) + } else { + c.Assert(schema, NotNil) + c.Assert(ok, IsTrue) + c.Assert(got, Equals, tb) + } + } + + assertTableExists := func(sc *infoschema.LocalTemporaryTables, schemaName, tableName string, exists bool) { + got := sc.TableExists(model.NewCIStr(schemaName), model.NewCIStr(tableName)) + c.Assert(got, Equals, exists) + } + + assertTableByID := func(sc *infoschema.LocalTemporaryTables, tbID int64, schema *model.DBInfo, tb table.Table) { + got, ok := sc.TableByID(tbID) + if tb == nil { + c.Assert(schema, IsNil) + c.Assert(ok, IsFalse) + c.Assert(got, IsNil) + } else { + c.Assert(schema, NotNil) + c.Assert(ok, IsTrue) + c.Assert(got, Equals, tb) + } + } + + assertSchemaByTable := func(sc *infoschema.LocalTemporaryTables, schema model.CIStr, tb *model.TableInfo) { + got, ok := sc.SchemaByTable(tb) + if tb == nil { + c.Assert(schema.L == "", IsTrue) + c.Assert(got, Equals, "") + c.Assert(ok, IsFalse) + } else { + c.Assert(ok, Equals, schema.L != "") + c.Assert(schema.L, Equals, got) + } + } + + sc := infoschema.NewLocalTemporaryTableInfoSchema() + db1 := createNewSchemaInfo("db1") + tb11 := createNewTable(db1.ID, "tb1", model.TempTableLocal) + tb12 := createNewTable(db1.ID, "Tb2", model.TempTableLocal) + tb13 := createNewTable(db1.ID, "tb3", model.TempTableLocal) + + // db1b has the same name with db1 + db1b := createNewSchemaInfo("db1") + tb15 := createNewTable(db1b.ID, "tb5", model.TempTableLocal) + tb16 := createNewTable(db1b.ID, "tb6", model.TempTableLocal) + tb17 := createNewTable(db1b.ID, "tb7", model.TempTableLocal) + + db2 := createNewSchemaInfo("db2") + tb21 := createNewTable(db2.ID, "tb1", model.TempTableLocal) + tb22 := createNewTable(db2.ID, "TB2", model.TempTableLocal) + tb24 := createNewTable(db2.ID, "tb4", model.TempTableLocal) + + prepareTables := []struct { + db *model.DBInfo + tb table.Table + }{ + {db1, tb11}, {db1, tb12}, {db1, tb13}, + {db1b, tb15}, {db1b, tb16}, {db1b, tb17}, + {db2, tb21}, {db2, tb22}, {db2, tb24}, + } + + for _, p := range prepareTables { + err = sc.AddTable(p.db, p.tb) + c.Assert(err, IsNil) + } + + // test exist tables + for _, p := range prepareTables { + dbName := p.db.Name + tbName := p.tb.Meta().Name + + assertTableByName(sc, dbName.O, tbName.O, p.db, p.tb) + assertTableByName(sc, dbName.L, tbName.L, p.db, p.tb) + assertTableByName( + sc, + strings.ToUpper(dbName.L[:1])+dbName.L[1:], + strings.ToUpper(tbName.L[:1])+tbName.L[1:], + p.db, p.tb, + ) + + assertTableExists(sc, dbName.O, tbName.O, true) + assertTableExists(sc, dbName.L, tbName.L, true) + assertTableExists( + sc, + strings.ToUpper(dbName.L[:1])+dbName.L[1:], + strings.ToUpper(tbName.L[:1])+tbName.L[1:], + true, + ) + + assertTableByID(sc, p.tb.Meta().ID, p.db, p.tb) + assertSchemaByTable(sc, p.db.Name, p.tb.Meta()) + } + + // test add dup table + err = sc.AddTable(db1, tb11) + c.Assert(infoschema.ErrTableExists.Equal(err), IsTrue) + err = sc.AddTable(db1b, tb15) + c.Assert(infoschema.ErrTableExists.Equal(err), IsTrue) + err = sc.AddTable(db1b, tb11) + c.Assert(infoschema.ErrTableExists.Equal(err), IsTrue) + db1c := createNewSchemaInfo("db1") + err = sc.AddTable(db1c, createNewTable(db1c.ID, "tb1", model.TempTableLocal)) + c.Assert(infoschema.ErrTableExists.Equal(err), IsTrue) + err = sc.AddTable(db1b, tb11) + c.Assert(infoschema.ErrTableExists.Equal(err), IsTrue) + + // failed add has no effect + assertTableByName(sc, db1.Name.L, tb11.Meta().Name.L, db1, tb11) + + // delete some tables + c.Assert(sc.RemoveTable(model.NewCIStr("db1"), model.NewCIStr("tb1")), IsTrue) + c.Assert(sc.RemoveTable(model.NewCIStr("Db2"), model.NewCIStr("tB2")), IsTrue) + c.Assert(sc.RemoveTable(model.NewCIStr("db1"), model.NewCIStr("tbx")), IsFalse) + c.Assert(sc.RemoveTable(model.NewCIStr("dbx"), model.NewCIStr("tbx")), IsFalse) + + // test non exist tables by name + for _, c := range []struct{ dbName, tbName string }{ + {"db1", "tb1"}, {"db1", "tb4"}, {"db1", "tbx"}, + {"db2", "tb2"}, {"db2", "tb3"}, {"db2", "tbx"}, + {"dbx", "tb1"}, + } { + assertTableByName(sc, c.dbName, c.tbName, nil, nil) + assertTableExists(sc, c.dbName, c.tbName, false) + } + + // test non exist tables by id + nonExistID, err := genGlobalID(store) + c.Assert(err, IsNil) + + for _, id := range []int64{nonExistID, tb11.Meta().ID, tb22.Meta().ID} { + assertTableByID(sc, id, nil, nil) + } + + // test non exist table schemaByTable + assertSchemaByTable(sc, model.NewCIStr(""), tb11.Meta()) + assertSchemaByTable(sc, model.NewCIStr(""), tb22.Meta()) + assertSchemaByTable(sc, model.NewCIStr(""), nil) + + // test TemporaryTableAttachedInfoSchema + dbTest := createNewSchemaInfo("test") + tmpTbTestA := createNewTable(dbTest.ID, "tba", model.TempTableLocal) + normalTbTestA := createNewTable(dbTest.ID, "tba", model.TempTableNone) + normalTbTestB := createNewTable(dbTest.ID, "tbb", model.TempTableNone) + + is := &infoschema.TemporaryTableAttachedInfoSchema{ + InfoSchema: infoschema.MockInfoSchema([]*model.TableInfo{normalTbTestA.Meta(), normalTbTestB.Meta()}), + LocalTemporaryTables: sc, + } + + err = sc.AddTable(dbTest, tmpTbTestA) + c.Assert(err, IsNil) + + // test TableByName + tbl, err := is.TableByName(dbTest.Name, normalTbTestA.Meta().Name) + c.Assert(err, IsNil) + c.Assert(tbl, Equals, tmpTbTestA) + tbl, err = is.TableByName(dbTest.Name, normalTbTestB.Meta().Name) + c.Assert(err, IsNil) + c.Assert(tbl.Meta(), Equals, normalTbTestB.Meta()) + tbl, err = is.TableByName(db1.Name, tb11.Meta().Name) + c.Assert(infoschema.ErrTableNotExists.Equal(err), IsTrue) + c.Assert(tbl, IsNil) + tbl, err = is.TableByName(db1.Name, tb12.Meta().Name) + c.Assert(err, IsNil) + c.Assert(tbl, Equals, tb12) + + // test TableByID + tbl, ok := is.TableByID(normalTbTestA.Meta().ID) + c.Assert(ok, IsTrue) + c.Assert(tbl.Meta(), Equals, normalTbTestA.Meta()) + tbl, ok = is.TableByID(normalTbTestB.Meta().ID) + c.Assert(ok, IsTrue) + c.Assert(tbl.Meta(), Equals, normalTbTestB.Meta()) + tbl, ok = is.TableByID(tmpTbTestA.Meta().ID) + c.Assert(ok, IsTrue) + c.Assert(tbl, Equals, tmpTbTestA) + tbl, ok = is.TableByID(tb12.Meta().ID) + c.Assert(ok, IsTrue) + c.Assert(tbl, Equals, tb12) + + // test SchemaByTable + info, ok := is.SchemaByTable(normalTbTestA.Meta()) + c.Assert(ok, IsTrue) + c.Assert(info.Name.L, Equals, dbTest.Name.L) + info, ok = is.SchemaByTable(normalTbTestB.Meta()) + c.Assert(ok, IsTrue) + c.Assert(info.Name.L, Equals, dbTest.Name.L) + info, ok = is.SchemaByTable(tmpTbTestA.Meta()) + c.Assert(ok, IsTrue) + c.Assert(info.Name.L, Equals, dbTest.Name.L) + info, ok = is.SchemaByTable(tb12.Meta()) + c.Assert(ok, IsFalse) + c.Assert(info, IsNil) +} From 9cf25b6c24a340b0ee0c5c2d777abde90c6a8f9f Mon Sep 17 00:00:00 2001 From: xhe Date: Thu, 1 Jul 2021 10:09:26 +0800 Subject: [PATCH 10/16] executor: check privilege before adding (#23519) --- errors.toml | 5 ++ executor/errors.go | 1 + executor/grant.go | 103 ++++++++++++++-------------------------- executor/grant_test.go | 13 ++++- executor/revoke.go | 6 +-- executor/revoke_test.go | 9 ++-- executor/utils.go | 13 ++++- 7 files changed, 71 insertions(+), 79 deletions(-) diff --git a/errors.toml b/errors.toml index f7508f8dba983..c7a7f2f4bbc19 100644 --- a/errors.toml +++ b/errors.toml @@ -501,6 +501,11 @@ error = ''' %-.128s command denied to user '%-.48s'@'%-.64s' for table '%-.64s' ''' +["executor:1144"] +error = ''' +Illegal GRANT/REVOKE command; please consult the manual to see which privileges can be used +''' + ["executor:1213"] error = ''' Deadlock found when trying to get lock; try restarting transaction diff --git a/executor/errors.go b/executor/errors.go index 93d5e83aa3d26..a4043a7d41790 100644 --- a/executor/errors.go +++ b/executor/errors.go @@ -30,6 +30,7 @@ var ( ErrBatchInsertFail = dbterror.ClassExecutor.NewStd(mysql.ErrBatchInsertFail) ErrUnsupportedPs = dbterror.ClassExecutor.NewStd(mysql.ErrUnsupportedPs) ErrSubqueryMoreThan1Row = dbterror.ClassExecutor.NewStd(mysql.ErrSubqueryNo1Row) + ErrIllegalGrantForTable = dbterror.ClassExecutor.NewStd(mysql.ErrIllegalGrantForTable) ErrCantCreateUserWithGrant = dbterror.ClassExecutor.NewStd(mysql.ErrCantCreateUserWithGrant) ErrPasswordNoMatch = dbterror.ClassExecutor.NewStd(mysql.ErrPasswordNoMatch) diff --git a/executor/grant.go b/executor/grant.go index 49536dc79aa86..62daa116b07ae 100644 --- a/executor/grant.go +++ b/executor/grant.go @@ -547,11 +547,10 @@ func (e *GrantExec) grantColumnLevel(priv *ast.PrivElem, user *ast.UserSpec, int // composeGlobalPrivUpdate composes update stmt assignment list string for global scope privilege update. func composeGlobalPrivUpdate(sql *strings.Builder, priv mysql.PrivilegeType, value string) error { if priv != mysql.AllPriv { - col, ok := mysql.Priv2UserCol[priv] - if !ok { - return errors.Errorf("Unknown priv: %v", priv) + if priv != mysql.GrantPriv && !mysql.AllGlobalPrivs.Has(priv) { + return ErrWrongUsage.GenWithStackByArgs("GLOBAL GRANT", "NON-GLOBAL PRIVILEGES") } - sqlexec.MustFormatSQL(sql, "%n=%?", col, value) + sqlexec.MustFormatSQL(sql, "%n=%?", priv.ColumnString(), value) return nil } @@ -559,13 +558,7 @@ func composeGlobalPrivUpdate(sql *strings.Builder, priv mysql.PrivilegeType, val if i > 0 { sqlexec.MustFormatSQL(sql, ",") } - - k, ok := mysql.Priv2UserCol[v] - if !ok { - return errors.Errorf("Unknown priv %v", priv) - } - - sqlexec.MustFormatSQL(sql, "%n=%?", k, value) + sqlexec.MustFormatSQL(sql, "%n=%?", v.ColumnString(), value) } return nil } @@ -573,11 +566,10 @@ func composeGlobalPrivUpdate(sql *strings.Builder, priv mysql.PrivilegeType, val // composeDBPrivUpdate composes update stmt assignment list for db scope privilege update. func composeDBPrivUpdate(sql *strings.Builder, priv mysql.PrivilegeType, value string) error { if priv != mysql.AllPriv { - col, ok := mysql.Priv2UserCol[priv] - if !ok { - return errors.Errorf("Unknown priv: %v", priv) + if priv != mysql.GrantPriv && !mysql.AllDBPrivs.Has(priv) { + return ErrWrongUsage.GenWithStackByArgs("DB GRANT", "NON-DB PRIVILEGES") } - sqlexec.MustFormatSQL(sql, "%n=%?", col, value) + sqlexec.MustFormatSQL(sql, "%n=%?", priv.ColumnString(), value) return nil } @@ -585,93 +577,68 @@ func composeDBPrivUpdate(sql *strings.Builder, priv mysql.PrivilegeType, value s if i > 0 { sqlexec.MustFormatSQL(sql, ",") } - - v, ok := mysql.Priv2UserCol[p] - if !ok { - return errors.Errorf("Unknown priv %v", priv) - } - - sqlexec.MustFormatSQL(sql, "%n=%?", v, value) + sqlexec.MustFormatSQL(sql, "%n=%?", p.ColumnString(), value) } return nil } -func privUpdateForGrant(cur []string, priv mysql.PrivilegeType) ([]string, error) { - p, ok := mysql.Priv2SetStr[priv] - if !ok { - return nil, errors.Errorf("Unknown priv: %v", priv) - } - cur = addToSet(cur, p) - return cur, nil -} - // composeTablePrivUpdateForGrant composes update stmt assignment list for table scope privilege update. func composeTablePrivUpdateForGrant(ctx sessionctx.Context, sql *strings.Builder, priv mysql.PrivilegeType, name string, host string, db string, tbl string) error { var newTablePriv, newColumnPriv []string - var tblPrivs, colPrivs []mysql.PrivilegeType if priv != mysql.AllPriv { + // TODO: https://github.com/pingcap/parser/pull/581 removed privs from all priv lists + // it is to avoid add GRANT in GRANT ALL SQLs + // WithGRANT seems broken, fix it later + if priv != mysql.GrantPriv && !mysql.AllTablePrivs.Has(priv) { + return ErrIllegalGrantForTable + } + currTablePriv, currColumnPriv, err := getTablePriv(ctx, name, host, db, tbl) if err != nil { return err } - newTablePriv = setFromString(currTablePriv) - newColumnPriv = setFromString(currColumnPriv) - tblPrivs = []mysql.PrivilegeType{priv} - for _, cp := range mysql.AllColumnPrivs { - // in case it is not a column priv - if cp == priv { - colPrivs = []mysql.PrivilegeType{priv} - break - } + newTablePriv = SetFromString(currTablePriv) + newTablePriv = addToSet(newTablePriv, priv.SetString()) + + newColumnPriv = SetFromString(currColumnPriv) + if mysql.AllColumnPrivs.Has(priv) { + newColumnPriv = addToSet(newColumnPriv, priv.SetString()) } } else { - tblPrivs = mysql.AllTablePrivs - colPrivs = mysql.AllColumnPrivs - } - - var err error - for _, p := range tblPrivs { - newTablePriv, err = privUpdateForGrant(newTablePriv, p) - if err != nil { - return err + for _, p := range mysql.AllTablePrivs { + newTablePriv = addToSet(newTablePriv, p.SetString()) } - } - for _, p := range colPrivs { - newColumnPriv, err = privUpdateForGrant(newColumnPriv, p) - if err != nil { - return err + for _, p := range mysql.AllColumnPrivs { + newColumnPriv = addToSet(newColumnPriv, p.SetString()) } } - sqlexec.MustFormatSQL(sql, `Table_priv=%?, Column_priv=%?, Grantor=%?`, strings.Join(newTablePriv, ","), strings.Join(newColumnPriv, ","), ctx.GetSessionVars().User.String()) + sqlexec.MustFormatSQL(sql, `Table_priv=%?, Column_priv=%?, Grantor=%?`, setToString(newTablePriv), setToString(newColumnPriv), ctx.GetSessionVars().User.String()) return nil } // composeColumnPrivUpdateForGrant composes update stmt assignment list for column scope privilege update. func composeColumnPrivUpdateForGrant(ctx sessionctx.Context, sql *strings.Builder, priv mysql.PrivilegeType, name string, host string, db string, tbl string, col string) error { var newColumnPriv []string - var colPrivs []mysql.PrivilegeType if priv != mysql.AllPriv { + if !mysql.AllColumnPrivs.Has(priv) { + return ErrWrongUsage.GenWithStackByArgs("COLUMN GRANT", "NON-COLUMN PRIVILEGES") + } + currColumnPriv, err := getColumnPriv(ctx, name, host, db, tbl, col) if err != nil { return err } - newColumnPriv = setFromString(currColumnPriv) - colPrivs = []mysql.PrivilegeType{priv} + newColumnPriv = SetFromString(currColumnPriv) + newColumnPriv = addToSet(newColumnPriv, priv.SetString()) } else { - colPrivs = mysql.AllColumnPrivs - } - - var err error - for _, p := range colPrivs { - newColumnPriv, err = privUpdateForGrant(newColumnPriv, p) - if err != nil { - return err + for _, p := range mysql.AllColumnPrivs { + newColumnPriv = addToSet(newColumnPriv, p.SetString()) } } - sqlexec.MustFormatSQL(sql, `Column_priv=%?`, strings.Join(newColumnPriv, ",")) + sqlexec.MustFormatSQL(sql, `Column_priv=%?`, setToString(newColumnPriv)) return nil } diff --git a/executor/grant_test.go b/executor/grant_test.go index df45169b23ff5..6e6369e40bd7b 100644 --- a/executor/grant_test.go +++ b/executor/grant_test.go @@ -93,6 +93,9 @@ func (s *testSuite3) TestGrantDBScope(c *C) { // Grant in wrong scope. _, err := tk.Exec(` grant create user on test.* to 'testDB1'@'localhost';`) c.Assert(terror.ErrorEqual(err, executor.ErrWrongUsage.GenWithStackByArgs("DB GRANT", "GLOBAL PRIVILEGES")), IsTrue) + + _, err = tk.Exec("GRANT SUPER ON test.* TO 'testDB1'@'localhost';") + c.Assert(terror.ErrorEqual(err, executor.ErrWrongUsage.GenWithStackByArgs("DB GRANT", "NON-DB PRIVILEGES")), IsTrue) } func (s *testSuite3) TestWithGrantOption(c *C) { @@ -116,7 +119,7 @@ func (s *testSuite3) TestWithGrantOption(c *C) { tk.MustQuery("SELECT grant_priv FROM mysql.user WHERE User=\"testWithGrant1\"").Check(testkit.Rows("Y")) } -func (s *testSuiteP1) TestTableScope(c *C) { +func (s *testSuiteP1) TestGrantTableScope(c *C) { tk := testkit.NewTestKit(c, s.store) // Create a new user. createUserSQL := `CREATE USER 'testTbl'@'localhost' IDENTIFIED BY '123';` @@ -152,9 +155,12 @@ func (s *testSuiteP1) TestTableScope(c *C) { p := fmt.Sprintf("%v", row[0]) c.Assert(strings.Index(p, mysql.Priv2SetStr[v]), Greater, -1) } + + _, err := tk.Exec("GRANT SUPER ON test2 TO 'testTbl1'@'localhost';") + c.Assert(err, ErrorMatches, "\\[executor:1144\\]Illegal GRANT/REVOKE command; please consult the manual to see which privileges can be used") } -func (s *testSuite3) TestColumnScope(c *C) { +func (s *testSuite3) TestGrantColumnScope(c *C) { tk := testkit.NewTestKit(c, s.store) // Create a new user. createUserSQL := `CREATE USER 'testCol'@'localhost' IDENTIFIED BY '123';` @@ -192,6 +198,9 @@ func (s *testSuite3) TestColumnScope(c *C) { p := fmt.Sprintf("%v", row[0]) c.Assert(strings.Index(p, mysql.Priv2SetStr[v]), Greater, -1) } + + _, err := tk.Exec("GRANT SUPER(c2) ON test3 TO 'testCol1'@'localhost';") + c.Assert(err, ErrorMatches, "\\[executor:1221\\]Incorrect usage of COLUMN GRANT and NON-COLUMN PRIVILEGES") } func (s *testSuite3) TestIssue2456(c *C) { diff --git a/executor/revoke.go b/executor/revoke.go index f84bbac9676da..3cc14ec3d1856 100644 --- a/executor/revoke.go +++ b/executor/revoke.go @@ -303,13 +303,13 @@ func composeTablePrivUpdateForRevoke(ctx sessionctx.Context, sql *strings.Builde return err } - newTablePriv = setFromString(currTablePriv) + newTablePriv = SetFromString(currTablePriv) newTablePriv, err = privUpdateForRevoke(newTablePriv, priv) if err != nil { return err } - newColumnPriv = setFromString(currColumnPriv) + newColumnPriv = SetFromString(currColumnPriv) newColumnPriv, err = privUpdateForRevoke(newColumnPriv, priv) if err != nil { return err @@ -329,7 +329,7 @@ func composeColumnPrivUpdateForRevoke(ctx sessionctx.Context, sql *strings.Build return err } - newColumnPriv = setFromString(currColumnPriv) + newColumnPriv = SetFromString(currColumnPriv) newColumnPriv, err = privUpdateForRevoke(newColumnPriv, priv) if err != nil { return err diff --git a/executor/revoke_test.go b/executor/revoke_test.go index f4a9ea8df231b..858ec777bfc09 100644 --- a/executor/revoke_test.go +++ b/executor/revoke_test.go @@ -94,15 +94,16 @@ func (s *testSuite1) TestRevokeTableScope(c *C) { c.Assert(rows, HasLen, 1) row := rows[0] c.Assert(row, HasLen, 1) - op := mysql.Priv2SetStr[v] + + op := v.SetString() found := false - for _, v := range strings.Split(fmt.Sprintf("%v", row[0]), ",") { - if v == op { + for _, p := range executor.SetFromString(fmt.Sprintf("%s", row[0])) { + if op == p { found = true break } } - c.Assert(found, IsFalse) + c.Assert(found, IsFalse, Commentf("%s", mysql.Priv2SetStr[v])) } // Revoke all table scope privs. diff --git a/executor/utils.go b/executor/utils.go index fbc9ab4dcff30..217e32ad487ee 100644 --- a/executor/utils.go +++ b/executor/utils.go @@ -13,15 +13,24 @@ package executor -import "strings" +import ( + "strings" +) -func setFromString(value string) []string { +// SetFromString constructs a slice of strings from a comma separated string. +// It is assumed that there is no duplicated entry. You could use addToSet to maintain this property. +// It is exported for tests. I HOPE YOU KNOW WHAT YOU ARE DOING. +func SetFromString(value string) []string { if len(value) == 0 { return nil } return strings.Split(value, ",") } +func setToString(set []string) string { + return strings.Join(set, ",") +} + // addToSet add a value to the set, e.g: // addToSet("Select,Insert,Update", "Update") returns "Select,Insert,Update". func addToSet(set []string, value string) []string { From 209334951cd920e5408f67fd60a0b542507526ee Mon Sep 17 00:00:00 2001 From: Mattias Jonsson Date: Thu, 1 Jul 2021 04:37:27 +0200 Subject: [PATCH 11/16] sessionctx: add tidb_enable_list_partition global system variable (#25743) --- session/session_test.go | 13 +++++++++++++ sessionctx/variable/sysvar.go | 2 +- sessionctx/variable/varsutil_test.go | 3 +++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/session/session_test.go b/session/session_test.go index 04b35919f2992..d6874b80d28f5 100644 --- a/session/session_test.go +++ b/session/session_test.go @@ -2800,6 +2800,9 @@ func (s *testSessionSuite3) TestEnablePartition(c *C) { tk.MustExec("set tidb_enable_list_partition=off") tk.MustQuery("show variables like 'tidb_enable_list_partition'").Check(testkit.Rows("tidb_enable_list_partition OFF")) + tk.MustExec("set global tidb_enable_list_partition=on") + tk.MustQuery("show global variables like 'tidb_enable_list_partition'").Check(testkit.Rows("tidb_enable_list_partition ON")) + tk.MustQuery("show variables like 'tidb_enable_list_partition'").Check(testkit.Rows("tidb_enable_list_partition OFF")) tk.MustExec("set tidb_enable_list_partition=1") tk.MustQuery("show variables like 'tidb_enable_list_partition'").Check(testkit.Rows("tidb_enable_list_partition ON")) @@ -2807,8 +2810,18 @@ func (s *testSessionSuite3) TestEnablePartition(c *C) { tk.MustExec("set tidb_enable_list_partition=on") tk.MustQuery("show variables like 'tidb_enable_list_partition'").Check(testkit.Rows("tidb_enable_list_partition ON")) + tk.MustQuery("show global variables like 'tidb_enable_list_partition'").Check(testkit.Rows("tidb_enable_list_partition ON")) + tk.MustExec("set global tidb_enable_list_partition=off") + tk.MustQuery("show global variables like 'tidb_enable_list_partition'").Check(testkit.Rows("tidb_enable_list_partition OFF")) + tk.MustQuery("show variables like 'tidb_enable_list_partition'").Check(testkit.Rows("tidb_enable_list_partition ON")) + tk.MustExec("set tidb_enable_list_partition=off") + tk.MustQuery("show variables like 'tidb_enable_list_partition'").Check(testkit.Rows("tidb_enable_list_partition OFF")) + + tk.MustExec("set global tidb_enable_list_partition=on") + tk.MustQuery("show global variables like 'tidb_enable_list_partition'").Check(testkit.Rows("tidb_enable_list_partition ON")) tk1 := testkit.NewTestKitWithInit(c, s.store) tk1.MustQuery("show variables like 'tidb_enable_table_partition'").Check(testkit.Rows("tidb_enable_table_partition ON")) + tk1.MustQuery("show variables like 'tidb_enable_list_partition'").Check(testkit.Rows("tidb_enable_list_partition ON")) } func (s *testSessionSerialSuite) TestTxnRetryErrMsg(c *C) { diff --git a/sessionctx/variable/sysvar.go b/sessionctx/variable/sysvar.go index 028c92acad0d3..6fe67eab20014 100644 --- a/sessionctx/variable/sysvar.go +++ b/sessionctx/variable/sysvar.go @@ -1128,7 +1128,7 @@ var defaultSysVars = []*SysVar{ s.EnableTablePartition = val return nil }}, - {Scope: ScopeSession, Name: TiDBEnableListTablePartition, Value: Off, Type: TypeBool, skipInit: true, SetSession: func(s *SessionVars, val string) error { + {Scope: ScopeGlobal | ScopeSession, Name: TiDBEnableListTablePartition, Value: Off, Type: TypeBool, SetSession: func(s *SessionVars, val string) error { s.EnableListTablePartition = TiDBOptOn(val) return nil }}, diff --git a/sessionctx/variable/varsutil_test.go b/sessionctx/variable/varsutil_test.go index 41f379a9d6353..93bea338df825 100644 --- a/sessionctx/variable/varsutil_test.go +++ b/sessionctx/variable/varsutil_test.go @@ -552,6 +552,9 @@ func (s *testVarsutilSuite) TestValidate(c *C) { {TiDBEnableTablePartition, "OFF", false}, {TiDBEnableTablePartition, "AUTO", false}, {TiDBEnableTablePartition, "UN", true}, + {TiDBEnableListTablePartition, "ON", false}, + {TiDBEnableListTablePartition, "OFF", false}, + {TiDBEnableListTablePartition, "list", true}, {TiDBOptCorrelationExpFactor, "a", true}, {TiDBOptCorrelationExpFactor, "-10", true}, {TiDBOptCorrelationThreshold, "a", true}, From 57997a9b20f999e68cfdee6770e373e61e57e9c4 Mon Sep 17 00:00:00 2001 From: Lei Zhao Date: Thu, 1 Jul 2021 16:29:26 +0800 Subject: [PATCH 12/16] *: upgrade client-go to fix backoff panic (#25865) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9daec0e6d9064..1014a369a1bc1 100644 --- a/go.mod +++ b/go.mod @@ -55,7 +55,7 @@ require ( github.com/shirou/gopsutil v3.21.2+incompatible github.com/soheilhy/cmux v0.1.4 github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 - github.com/tikv/client-go/v2 v2.0.0-alpha.0.20210630040115-58b6783d1b56 + github.com/tikv/client-go/v2 v2.0.0-alpha.0.20210701075128-88f909bcdd3f github.com/tikv/pd v1.1.0-beta.0.20210323121136-78679e5e209d github.com/twmb/murmur3 v1.1.3 github.com/uber-go/atomic v1.4.0 diff --git a/go.sum b/go.sum index f8823b5413c4b..5acdf43775659 100644 --- a/go.sum +++ b/go.sum @@ -558,8 +558,8 @@ github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2/go.mod h1:2PfK github.com/tidwall/gjson v1.3.5/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tikv/client-go/v2 v2.0.0-alpha.0.20210630040115-58b6783d1b56 h1:R1jC5I6wJmmfrYGNxAPR+ABbo1T2KMWcEWPr+UN5Bec= -github.com/tikv/client-go/v2 v2.0.0-alpha.0.20210630040115-58b6783d1b56/go.mod h1:crzTwbliZf57xC5ZSzmQx4iMZCLCGhA364to+E2JAPU= +github.com/tikv/client-go/v2 v2.0.0-alpha.0.20210701075128-88f909bcdd3f h1:T3zFmJfdvmF+sVUvLsZKJZmCzfkbo0O0DjlbQdmd74A= +github.com/tikv/client-go/v2 v2.0.0-alpha.0.20210701075128-88f909bcdd3f/go.mod h1:crzTwbliZf57xC5ZSzmQx4iMZCLCGhA364to+E2JAPU= github.com/tikv/pd v1.1.0-beta.0.20210323121136-78679e5e209d h1:K0XnvsnT6ofLDuM8Rt3PuFQO4p8bNraeHYstspD316g= github.com/tikv/pd v1.1.0-beta.0.20210323121136-78679e5e209d/go.mod h1:Jw9KG11C/23Rr7DW4XWQ7H5xOgGZo6DFL1OKAF4+Igw= github.com/tklauser/go-sysconf v0.3.4 h1:HT8SVixZd3IzLdfs/xlpq0jeSfTX57g1v6wB1EuzV7M= From 28202b3521a50469d102516975c1522cac2fb0c1 Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Thu, 1 Jul 2021 17:29:26 +0800 Subject: [PATCH 13/16] Makefile,session: collect the benchmark data and generate json output file (#25834) --- Makefile | 7 +++ session/bench_test.go | 109 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+) diff --git a/Makefile b/Makefile index 6451340aed4e3..7db0be5936a34 100644 --- a/Makefile +++ b/Makefile @@ -262,3 +262,10 @@ else $(GOTEST) -ldflags '$(TEST_LDFLAGS)' -cover github.com/pingcap/tidb/$(pkg) -check.p true -check.timeout 4s || { $(FAILPOINT_DISABLE); exit 1; } endif @$(FAILPOINT_DISABLE) + +# Collect the daily benchmark data. +# Usage: +# make bench-daily TO=/path/to/file.json +bench-daily: + cd ./session && \ + go test -run TestBenchDaily --date `git log -n1 --date=unix --pretty=format:%cd` --commit `git log -n1 --pretty=format:%h` --outfile $(TO) diff --git a/session/bench_test.go b/session/bench_test.go index 590cab932891b..60e700304a212 100644 --- a/session/bench_test.go +++ b/session/bench_test.go @@ -15,8 +15,13 @@ package session import ( "context" + "encoding/json" + "flag" "fmt" "math/rand" + "os" + "reflect" + "runtime" "strconv" "strings" "testing" @@ -1584,3 +1589,107 @@ func BenchmarkHashPartitionPruningMultiSelect(b *testing.B) { } b.StopTimer() } + +type BenchOutput struct { + Date string + Commit string + Result []BenchResult +} + +type BenchResult struct { + Name string + NsPerOp int64 + AllocsPerOp int64 + BytesPerOp int64 +} + +func benchmarkResultToJSON(name string, r testing.BenchmarkResult) BenchResult { + return BenchResult{ + Name: name, + NsPerOp: r.NsPerOp(), + AllocsPerOp: r.AllocsPerOp(), + BytesPerOp: r.AllocedBytesPerOp(), + } +} + +func callerName(f func(b *testing.B)) string { + fullName := runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name() + idx := strings.LastIndexByte(fullName, '.') + if idx > 0 && idx+1 < len(fullName) { + return fullName[idx+1:] + } + return fullName +} + +var ( + date = flag.String("date", "", " commit date") + commitHash = flag.String("commit", "unknown", "brief git commit hash") + outfile = flag.String("outfile", "bench-daily.json", "specify the output file") +) + +// TestBenchDaily collects the daily benchmark test result and generates a json output file. +// The format of the json output is described by the BenchOutput. +// Used by this command in the Makefile +// make bench-daily TO=xxx.json +func TestBenchDaily(t *testing.T) { + if !flag.Parsed() { + flag.Parse() + } + + if *date == "" { + // Don't run unless 'date' is specified. + // Avoiding slow down the CI. + return + } + + tests := []func(b *testing.B){ + BenchmarkBasic, + BenchmarkTableScan, + BenchmarkTableLookup, + BenchmarkExplainTableLookup, + BenchmarkStringIndexScan, + BenchmarkExplainStringIndexScan, + BenchmarkStringIndexLookup, + BenchmarkIntegerIndexScan, + BenchmarkIntegerIndexLookup, + BenchmarkDecimalIndexScan, + BenchmarkDecimalIndexLookup, + BenchmarkInsertWithIndex, + BenchmarkInsertNoIndex, + BenchmarkSort, + BenchmarkJoin, + BenchmarkJoinLimit, + BenchmarkPartitionPruning, + BenchmarkRangeColumnPartitionPruning, + BenchmarkHashPartitionPruningPointSelect, + BenchmarkHashPartitionPruningMultiSelect, + } + + res := make([]BenchResult, 0, len(tests)) + for _, t := range tests { + name := callerName(t) + r1 := testing.Benchmark(t) + r2 := benchmarkResultToJSON(name, r1) + res = append(res, r2) + } + + if *outfile == "" { + *outfile = fmt.Sprintf("%s_%s.json", *date, *commitHash) + } + out, err := os.Create(*outfile) + if err != nil { + t.Fatal(err) + } + defer out.Close() + + output := BenchOutput{ + Date: *date, + Commit: *commitHash, + Result: res, + } + enc := json.NewEncoder(out) + err = enc.Encode(output) + if err != nil { + t.Fatal(err) + } +} From 008d5e3fd2bccb0b2729b272b0ae6c8ff7827a42 Mon Sep 17 00:00:00 2001 From: zeroslope <10218146+zeroslope@users.noreply.github.com> Date: Thu, 1 Jul 2021 17:39:26 +0800 Subject: [PATCH 14/16] parser, core: Implement force_index hint in parser and TiDB (#23836) --- go.mod | 2 +- go.sum | 4 +- planner/core/integration_test.go | 8 +++ planner/core/logical_plan_builder.go | 21 +++++- planner/core/logical_plan_test.go | 1 + .../core/testdata/integration_suite_out.json | 8 +-- planner/core/testdata/plan_suite_in.json | 15 +++- planner/core/testdata/plan_suite_out.json | 72 +++++++++++++++++++ 8 files changed, 121 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 1014a369a1bc1..497071bfa388e 100644 --- a/go.mod +++ b/go.mod @@ -44,7 +44,7 @@ require ( github.com/pingcap/fn v0.0.0-20200306044125-d5540d389059 github.com/pingcap/kvproto v0.0.0-20210611081648-a215b4e61d2f github.com/pingcap/log v0.0.0-20210317133921-96f4fcab92a4 - github.com/pingcap/parser v0.0.0-20210618053735-57843e8185c4 + github.com/pingcap/parser v0.0.0-20210618124025-07ae8d9291c4 github.com/pingcap/sysutil v0.0.0-20210315073920-cc0985d983a3 github.com/pingcap/tidb-tools v4.0.9-0.20201127090955-2707c97b3853+incompatible github.com/pingcap/tipb v0.0.0-20210628060001-1793e022b962 diff --git a/go.sum b/go.sum index 5acdf43775659..8cc16ae095365 100644 --- a/go.sum +++ b/go.sum @@ -443,8 +443,8 @@ github.com/pingcap/log v0.0.0-20201112100606-8f1e84a3abc8/go.mod h1:4rbK1p9ILyIf github.com/pingcap/log v0.0.0-20210317133921-96f4fcab92a4 h1:ERrF0fTuIOnwfGbt71Ji3DKbOEaP189tjym50u8gpC8= github.com/pingcap/log v0.0.0-20210317133921-96f4fcab92a4/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/parser v0.0.0-20210525032559-c37778aff307/go.mod h1:xZC8I7bug4GJ5KtHhgAikjTfU4kBv1Sbo3Pf1MZ6lVw= -github.com/pingcap/parser v0.0.0-20210618053735-57843e8185c4 h1:NASsbyMTNW8pbYfoO/YTykO6MQJiNRa094lwCPU6R2Q= -github.com/pingcap/parser v0.0.0-20210618053735-57843e8185c4/go.mod h1:xZC8I7bug4GJ5KtHhgAikjTfU4kBv1Sbo3Pf1MZ6lVw= +github.com/pingcap/parser v0.0.0-20210618124025-07ae8d9291c4 h1:np2wD3C5QbdTUD17ubVlrgEsklQOKN5iBm/SiV846OM= +github.com/pingcap/parser v0.0.0-20210618124025-07ae8d9291c4/go.mod h1:xZC8I7bug4GJ5KtHhgAikjTfU4kBv1Sbo3Pf1MZ6lVw= github.com/pingcap/sysutil v0.0.0-20200206130906-2bfa6dc40bcd/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI= github.com/pingcap/sysutil v0.0.0-20210315073920-cc0985d983a3 h1:A9KL9R+lWSVPH8IqUuH1QSTRJ5FGoY1bT2IcfPKsWD8= github.com/pingcap/sysutil v0.0.0-20210315073920-cc0985d983a3/go.mod h1:tckvA041UWP+NqYzrJ3fMgC/Hw9wnmQ/tUkp/JaHly8= diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index 7a882a85ba8f9..07bb8e121b081 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -1461,6 +1461,14 @@ func (s *testIntegrationSuite) TestInvisibleIndex(c *C) { tk.MustQuery("select /*+ IGNORE_INDEX(t, i_a), USE_INDEX(t, i_b) */ a from t order by a") c.Assert(tk.Se.GetSessionVars().StmtCtx.GetWarnings(), HasLen, 1) c.Assert(tk.Se.GetSessionVars().StmtCtx.GetWarnings()[0].Err.Error(), Equals, errStr) + tk.MustQuery("select /*+ FORCE_INDEX(t, i_a), USE_INDEX(t, i_b) */ a from t order by a") + c.Assert(tk.Se.GetSessionVars().StmtCtx.GetWarnings(), HasLen, 1) + c.Assert(tk.Se.GetSessionVars().StmtCtx.GetWarnings()[0].Err.Error(), Equals, errStr) + // For issue 15519 + inapplicableErrStr := "[planner:1815]force_index(test.aaa) is inapplicable, check whether the table(test.aaa) exists" + tk.MustQuery("select /*+ FORCE_INDEX(aaa) */ * from t") + c.Assert(tk.Se.GetSessionVars().StmtCtx.GetWarnings(), HasLen, 1) + c.Assert(tk.Se.GetSessionVars().StmtCtx.GetWarnings()[0].Err.Error(), Equals, inapplicableErrStr) tk.MustExec("admin check table t") tk.MustExec("admin check index t i_a") diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index c0ffdc9921edb..c024518a90156 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -90,6 +90,8 @@ const ( HintUseIndex = "use_index" // HintIgnoreIndex is hint enforce ignoring some indexes. HintIgnoreIndex = "ignore_index" + // HintForceIndex make optimizer to use this index even if it thinks a table scan is more efficient. + HintForceIndex = "force_index" // HintAggToCop is hint enforce pushing aggregation to coprocessor. HintAggToCop = "agg_to_cop" // HintReadFromStorage is hint enforce some tables read from specific type of storage. @@ -3157,7 +3159,7 @@ func (b *PlanBuilder) pushTableHints(hints []*ast.TableOptimizerHint, currentLev // Set warning for the hint that requires the table name. switch hint.HintName.L { case TiDBMergeJoin, HintSMJ, TiDBIndexNestedLoopJoin, HintINLJ, HintINLHJ, HintINLMJ, - TiDBHashJoin, HintHJ, HintUseIndex, HintIgnoreIndex, HintIndexMerge: + TiDBHashJoin, HintHJ, HintUseIndex, HintIgnoreIndex, HintForceIndex, HintIndexMerge: if len(hint.Tables) == 0 { b.pushHintWithoutTableWarning(hint) continue @@ -3215,6 +3217,21 @@ func (b *PlanBuilder) pushTableHints(hints []*ast.TableOptimizerHint, currentLev HintScope: ast.HintForScan, }, }) + case HintForceIndex: + dbName := hint.Tables[0].DBName + if dbName.L == "" { + dbName = model.NewCIStr(b.ctx.GetSessionVars().CurrentDB) + } + indexHintList = append(indexHintList, indexHintInfo{ + dbName: dbName, + tblName: hint.Tables[0].TableName, + partitions: hint.Tables[0].PartitionList, + indexHint: &ast.IndexHint{ + IndexNames: hint.Indexes, + HintType: ast.HintForce, + HintScope: ast.HintForScan, + }, + }) case HintReadFromStorage: switch hint.HintData.(model.CIStr).L { case HintTiFlash: @@ -3901,7 +3918,7 @@ func (b *PlanBuilder) buildDataSource(ctx context.Context, tn *ast.TableName, as } else { // Append warning if there are invalid index names. errMsg := fmt.Sprintf("use_index_merge(%s) is inapplicable, check whether the indexes (%s) "+ - "exist, or the indexes are conflicted with use_index/ignore_index hints.", + "exist, or the indexes are conflicted with use_index/ignore_index/force_index hints.", hint.indexString(), strings.Join(invalidIdxNames, ", ")) b.ctx.GetSessionVars().StmtCtx.AppendWarning(ErrInternal.GenWithStack(errMsg)) } diff --git a/planner/core/logical_plan_test.go b/planner/core/logical_plan_test.go index 9d3df6ab911a4..c4bed173fae17 100644 --- a/planner/core/logical_plan_test.go +++ b/planner/core/logical_plan_test.go @@ -1473,6 +1473,7 @@ func (s *testPlanSuite) TestNameResolver(c *C) { {"delete a from (select * from t ) as a, t", "[planner:1288]The target table a of the DELETE is not updatable"}, {"delete b from (select * from t ) as a, t", "[planner:1109]Unknown table 'b' in MULTI DELETE"}, {"select '' as fakeCol from t group by values(fakeCol)", "[planner:1054]Unknown column '' in 'VALUES() function'"}, + {"update t, (select * from t) as b set b.a = t.a", "[planner:1288]The target table b of the UPDATE is not updatable"}, {"select row_number() over () from t group by 1", "[planner:1056]Can't group on 'row_number() over ()'"}, {"select row_number() over () as x from t group by 1", "[planner:1056]Can't group on 'x'"}, {"select sum(a) as x from t group by 1", "[planner:1056]Can't group on 'x'"}, diff --git a/planner/core/testdata/integration_suite_out.json b/planner/core/testdata/integration_suite_out.json index e0fa32e578537..37330e65673c9 100644 --- a/planner/core/testdata/integration_suite_out.json +++ b/planner/core/testdata/integration_suite_out.json @@ -604,25 +604,25 @@ { "SQL": "select /*+ USE_INDEX_MERGE(t1, a, b, c, d) */ * from t1", "Warnings": [ - "[planner:1815]use_index_merge(test.t1, a, b, c, d) is inapplicable, check whether the indexes (c, d) exist, or the indexes are conflicted with use_index/ignore_index hints." + "[planner:1815]use_index_merge(test.t1, a, b, c, d) is inapplicable, check whether the indexes (c, d) exist, or the indexes are conflicted with use_index/ignore_index/force_index hints." ] }, { "SQL": "select /*+ USE_INDEX_MERGE(t1, a, b), USE_INDEX(t1, a) */ * from t1", "Warnings": [ - "[planner:1815]use_index_merge(test.t1, a, b) is inapplicable, check whether the indexes (b) exist, or the indexes are conflicted with use_index/ignore_index hints." + "[planner:1815]use_index_merge(test.t1, a, b) is inapplicable, check whether the indexes (b) exist, or the indexes are conflicted with use_index/ignore_index/force_index hints." ] }, { "SQL": "select /*+ USE_INDEX_MERGE(t1, a, b), IGNORE_INDEX(t1, a) */ * from t1", "Warnings": [ - "[planner:1815]use_index_merge(test.t1, a, b) is inapplicable, check whether the indexes (a) exist, or the indexes are conflicted with use_index/ignore_index hints." + "[planner:1815]use_index_merge(test.t1, a, b) is inapplicable, check whether the indexes (a) exist, or the indexes are conflicted with use_index/ignore_index/force_index hints." ] }, { "SQL": "select /*+ USE_INDEX_MERGE(t1, primary, a, b, c) */ * from t1", "Warnings": [ - "[planner:1815]use_index_merge(test.t1, primary, a, b, c) is inapplicable, check whether the indexes (c) exist, or the indexes are conflicted with use_index/ignore_index hints." + "[planner:1815]use_index_merge(test.t1, primary, a, b, c) is inapplicable, check whether the indexes (c) exist, or the indexes are conflicted with use_index/ignore_index/force_index hints." ] } ] diff --git a/planner/core/testdata/plan_suite_in.json b/planner/core/testdata/plan_suite_in.json index dc420972aaa50..d16c67e69a251 100644 --- a/planner/core/testdata/plan_suite_in.json +++ b/planner/core/testdata/plan_suite_in.json @@ -31,24 +31,37 @@ "select /*+ USE_INDEX(test.t, c_d_e) */ * from t", "select /*+ IGNORE_INDEX(t, c_d_e) */ c from t order by c", "select /*+ IGNORE_INDEX(test.t, c_d_e) */ c from t order by c", + "select /*+ FORCE_INDEX(t, c_d_e) */ * from t", + "select /*+ FORCE_INDEX(test.t, c_d_e) */ * from t", "select /*+ USE_INDEX(t, c_d_e) */ * from t t1", "select /*+ IGNORE_INDEX(t, c_d_e) */ t1.c from t t1 order by t1.c", + "select /*+ FORCE_INDEX(t, c_d_e) */ * from t t1", "select /*+ USE_INDEX(t1, c_d_e) */ * from t t1", "select /*+ IGNORE_INDEX(t1, c_d_e) */ t1.c from t t1 order by t1.c", + "select /*+ FORCE_INDEX(t1, c_d_e) */ * from t t1", "select /*+ USE_INDEX(t1, c_d_e), USE_INDEX(t2, f) */ * from t t1, t t2 where t1.a = t2.b", "select /*+ IGNORE_INDEX(t1, c_d_e), IGNORE_INDEX(t2, f), HASH_JOIN(t1) */ * from t t1, t t2 where t1.a = t2.b", + "select /*+ FORCE_INDEX(t1, c_d_e), FORCE_INDEX(t2, f) */ * from t t1, t t2 where t1.a = t2.b", // test multiple indexes "select /*+ USE_INDEX(t, c_d_e, f, g) */ * from t order by f", + "select /*+ FORCE_INDEX(t, c_d_e, f, g) */ * from t order by f", // use TablePath when the hint only contains table. "select /*+ USE_INDEX(t) */ f from t where f > 10", + "select /*+ FORCE_INDEX(t) */ f from t where f > 10", // there will be a warning instead of error when index not exist "select /*+ USE_INDEX(t, no_such_index) */ * from t", "select /*+ IGNORE_INDEX(t, no_such_index) */ * from t", + "select /*+ FORCE_INDEX(t, no_such_index) */ * from t", // use both use_index and ignore_index, same as index hints in sql. "select /*+ USE_INDEX(t, c_d_e), IGNORE_INDEX(t, f) */ c from t order by c", "select /*+ USE_INDEX(t, f), IGNORE_INDEX(t, f) */ c from t order by c", "select /*+ USE_INDEX(t, c_d_e), IGNORE_INDEX(t, c_d_e) */ c from t order by c", - "select /*+ USE_INDEX(t, c_d_e, f), IGNORE_INDEX(t, c_d_e) */ c from t order by c" + "select /*+ USE_INDEX(t, c_d_e, f), IGNORE_INDEX(t, c_d_e) */ c from t order by c", + // use both force_index and ignore_index, same as index hints in sql. + "select /*+ FORCE_INDEX(t, c_d_e), IGNORE_INDEX(t, f) */ c from t order by c", + "select /*+ FORCE_INDEX(t, f), IGNORE_INDEX(t, f) */ c from t order by c", + "select /*+ FORCE_INDEX(t, c_d_e), IGNORE_INDEX(t, c_d_e) */ c from t order by c", + "select /*+ FORCE_INDEX(t, c_d_e, f), IGNORE_INDEX(t, c_d_e) */ c from t order by c" ] }, { diff --git a/planner/core/testdata/plan_suite_out.json b/planner/core/testdata/plan_suite_out.json index 9e2d5b248a7ac..c3156e0c1ad97 100644 --- a/planner/core/testdata/plan_suite_out.json +++ b/planner/core/testdata/plan_suite_out.json @@ -99,6 +99,18 @@ "HasWarn": false, "Hints": "use_index(@`sel_1` `test`.`t` )" }, + { + "SQL": "select /*+ FORCE_INDEX(t, c_d_e) */ * from t", + "Best": "IndexLookUp(Index(t.c_d_e)[[NULL,+inf]], Table(t))", + "HasWarn": false, + "Hints": "use_index(@`sel_1` `test`.`t` `c_d_e`)" + }, + { + "SQL": "select /*+ FORCE_INDEX(test.t, c_d_e) */ * from t", + "Best": "IndexLookUp(Index(t.c_d_e)[[NULL,+inf]], Table(t))", + "HasWarn": false, + "Hints": "use_index(@`sel_1` `test`.`t` `c_d_e`)" + }, { "SQL": "select /*+ USE_INDEX(t, c_d_e) */ * from t t1", "Best": "TableReader(Table(t))", @@ -111,6 +123,12 @@ "HasWarn": true, "Hints": "use_index(@`sel_1` `test`.`t1` `c_d_e`)" }, + { + "SQL": "select /*+ FORCE_INDEX(t, c_d_e) */ * from t t1", + "Best": "TableReader(Table(t))", + "HasWarn": true, + "Hints": "use_index(@`sel_1` `test`.`t1` )" + }, { "SQL": "select /*+ USE_INDEX(t1, c_d_e) */ * from t t1", "Best": "IndexLookUp(Index(t.c_d_e)[[NULL,+inf]], Table(t))", @@ -123,6 +141,12 @@ "HasWarn": false, "Hints": "use_index(@`sel_1` `test`.`t1` )" }, + { + "SQL": "select /*+ FORCE_INDEX(t1, c_d_e) */ * from t t1", + "Best": "IndexLookUp(Index(t.c_d_e)[[NULL,+inf]], Table(t))", + "HasWarn": false, + "Hints": "use_index(@`sel_1` `test`.`t1` `c_d_e`)" + }, { "SQL": "select /*+ USE_INDEX(t1, c_d_e), USE_INDEX(t2, f) */ * from t t1, t t2 where t1.a = t2.b", "Best": "LeftHashJoin{IndexLookUp(Index(t.c_d_e)[[NULL,+inf]], Table(t))->IndexLookUp(Index(t.f)[[NULL,+inf]], Table(t))}(test.t.a,test.t.b)", @@ -135,18 +159,36 @@ "HasWarn": false, "Hints": "use_index(@`sel_1` `test`.`t1` ), use_index(@`sel_1` `test`.`t2` ), hash_join(@`sel_1` `test`.`t1`)" }, + { + "SQL": "select /*+ FORCE_INDEX(t1, c_d_e), FORCE_INDEX(t2, f) */ * from t t1, t t2 where t1.a = t2.b", + "Best": "LeftHashJoin{IndexLookUp(Index(t.c_d_e)[[NULL,+inf]], Table(t))->IndexLookUp(Index(t.f)[[NULL,+inf]], Table(t))}(test.t.a,test.t.b)", + "HasWarn": false, + "Hints": "use_index(@`sel_1` `test`.`t1` `c_d_e`), use_index(@`sel_1` `test`.`t2` `f`), hash_join(@`sel_1` `test`.`t1`)" + }, { "SQL": "select /*+ USE_INDEX(t, c_d_e, f, g) */ * from t order by f", "Best": "IndexLookUp(Index(t.f)[[NULL,+inf]], Table(t))", "HasWarn": false, "Hints": "use_index(@`sel_1` `test`.`t` `f`)" }, + { + "SQL": "select /*+ FORCE_INDEX(t, c_d_e, f, g) */ * from t order by f", + "Best": "IndexLookUp(Index(t.f)[[NULL,+inf]], Table(t))", + "HasWarn": false, + "Hints": "use_index(@`sel_1` `test`.`t` `f`)" + }, { "SQL": "select /*+ USE_INDEX(t) */ f from t where f > 10", "Best": "TableReader(Table(t)->Sel([gt(test.t.f, 10)]))", "HasWarn": false, "Hints": "use_index(@`sel_1` `test`.`t` )" }, + { + "SQL": "select /*+ FORCE_INDEX(t) */ f from t where f > 10", + "Best": "TableReader(Table(t)->Sel([gt(test.t.f, 10)]))", + "HasWarn": false, + "Hints": "use_index(@`sel_1` `test`.`t` )" + }, { "SQL": "select /*+ USE_INDEX(t, no_such_index) */ * from t", "Best": "TableReader(Table(t))", @@ -159,6 +201,12 @@ "HasWarn": true, "Hints": "use_index(@`sel_1` `test`.`t` )" }, + { + "SQL": "select /*+ FORCE_INDEX(t, no_such_index) */ * from t", + "Best": "TableReader(Table(t))", + "HasWarn": true, + "Hints": "use_index(@`sel_1` `test`.`t` )" + }, { "SQL": "select /*+ USE_INDEX(t, c_d_e), IGNORE_INDEX(t, f) */ c from t order by c", "Best": "IndexReader(Index(t.c_d_e)[[NULL,+inf]])", @@ -182,6 +230,30 @@ "Best": "IndexLookUp(Index(t.f)[[NULL,+inf]], Table(t))->Sort", "HasWarn": false, "Hints": "use_index(@`sel_1` `test`.`t` `f`)" + }, + { + "SQL": "select /*+ FORCE_INDEX(t, c_d_e), IGNORE_INDEX(t, f) */ c from t order by c", + "Best": "IndexReader(Index(t.c_d_e)[[NULL,+inf]])", + "HasWarn": false, + "Hints": "use_index(@`sel_1` `test`.`t` `c_d_e`)" + }, + { + "SQL": "select /*+ FORCE_INDEX(t, f), IGNORE_INDEX(t, f) */ c from t order by c", + "Best": "TableReader(Table(t))->Sort", + "HasWarn": false, + "Hints": "use_index(@`sel_1` `test`.`t` )" + }, + { + "SQL": "select /*+ FORCE_INDEX(t, c_d_e), IGNORE_INDEX(t, c_d_e) */ c from t order by c", + "Best": "TableReader(Table(t))->Sort", + "HasWarn": false, + "Hints": "use_index(@`sel_1` `test`.`t` )" + }, + { + "SQL": "select /*+ FORCE_INDEX(t, c_d_e, f), IGNORE_INDEX(t, c_d_e) */ c from t order by c", + "Best": "IndexLookUp(Index(t.f)[[NULL,+inf]], Table(t))->Sort", + "HasWarn": false, + "Hints": "use_index(@`sel_1` `test`.`t` `f`)" } ] }, From c4424c2920b5680d106181bf982e9412d61e2667 Mon Sep 17 00:00:00 2001 From: crazycs Date: Thu, 1 Jul 2021 18:31:26 +0800 Subject: [PATCH 15/16] util/stmtsummary: discard the plan if it is too long and enlarge the tidb_stmt_summary_max_stmt_count value to 3000 (#25843) --- config/config.go | 2 +- config/config.toml.example | 2 +- util/plancodec/codec.go | 9 ++++++++ util/plancodec/codec_test.go | 6 +++++ util/stmtsummary/statement_summary.go | 5 +++++ util/stmtsummary/statement_summary_test.go | 26 ++++++++++++++++++++++ 6 files changed, 48 insertions(+), 2 deletions(-) diff --git a/config/config.go b/config/config.go index 8b214f694ffcc..13593298b3bac 100644 --- a/config/config.go +++ b/config/config.go @@ -654,7 +654,7 @@ var defaultConf = Config{ StmtSummary: StmtSummary{ Enable: true, EnableInternalQuery: false, - MaxStmtCount: 200, + MaxStmtCount: 3000, MaxSQLLength: 4096, RefreshInterval: 1800, HistorySize: 24, diff --git a/config/config.toml.example b/config/config.toml.example index 909c40138055a..3b5e263c6d824 100644 --- a/config/config.toml.example +++ b/config/config.toml.example @@ -454,7 +454,7 @@ enable = true enable-internal-query = false # max number of statements kept in memory. -max-stmt-count = 200 +max-stmt-count = 3000 # max length of displayed normalized sql and sample sql. max-sql-length = 4096 diff --git a/util/plancodec/codec.go b/util/plancodec/codec.go index e7568493fed94..65fd2c64b899f 100644 --- a/util/plancodec/codec.go +++ b/util/plancodec/codec.go @@ -40,6 +40,12 @@ const ( separatorStr = "\t" ) +var ( + // PlanDiscardedEncoded indicates the discard plan because it is too long + PlanDiscardedEncoded = "[discard]" + planDiscardedDecoded = "(plan discarded because too long)" +) + var decoderPool = sync.Pool{ New: func() interface{} { return &planDecoder{} @@ -87,6 +93,9 @@ type planInfo struct { func (pd *planDecoder) decode(planString string) (string, error) { str, err := decompress(planString) if err != nil { + if planString == PlanDiscardedEncoded { + return planDiscardedDecoded, nil + } return "", err } return pd.buildPlanTree(str) diff --git a/util/plancodec/codec_test.go b/util/plancodec/codec_test.go index 1f98adda4cf99..a3375673c95d4 100644 --- a/util/plancodec/codec_test.go +++ b/util/plancodec/codec_test.go @@ -50,3 +50,9 @@ func (s *testPlanCodecSuite) TestEncodeTaskType(c *C) { _, err = decodeTaskType("1_x") c.Assert(err, NotNil) } + +func (s *testPlanCodecSuite) TestDecodeDiscardPlan(c *C) { + plan, err := DecodePlan(PlanDiscardedEncoded) + c.Assert(err, IsNil) + c.Assert(plan, DeepEquals, planDiscardedDecoded) +} diff --git a/util/stmtsummary/statement_summary.go b/util/stmtsummary/statement_summary.go index dfd0fe2207e95..2cf6625a9d930 100644 --- a/util/stmtsummary/statement_summary.go +++ b/util/stmtsummary/statement_summary.go @@ -638,10 +638,15 @@ func (ssbd *stmtSummaryByDigest) collectHistorySummaries(historySize int) []*stm return ssElements } +var maxEncodedPlanSizeInBytes = 1024 * 1024 + func newStmtSummaryByDigestElement(sei *StmtExecInfo, beginTime int64, intervalSeconds int64) *stmtSummaryByDigestElement { // sampleSQL / authUsers(sampleUser) / samplePlan / prevSQL / indexNames store the values shown at the first time, // because it compacts performance to update every time. samplePlan, planHint := sei.PlanGenerator() + if len(samplePlan) > maxEncodedPlanSizeInBytes { + samplePlan = plancodec.PlanDiscardedEncoded + } ssElement := &stmtSummaryByDigestElement{ beginTime: beginTime, sampleSQL: formatSQL(sei.OriginalSQL), diff --git a/util/stmtsummary/statement_summary_test.go b/util/stmtsummary/statement_summary_test.go index 2227c5c242939..b6bf79f3176f2 100644 --- a/util/stmtsummary/statement_summary_test.go +++ b/util/stmtsummary/statement_summary_test.go @@ -29,6 +29,7 @@ import ( "github.com/pingcap/tidb/sessionctx/stmtctx" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/execdetails" + "github.com/pingcap/tidb/util/plancodec" "github.com/tikv/client-go/v2/util" ) @@ -435,6 +436,31 @@ func (s *testStmtSummarySuite) TestAddStatement(c *C) { c.Assert(s.ssMap.summaryMap.Size(), Equals, 4) _, ok = s.ssMap.summaryMap.Get(key) c.Assert(ok, IsTrue) + + // Test for plan too large + stmtExecInfo7 := stmtExecInfo1 + stmtExecInfo7.PlanDigest = "plan_digest7" + stmtExecInfo7.PlanGenerator = func() (string, string) { + buf := make([]byte, maxEncodedPlanSizeInBytes+1) + for i := range buf { + buf[i] = 'a' + } + return string(buf), "" + } + key = &stmtSummaryByDigestKey{ + schemaName: stmtExecInfo7.SchemaName, + digest: stmtExecInfo7.Digest, + planDigest: stmtExecInfo7.PlanDigest, + } + s.ssMap.AddStatement(stmtExecInfo7) + c.Assert(s.ssMap.summaryMap.Size(), Equals, 5) + v, ok := s.ssMap.summaryMap.Get(key) + c.Assert(ok, IsTrue) + stmt := v.(*stmtSummaryByDigest) + c.Assert(stmt.digest, DeepEquals, key.digest) + e := stmt.history.Back() + ssElement := e.Value.(*stmtSummaryByDigestElement) + c.Assert(ssElement.samplePlan, Equals, plancodec.PlanDiscardedEncoded) } func matchStmtSummaryByDigest(first, second *stmtSummaryByDigest) bool { From 12cc1d038218ad5306d85ee804a860ddb8c81b9e Mon Sep 17 00:00:00 2001 From: Yuanjia Zhang Date: Thu, 1 Jul 2021 18:57:26 +0800 Subject: [PATCH 16/16] executor: skip all test cases related to TiFlash+Partition since they are too slow (#25866) --- executor/tiflash_test.go | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/executor/tiflash_test.go b/executor/tiflash_test.go index 183572634c814..e06237eefdc39 100644 --- a/executor/tiflash_test.go +++ b/executor/tiflash_test.go @@ -260,9 +260,7 @@ func (s *tiflashTestSuite) TestInjectExtraProj(c *C) { } func (s *tiflashTestSuite) TestTiFlashPartitionTableShuffledHashJoin(c *C) { - if israce.RaceEnabled { - c.Skip("exhaustive types test, skip race test") - } + c.Skip("too slow") tk := testkit.NewTestKit(c, s.store) tk.MustExec(`create database tiflash_partition_SHJ`) @@ -336,9 +334,7 @@ func (s *tiflashTestSuite) TestTiFlashPartitionTableShuffledHashJoin(c *C) { } func (s *tiflashTestSuite) TestTiFlashPartitionTableReader(c *C) { - if israce.RaceEnabled { - c.Skip("exhaustive types test, skip race test") - } + c.Skip("too slow") tk := testkit.NewTestKit(c, s.store) tk.MustExec(`create database tiflash_partition_tablereader`) @@ -713,9 +709,7 @@ func (s *tiflashTestSuite) TestTiFlashVirtualColumn(c *C) { } func (s *tiflashTestSuite) TestTiFlashPartitionTableShuffledHashAggregation(c *C) { - if israce.RaceEnabled { - c.Skip("exhaustive types test, skip race test") - } + c.Skip("too slow") tk := testkit.NewTestKit(c, s.store) tk.MustExec("create database tiflash_partition_AGG") @@ -785,9 +779,7 @@ func (s *tiflashTestSuite) TestTiFlashPartitionTableShuffledHashAggregation(c *C } func (s *tiflashTestSuite) TestTiFlashPartitionTableBroadcastJoin(c *C) { - if israce.RaceEnabled { - c.Skip("exhaustive types test, skip race test") - } + c.Skip("too slow") tk := testkit.NewTestKit(c, s.store) tk.MustExec("create database tiflash_partition_BCJ")