diff --git a/errno/errcode.go b/errno/errcode.go index 0cb35c4cacae1..4296db33d3dd5 100644 --- a/errno/errcode.go +++ b/errno/errcode.go @@ -1067,6 +1067,7 @@ const ( ErrPlacementPolicyInUse = 8241 ErrOptOnCacheTable = 8242 ErrHTTPServiceError = 8243 + ErrPartitionColumnStatsMissing = 8244 // TiKV/PD/TiFlash errors. ErrPDServerTimeout = 9001 ErrTiKVServerTimeout = 9002 diff --git a/errno/errname.go b/errno/errname.go index d7925462dea21..3cf46e82fe6b5 100644 --- a/errno/errname.go +++ b/errno/errname.go @@ -1059,9 +1059,10 @@ var MySQLErrName = map[uint16]*mysql.ErrMessage{ ErrInvalidTableSample: mysql.Message("Invalid TABLESAMPLE: %s", nil), - ErrJSONObjectKeyTooLong: mysql.Message("TiDB does not yet support JSON objects with the key length >= 65536", nil), - ErrPartitionStatsMissing: mysql.Message("Build table: %s global-level stats failed due to missing partition-level stats", nil), - ErrNotSupportedWithSem: mysql.Message("Feature '%s' is not supported when security enhanced mode is enabled", nil), + ErrJSONObjectKeyTooLong: mysql.Message("TiDB does not yet support JSON objects with the key length >= 65536", nil), + ErrPartitionStatsMissing: mysql.Message("Build table: %s global-level stats failed due to missing partition-level stats", nil), + ErrPartitionColumnStatsMissing: mysql.Message("Build table: %s global-level stats failed due to missing partition-level column stats, please run analyze table to refresh columns of all partitions", nil), + ErrNotSupportedWithSem: mysql.Message("Feature '%s' is not supported when security enhanced mode is enabled", nil), ErrPlacementPolicyCheck: mysql.Message("Placement policy didn't meet the constraint, reason: %s", nil), ErrMultiStatementDisabled: mysql.Message("client has multi-statement capability disabled. Run SET GLOBAL tidb_multi_statement_mode='ON' after you understand the security risk", nil), diff --git a/errors.toml b/errors.toml index 405b7aab6d34f..d8112d297e6fe 100755 --- a/errors.toml +++ b/errors.toml @@ -2586,6 +2586,11 @@ error = ''' Build table: %s global-level stats failed due to missing partition-level stats ''' +["types:8244"] +error = ''' +Build table: %s global-level stats failed due to missing partition-level column stats, please run analyze table to refresh columns of all partitions +''' + ["variable:1193"] error = ''' Unknown system variable '%-.64s' diff --git a/executor/analyze.go b/executor/analyze.go index 5534b3c032b52..69cbd76ef1d37 100644 --- a/executor/analyze.go +++ b/executor/analyze.go @@ -209,7 +209,7 @@ func (e *AnalyzeExec) Next(ctx context.Context, req *chunk.Chunk) error { } globalStats, err := statsHandle.MergePartitionStats2GlobalStatsByTableID(e.ctx, globalOpts, e.ctx.GetInfoSchema().(infoschema.InfoSchema), globalStatsID.tableID, info.isIndex, info.histIDs) if err != nil { - if types.ErrPartitionStatsMissing.Equal(err) { + if types.ErrPartitionStatsMissing.Equal(err) || types.ErrPartitionColumnStatsMissing.Equal(err) { // When we find some partition-level stats are missing, we need to report warning. e.ctx.GetSessionVars().StmtCtx.AppendWarning(err) continue @@ -230,7 +230,7 @@ func (e *AnalyzeExec) Next(ctx context.Context, req *chunk.Chunk) error { } } } - err = e.saveAnalyzeOptsV2() + err = e.saveV2AnalyzeOpts() if err != nil { e.ctx.GetSessionVars().StmtCtx.AppendWarning(err) } @@ -258,14 +258,22 @@ func finishJobWithLog(sctx sessionctx.Context, job *statistics.AnalyzeJob, analy } } -func (e *AnalyzeExec) saveAnalyzeOptsV2() error { +func (e *AnalyzeExec) saveV2AnalyzeOpts() error { if !variable.PersistAnalyzeOptions.Load() || len(e.OptionsMap) == 0 { return nil } + // only to save table options if dynamic prune mode + dynamicPrune := variable.PartitionPruneMode(e.ctx.GetSessionVars().PartitionPruneMode.Load()) == variable.Dynamic + toSaveMap := make(map[int64]core.V2AnalyzeOptions) + for id, opts := range e.OptionsMap { + if !opts.IsPartition || !dynamicPrune { + toSaveMap[id] = opts + } + } sql := new(strings.Builder) sqlexec.MustFormatSQL(sql, "REPLACE INTO mysql.analyze_options (table_id,sample_num,sample_rate,buckets,topn,column_choice,column_ids) VALUES ") idx := 0 - for _, opts := range e.OptionsMap { + for _, opts := range toSaveMap { sampleNum := opts.RawOpts[ast.AnalyzeOptNumSamples] sampleRate := float64(0) if val, ok := opts.RawOpts[ast.AnalyzeOptSampleRate]; ok { @@ -283,7 +291,7 @@ func (e *AnalyzeExec) saveAnalyzeOptsV2() error { } colIDStrs := strings.Join(colIDs, ",") sqlexec.MustFormatSQL(sql, "(%?,%?,%?,%?,%?,%?,%?)", opts.PhyTableID, sampleNum, sampleRate, buckets, topn, colChoice, colIDStrs) - if idx < len(e.OptionsMap)-1 { + if idx < len(toSaveMap)-1 { sqlexec.MustFormatSQL(sql, ",") } idx += 1 diff --git a/executor/analyze_test.go b/executor/analyze_test.go index df657305edf5b..10c9471110823 100644 --- a/executor/analyze_test.go +++ b/executor/analyze_test.go @@ -1276,11 +1276,10 @@ func TestSavedAnalyzeOptions(t *testing.T) { require.Equal(t, 2, len(col1.Buckets)) col2 := tbl.Columns[tableInfo.Columns[2].ID] require.Equal(t, 2, len(col2.Buckets)) - // The columns are: table_id, sample_num, sample_rate, buckets, topn, column_choice, column_ids. - rs := tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(tbl.PhysicalID, 10)) + rs := tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(tbl.PhysicalID, 10)) require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "2", rs.Rows()[0][3]) - require.Equal(t, "1", rs.Rows()[0][4]) + require.Equal(t, "2", rs.Rows()[0][0]) + require.Equal(t, "1", rs.Rows()[0][1]) // auto-analyze uses the table-level options tk.MustExec("insert into t values (10,10,10)") @@ -1306,15 +1305,14 @@ func TestSavedAnalyzeOptions(t *testing.T) { require.Equal(t, 1, len(col1.TopN.TopN)) col2 = tbl.Columns[tableInfo.Columns[2].ID] require.Less(t, col2.LastUpdateVersion, col0.LastUpdateVersion) // not updated since removed from list - // The columns are: table_id, sample_num, sample_rate, buckets, topn, column_choice, column_ids. - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(tbl.PhysicalID, 10)) + rs = tk.MustQuery("select sample_rate,buckets,topn,column_choice,column_ids from mysql.analyze_options where table_id=" + strconv.FormatInt(tbl.PhysicalID, 10)) require.Equal(t, 1, len(rs.Rows())) + require.Equal(t, "1", rs.Rows()[0][0]) + require.Equal(t, "3", rs.Rows()[0][1]) require.Equal(t, "1", rs.Rows()[0][2]) - require.Equal(t, "3", rs.Rows()[0][3]) - require.Equal(t, "1", rs.Rows()[0][4]) - require.Equal(t, "LIST", rs.Rows()[0][5]) + require.Equal(t, "LIST", rs.Rows()[0][3]) colIDStrs := strings.Join([]string{strconv.FormatInt(tableInfo.Columns[0].ID, 10), strconv.FormatInt(tableInfo.Columns[1].ID, 10)}, ",") - require.Equal(t, colIDStrs, rs.Rows()[0][6]) + require.Equal(t, colIDStrs, rs.Rows()[0][4]) // disable option persistence tk.MustExec("set global tidb_persist_analyze_options = false") @@ -1324,10 +1322,9 @@ func TestSavedAnalyzeOptions(t *testing.T) { require.Greater(t, tbl.Version, lastVersion) col0 = tbl.Columns[tableInfo.Columns[0].ID] require.NotEqual(t, 3, len(col0.Buckets)) - // The columns are: table_id, sample_num, sample_rate, buckets, topn, column_choice, column_ids. - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(tbl.PhysicalID, 10)) + rs = tk.MustQuery("select topn from mysql.analyze_options where table_id=" + strconv.FormatInt(tbl.PhysicalID, 10)) require.Equal(t, 1, len(rs.Rows())) - require.NotEqual(t, "2", rs.Rows()[0][4]) + require.NotEqual(t, "2", rs.Rows()[0][0]) } func TestSavedPartitionAnalyzeOptions(t *testing.T) { @@ -1370,15 +1367,14 @@ PARTITION BY RANGE ( a ) ( p0 := h.GetPartitionStats(tableInfo, pi.Definitions[0].ID) lastVersion := p0.Version require.Equal(t, 3, len(p0.Columns[tableInfo.Columns[0].ID].Buckets)) - // The columns are: table_id, sample_num, sample_rate, buckets, topn, column_choice, column_ids. - rs := tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(p0.PhysicalID, 10)) + rs := tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(p0.PhysicalID, 10)) require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "3", rs.Rows()[0][3]) - require.Equal(t, "1", rs.Rows()[0][4]) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) + require.Equal(t, "3", rs.Rows()[0][0]) + require.Equal(t, "1", rs.Rows()[0][1]) + rs = tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "0", rs.Rows()[0][3]) - require.Equal(t, "-1", rs.Rows()[0][4]) + require.Equal(t, "0", rs.Rows()[0][0]) + require.Equal(t, "-1", rs.Rows()[0][1]) // merge partition & table level options tk.MustExec("analyze table t columns a,b with 0 topn, 2 buckets") @@ -1393,29 +1389,28 @@ PARTITION BY RANGE ( a ) ( // check column c is not analyzed require.Less(t, p0.Columns[tableInfo.Columns[2].ID].LastUpdateVersion, p0.Columns[tableInfo.Columns[0].ID].LastUpdateVersion) require.Less(t, p1.Columns[tableInfo.Columns[2].ID].LastUpdateVersion, p1.Columns[tableInfo.Columns[0].ID].LastUpdateVersion) - // The columns are: table_id, sample_num, sample_rate, buckets, topn, column_choice, column_ids. - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) + rs = tk.MustQuery("select sample_rate,buckets,topn,column_choice,column_ids from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) require.Equal(t, 1, len(rs.Rows())) + require.Equal(t, "0", rs.Rows()[0][0]) + require.Equal(t, "2", rs.Rows()[0][1]) require.Equal(t, "0", rs.Rows()[0][2]) - require.Equal(t, "2", rs.Rows()[0][3]) - require.Equal(t, "0", rs.Rows()[0][4]) - require.Equal(t, "LIST", rs.Rows()[0][5]) + require.Equal(t, "LIST", rs.Rows()[0][3]) colIDStrsAB := strings.Join([]string{strconv.FormatInt(tableInfo.Columns[0].ID, 10), strconv.FormatInt(tableInfo.Columns[1].ID, 10)}, ",") - require.Equal(t, colIDStrsAB, rs.Rows()[0][6]) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(p0.PhysicalID, 10)) + require.Equal(t, colIDStrsAB, rs.Rows()[0][4]) + rs = tk.MustQuery("select sample_rate,buckets,topn,column_choice,column_ids from mysql.analyze_options where table_id=" + strconv.FormatInt(p0.PhysicalID, 10)) require.Equal(t, 1, len(rs.Rows())) + require.Equal(t, "0", rs.Rows()[0][0]) + require.Equal(t, "2", rs.Rows()[0][1]) require.Equal(t, "0", rs.Rows()[0][2]) - require.Equal(t, "2", rs.Rows()[0][3]) - require.Equal(t, "0", rs.Rows()[0][4]) - require.Equal(t, "LIST", rs.Rows()[0][5]) - require.Equal(t, colIDStrsAB, rs.Rows()[0][6]) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(p1.PhysicalID, 10)) + require.Equal(t, "LIST", rs.Rows()[0][3]) + require.Equal(t, colIDStrsAB, rs.Rows()[0][4]) + rs = tk.MustQuery("select sample_rate,buckets,topn,column_choice,column_ids from mysql.analyze_options where table_id=" + strconv.FormatInt(p1.PhysicalID, 10)) require.Equal(t, 1, len(rs.Rows())) + require.Equal(t, "0", rs.Rows()[0][0]) + require.Equal(t, "2", rs.Rows()[0][1]) require.Equal(t, "0", rs.Rows()[0][2]) - require.Equal(t, "2", rs.Rows()[0][3]) - require.Equal(t, "0", rs.Rows()[0][4]) - require.Equal(t, "LIST", rs.Rows()[0][5]) - require.Equal(t, colIDStrsAB, rs.Rows()[0][6]) + require.Equal(t, "LIST", rs.Rows()[0][3]) + require.Equal(t, colIDStrsAB, rs.Rows()[0][4]) // analyze partition only updates this partition, and set different collist tk.MustExec("analyze table t partition p1 columns a,c with 1 buckets") @@ -1432,19 +1427,18 @@ PARTITION BY RANGE ( a ) ( require.Equal(t, 1, len(p1.Columns[tableInfo.Columns[2].ID].Buckets)) require.NotEqual(t, 1, len(p0.Columns[tableInfo.Columns[2].ID].Buckets)) colIDStrsABC := strings.Join([]string{strconv.FormatInt(tableInfo.Columns[0].ID, 10), strconv.FormatInt(tableInfo.Columns[1].ID, 10), strconv.FormatInt(tableInfo.Columns[2].ID, 10)}, ",") - // The columns are: table_id, sample_num, sample_rate, buckets, topn, column_choice, column_ids. - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) + rs = tk.MustQuery("select buckets,column_ids from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "2", rs.Rows()[0][3]) - require.Equal(t, colIDStrsAB, rs.Rows()[0][6]) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(p1.PhysicalID, 10)) + require.Equal(t, "2", rs.Rows()[0][0]) + require.Equal(t, colIDStrsAB, rs.Rows()[0][1]) + rs = tk.MustQuery("select buckets,column_ids from mysql.analyze_options where table_id=" + strconv.FormatInt(p1.PhysicalID, 10)) require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "1", rs.Rows()[0][3]) - require.Equal(t, colIDStrsABC, rs.Rows()[0][6]) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(p0.PhysicalID, 10)) + require.Equal(t, "1", rs.Rows()[0][0]) + require.Equal(t, colIDStrsABC, rs.Rows()[0][1]) + rs = tk.MustQuery("select buckets,column_ids from mysql.analyze_options where table_id=" + strconv.FormatInt(p0.PhysicalID, 10)) require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "2", rs.Rows()[0][3]) - require.Equal(t, colIDStrsAB, rs.Rows()[0][6]) + require.Equal(t, "2", rs.Rows()[0][0]) + require.Equal(t, colIDStrsAB, rs.Rows()[0][1]) // analyze partition without options uses saved partition options tk.MustExec("analyze table t partition p0") @@ -1452,26 +1446,25 @@ PARTITION BY RANGE ( a ) ( require.Greater(t, p0.Version, lastVersion) lastVersion = p0.Version require.Equal(t, 2, len(p0.Columns[tableInfo.Columns[0].ID].Buckets)) - // The columns are: table_id, sample_num, sample_rate, buckets, topn, column_choice, column_ids. - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) + rs = tk.MustQuery("select buckets from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "2", rs.Rows()[0][3]) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(p0.PhysicalID, 10)) + require.Equal(t, "2", rs.Rows()[0][0]) + rs = tk.MustQuery("select buckets from mysql.analyze_options where table_id=" + strconv.FormatInt(p0.PhysicalID, 10)) require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "2", rs.Rows()[0][3]) + require.Equal(t, "2", rs.Rows()[0][0]) // merge options of statement's, partition's and table's tk.MustExec("analyze table t partition p0 with 3 buckets") p0 = h.GetPartitionStats(tableInfo, pi.Definitions[0].ID) require.Greater(t, p0.Version, lastVersion) require.Equal(t, 3, len(p0.Columns[tableInfo.Columns[0].ID].Buckets)) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(p0.PhysicalID, 10)) + rs = tk.MustQuery("select sample_rate,buckets,topn,column_choice,column_ids from mysql.analyze_options where table_id=" + strconv.FormatInt(p0.PhysicalID, 10)) require.Equal(t, 1, len(rs.Rows())) + require.Equal(t, "0", rs.Rows()[0][0]) + require.Equal(t, "3", rs.Rows()[0][1]) require.Equal(t, "0", rs.Rows()[0][2]) - require.Equal(t, "3", rs.Rows()[0][3]) - require.Equal(t, "0", rs.Rows()[0][4]) - require.Equal(t, "LIST", rs.Rows()[0][5]) - require.Equal(t, colIDStrsAB, rs.Rows()[0][6]) + require.Equal(t, "LIST", rs.Rows()[0][3]) + require.Equal(t, colIDStrsAB, rs.Rows()[0][4]) // add new partitions, use table options as default tk.MustExec("ALTER TABLE t ADD PARTITION (PARTITION p2 VALUES LESS THAN (30))") @@ -1484,20 +1477,20 @@ PARTITION BY RANGE ( a ) ( pi = tableInfo.GetPartitionInfo() p2 := h.GetPartitionStats(tableInfo, pi.Definitions[2].ID) require.Equal(t, 2, len(p2.Columns[tableInfo.Columns[0].ID].Buckets)) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(p2.PhysicalID, 10)) + rs = tk.MustQuery("select sample_rate,buckets,topn,column_choice,column_ids from mysql.analyze_options where table_id=" + strconv.FormatInt(p2.PhysicalID, 10)) require.Equal(t, 1, len(rs.Rows())) + require.Equal(t, "0", rs.Rows()[0][0]) + require.Equal(t, "2", rs.Rows()[0][1]) require.Equal(t, "0", rs.Rows()[0][2]) - require.Equal(t, "2", rs.Rows()[0][3]) - require.Equal(t, "0", rs.Rows()[0][4]) - require.Equal(t, "LIST", rs.Rows()[0][5]) - require.Equal(t, colIDStrsAB, rs.Rows()[0][6]) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) + require.Equal(t, "LIST", rs.Rows()[0][3]) + require.Equal(t, colIDStrsAB, rs.Rows()[0][4]) + rs = tk.MustQuery("select sample_rate,buckets,topn,column_choice,column_ids from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) require.Equal(t, 1, len(rs.Rows())) + require.Equal(t, "0", rs.Rows()[0][0]) + require.Equal(t, "2", rs.Rows()[0][1]) require.Equal(t, "0", rs.Rows()[0][2]) - require.Equal(t, "2", rs.Rows()[0][3]) - require.Equal(t, "0", rs.Rows()[0][4]) - require.Equal(t, "LIST", rs.Rows()[0][5]) - require.Equal(t, colIDStrsAB, rs.Rows()[0][6]) + require.Equal(t, "LIST", rs.Rows()[0][3]) + require.Equal(t, colIDStrsAB, rs.Rows()[0][4]) // set analyze version back to 1, will not use persisted tk.MustExec("set @@session.tidb_analyze_version = 1") @@ -1512,15 +1505,15 @@ PARTITION BY RANGE ( a ) ( tk.MustExec("analyze table t") colIDStrsA := strings.Join([]string{strconv.FormatInt(tableInfo.Columns[0].ID, 10)}, ",") colIDStrsAC := strings.Join([]string{strconv.FormatInt(tableInfo.Columns[0].ID, 10), strconv.FormatInt(tableInfo.Columns[2].ID, 10)}, ",") - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) + rs = tk.MustQuery("select column_ids from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, colIDStrsA, rs.Rows()[0][6]) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(p0.PhysicalID, 10)) + require.Equal(t, colIDStrsA, rs.Rows()[0][0]) + rs = tk.MustQuery("select column_ids from mysql.analyze_options where table_id=" + strconv.FormatInt(p0.PhysicalID, 10)) require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, colIDStrsA, rs.Rows()[0][6]) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(p1.PhysicalID, 10)) + require.Equal(t, colIDStrsA, rs.Rows()[0][0]) + rs = tk.MustQuery("select column_ids from mysql.analyze_options where table_id=" + strconv.FormatInt(p1.PhysicalID, 10)) require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, colIDStrsAC, rs.Rows()[0][6]) + require.Equal(t, colIDStrsAC, rs.Rows()[0][0]) // drop partition tk.MustExec("alter table t drop partition p1") @@ -1543,124 +1536,6 @@ PARTITION BY RANGE ( a ) ( require.Equal(t, 0, len(rs.Rows())) } -func TestSavedPartitionAnalyzeOptionsDynamic(t *testing.T) { - store, dom, clean := testkit.CreateMockStoreAndDomain(t) - defer clean() - tk := testkit.NewTestKit(t, store) - originalVal := tk.MustQuery("select @@tidb_persist_analyze_options").Rows()[0][0].(string) - defer func() { - tk.MustExec(fmt.Sprintf("set global tidb_persist_analyze_options = %v", originalVal)) - }() - tk.MustExec("set global tidb_persist_analyze_options = true") - - tk.MustExec("use test") - tk.MustExec("set @@session.tidb_analyze_version = 2") - tk.MustExec("set @@session.tidb_partition_prune_mode = 'dynamic'") - createTable := `CREATE TABLE t (a int, b int, c varchar(10), primary key(a), index idx(b)) -PARTITION BY RANGE ( a ) ( - PARTITION p0 VALUES LESS THAN (10), - PARTITION p1 VALUES LESS THAN (20) -)` - tk.MustExec(createTable) - tk.MustExec("insert into t values (1,1,1),(2,1,2),(3,1,3),(4,1,4),(5,1,5),(6,1,6),(7,7,7),(8,8,8),(9,9,9),(10,10,10),(11,11,11),(12,12,12),(13,13,13),(14,14,14)") - - h := dom.StatsHandle() - oriLease := h.Lease() - h.SetLease(1) - defer func() { - h.SetLease(oriLease) - }() - is := dom.InfoSchema() - table, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) - require.NoError(t, err) - tableInfo := table.Meta() - pi := tableInfo.GetPartitionInfo() - require.NotNil(t, pi) - - // analyze partition only sets options of partition - tk.MustExec("analyze table t partition p0 with 1 topn, 3 buckets") - tk.MustQuery("select * from t where b > 1 and c > 1") - require.NoError(t, h.LoadNeededHistograms()) - tbl := h.GetTableStats(tableInfo) - lastVersion := tbl.Version - // no global stats since partition stats missing - require.Equal(t, 0, len(tbl.Columns[tableInfo.Columns[0].ID].Buckets)) - // The columns are: table_id, sample_num, sample_rate, buckets, topn, column_choice, column_ids. - rs := tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[0].ID, 10)) - require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "3", rs.Rows()[0][3]) - require.Equal(t, "1", rs.Rows()[0][4]) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) - require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "0", rs.Rows()[0][3]) - require.Equal(t, "-1", rs.Rows()[0][4]) - - // merge partition & table level options - tk.MustExec("analyze table t columns a,b with 0 topn, 2 buckets") - tk.MustQuery("select * from t where b > 1 and c > 1") - require.NoError(t, h.LoadNeededHistograms()) - tbl = h.GetTableStats(tableInfo) - require.Greater(t, tbl.Version, lastVersion) - lastVersion = tbl.Version - p0 := h.GetPartitionStats(tableInfo, pi.Definitions[0].ID) - p1 := h.GetPartitionStats(tableInfo, pi.Definitions[1].ID) - require.Equal(t, 2, len(p0.Columns[tableInfo.Columns[0].ID].Buckets)) - require.Equal(t, 2, len(p1.Columns[tableInfo.Columns[0].ID].Buckets)) - // merged global stats - require.Equal(t, 2, len(tbl.Columns[tableInfo.Columns[0].ID].Buckets)) - // check column c is not analyzed - require.Less(t, tbl.Columns[tableInfo.Columns[2].ID].LastUpdateVersion, tbl.Columns[tableInfo.Columns[0].ID].LastUpdateVersion) - require.Less(t, p0.Columns[tableInfo.Columns[2].ID].LastUpdateVersion, p0.Columns[tableInfo.Columns[0].ID].LastUpdateVersion) - require.Less(t, p1.Columns[tableInfo.Columns[2].ID].LastUpdateVersion, p1.Columns[tableInfo.Columns[0].ID].LastUpdateVersion) - // The columns are: table_id, sample_num, sample_rate, buckets, topn, column_choice, column_ids. - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(tbl.PhysicalID, 10)) - require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "0", rs.Rows()[0][2]) - require.Equal(t, "2", rs.Rows()[0][3]) - require.Equal(t, "0", rs.Rows()[0][4]) - require.Equal(t, "LIST", rs.Rows()[0][5]) - colIDStrs := strings.Join([]string{strconv.FormatInt(tableInfo.Columns[0].ID, 10), strconv.FormatInt(tableInfo.Columns[1].ID, 10)}, ",") - require.Equal(t, colIDStrs, rs.Rows()[0][6]) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(p0.PhysicalID, 10)) - require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "0", rs.Rows()[0][2]) - require.Equal(t, "2", rs.Rows()[0][3]) - require.Equal(t, "0", rs.Rows()[0][4]) - require.Equal(t, "LIST", rs.Rows()[0][5]) - require.Equal(t, colIDStrs, rs.Rows()[0][6]) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(p1.PhysicalID, 10)) - require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "0", rs.Rows()[0][2]) - require.Equal(t, "2", rs.Rows()[0][3]) - require.Equal(t, "0", rs.Rows()[0][4]) - require.Equal(t, "LIST", rs.Rows()[0][5]) - require.Equal(t, colIDStrs, rs.Rows()[0][6]) - - // analyze partition only updates this partition, and set different collist - tk.MustExec("analyze table t partition p1 columns a,c with 1 buckets") - tk.MustQuery("select * from t where b > 1 and c > 1") - require.NoError(t, h.LoadNeededHistograms()) - tbl = h.GetTableStats(tableInfo) - require.Greater(t, tbl.Version, lastVersion) - p0 = h.GetPartitionStats(tableInfo, pi.Definitions[0].ID) - p1 = h.GetPartitionStats(tableInfo, pi.Definitions[1].ID) - require.Equal(t, 1, len(p1.Columns[tableInfo.Columns[0].ID].Buckets)) - require.Equal(t, 2, len(tbl.Columns[tableInfo.Columns[0].ID].Buckets)) - // check column c is analyzed, but only global stats loaded - require.Equal(t, 0, len(p1.Columns[tableInfo.Columns[2].ID].Buckets)) - require.Equal(t, 2, len(tbl.Columns[tableInfo.Columns[2].ID].Buckets)) - // The columns are: table_id, sample_num, sample_rate, buckets, topn, column_choice, column_ids. - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(tbl.PhysicalID, 10)) - require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "2", rs.Rows()[0][3]) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(p1.PhysicalID, 10)) - require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "1", rs.Rows()[0][3]) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(p0.PhysicalID, 10)) - require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "2", rs.Rows()[0][3]) -} - func TestSavedAnalyzeOptionsForMultipleTables(t *testing.T) { store, dom, clean := testkit.CreateMockStoreAndDomain(t) defer clean() @@ -1702,15 +1577,14 @@ func TestSavedAnalyzeOptionsForMultipleTables(t *testing.T) { tbl2Col0 := tblStats2.Columns[tableInfo2.Columns[0].ID] require.Equal(t, 3, len(tbl1Col0.Buckets)) require.Equal(t, 2, len(tbl2Col0.Buckets)) - // The columns are: table_id, sample_num, sample_rate, buckets, topn, column_choice, column_ids. - rs := tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo1.ID, 10)) + rs := tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo1.ID, 10)) require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "3", rs.Rows()[0][3]) - require.Equal(t, "2", rs.Rows()[0][4]) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo2.ID, 10)) + require.Equal(t, "3", rs.Rows()[0][0]) + require.Equal(t, "2", rs.Rows()[0][1]) + rs = tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo2.ID, 10)) require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "2", rs.Rows()[0][3]) - require.Equal(t, "2", rs.Rows()[0][4]) + require.Equal(t, "2", rs.Rows()[0][0]) + require.Equal(t, "2", rs.Rows()[0][1]) } func TestSavedAnalyzeColumnOptions(t *testing.T) { @@ -2986,3 +2860,424 @@ func TestShowAanalyzeStatusJobInfo(t *testing.T) { tk.MustExec("analyze table t") checkJobInfo("analyze table columns a, b, d with 3 buckets, 1 topn, 1 samplerate") } + +func TestAnalyzePartitionTableWithDynamicMode(t *testing.T) { + store, dom, clean := testkit.CreateMockStoreAndDomain(t) + defer clean() + tk := testkit.NewTestKit(t, store) + originalVal := tk.MustQuery("select @@tidb_persist_analyze_options").Rows()[0][0].(string) + defer func() { + tk.MustExec(fmt.Sprintf("set global tidb_persist_analyze_options = %v", originalVal)) + }() + tk.MustExec("set global tidb_persist_analyze_options = true") + + tk.MustExec("use test") + tk.MustExec("set @@session.tidb_analyze_version = 2") + tk.MustExec("set @@session.tidb_partition_prune_mode = 'dynamic'") + createTable := `CREATE TABLE t (a int, b int, c varchar(10), d int, primary key(a), index idx(b)) +PARTITION BY RANGE ( a ) ( + PARTITION p0 VALUES LESS THAN (10), + PARTITION p1 VALUES LESS THAN (20) +)` + tk.MustExec(createTable) + tk.MustExec("insert into t values (1,1,1,1),(2,1,2,2),(3,1,3,3),(4,1,4,4),(5,1,5,5),(6,1,6,6),(7,7,7,7),(8,8,8,8),(9,9,9,9)") + tk.MustExec("insert into t values (10,10,10,10),(11,11,11,11),(12,12,12,12),(13,13,13,13),(14,14,14,14)") + h := dom.StatsHandle() + oriLease := h.Lease() + h.SetLease(1) + defer func() { + h.SetLease(oriLease) + }() + is := dom.InfoSchema() + table, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) + require.NoError(t, err) + tableInfo := table.Meta() + pi := tableInfo.GetPartitionInfo() + require.NotNil(t, pi) + + // analyze table only sets table options and gen globalStats + tk.MustExec("analyze table t columns a,c with 1 topn, 3 buckets") + tk.MustQuery("select * from t where b > 1 and c > 1") + require.NoError(t, h.LoadNeededHistograms()) + tbl := h.GetTableStats(tableInfo) + lastVersion := tbl.Version + // both globalStats and partition stats generated and options saved for column a,c + require.Equal(t, 3, len(tbl.Columns[tableInfo.Columns[0].ID].Buckets)) + require.Equal(t, 1, len(tbl.Columns[tableInfo.Columns[0].ID].TopN.TopN)) + require.Equal(t, 3, len(tbl.Columns[tableInfo.Columns[2].ID].Buckets)) + require.Equal(t, 1, len(tbl.Columns[tableInfo.Columns[2].ID].TopN.TopN)) + rs := tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[0].ID, 10)) + require.Equal(t, 0, len(rs.Rows())) + rs = tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[1].ID, 10)) + require.Equal(t, 0, len(rs.Rows())) + rs = tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) + require.Equal(t, 1, len(rs.Rows())) + require.Equal(t, "3", rs.Rows()[0][0]) + require.Equal(t, "1", rs.Rows()[0][1]) + + // analyze table with persisted table-level options + tk.MustExec("analyze table t") + tk.MustQuery("select * from t where b > 1 and c > 1") + require.NoError(t, h.LoadNeededHistograms()) + tbl = h.GetTableStats(tableInfo) + require.Greater(t, tbl.Version, lastVersion) + lastVersion = tbl.Version + require.Equal(t, 3, len(tbl.Columns[tableInfo.Columns[0].ID].Buckets)) + require.Equal(t, 1, len(tbl.Columns[tableInfo.Columns[0].ID].TopN.TopN)) + require.Equal(t, 3, len(tbl.Columns[tableInfo.Columns[2].ID].Buckets)) + require.Equal(t, 1, len(tbl.Columns[tableInfo.Columns[2].ID].TopN.TopN)) + rs = tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[0].ID, 10)) + require.Equal(t, 0, len(rs.Rows())) + rs = tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[1].ID, 10)) + require.Equal(t, 0, len(rs.Rows())) + rs = tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) + require.Equal(t, 1, len(rs.Rows())) + require.Equal(t, "3", rs.Rows()[0][0]) + require.Equal(t, "1", rs.Rows()[0][1]) + + // analyze table with merged table-level options + tk.MustExec("analyze table t with 2 topn, 2 buckets") + tk.MustQuery("select * from t where b > 1 and c > 1") + require.NoError(t, h.LoadNeededHistograms()) + tbl = h.GetTableStats(tableInfo) + require.Greater(t, tbl.Version, lastVersion) + require.Equal(t, 2, len(tbl.Columns[tableInfo.Columns[0].ID].Buckets)) + require.Equal(t, 2, len(tbl.Columns[tableInfo.Columns[0].ID].TopN.TopN)) + require.Equal(t, 2, len(tbl.Columns[tableInfo.Columns[2].ID].Buckets)) + require.Equal(t, 2, len(tbl.Columns[tableInfo.Columns[2].ID].TopN.TopN)) + rs = tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[0].ID, 10)) + require.Equal(t, 0, len(rs.Rows())) + rs = tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[1].ID, 10)) + require.Equal(t, 0, len(rs.Rows())) + rs = tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) + require.Equal(t, 1, len(rs.Rows())) + require.Equal(t, "2", rs.Rows()[0][0]) + require.Equal(t, "2", rs.Rows()[0][1]) +} + +func TestAnalyzePartitionTableStaticToDynamic(t *testing.T) { + store, dom, clean := testkit.CreateMockStoreAndDomain(t) + defer clean() + tk := testkit.NewTestKit(t, store) + originalVal := tk.MustQuery("select @@tidb_persist_analyze_options").Rows()[0][0].(string) + defer func() { + tk.MustExec(fmt.Sprintf("set global tidb_persist_analyze_options = %v", originalVal)) + }() + tk.MustExec("set global tidb_persist_analyze_options = true") + + tk.MustExec("use test") + tk.MustExec("set @@session.tidb_analyze_version = 2") + tk.MustExec("set @@session.tidb_partition_prune_mode = 'static'") + createTable := `CREATE TABLE t (a int, b int, c varchar(10), d int, primary key(a), index idx(b)) +PARTITION BY RANGE ( a ) ( + PARTITION p0 VALUES LESS THAN (10), + PARTITION p1 VALUES LESS THAN (20) +)` + tk.MustExec(createTable) + tk.MustExec("insert into t values (1,1,1,1),(2,1,2,2),(3,1,3,3),(4,1,4,4),(5,1,5,5),(6,1,6,6),(7,7,7,7),(8,8,8,8),(9,9,9,9)") + tk.MustExec("insert into t values (10,10,10,10),(11,11,11,11),(12,12,12,12),(13,13,13,13),(14,14,14,14)") + h := dom.StatsHandle() + oriLease := h.Lease() + h.SetLease(1) + defer func() { + h.SetLease(oriLease) + }() + is := dom.InfoSchema() + table, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) + require.NoError(t, err) + tableInfo := table.Meta() + pi := tableInfo.GetPartitionInfo() + require.NotNil(t, pi) + + // analyze partition under static mode with options + tk.MustExec("analyze table t partition p0 columns a,c with 1 topn, 3 buckets") + tk.MustQuery("select * from t where b > 1 and c > 1") + require.NoError(t, h.LoadNeededHistograms()) + tbl := h.GetTableStats(tableInfo) + p0 := h.GetPartitionStats(tableInfo, pi.Definitions[0].ID) + p1 := h.GetPartitionStats(tableInfo, pi.Definitions[1].ID) + lastVersion := tbl.Version + require.Equal(t, 3, len(p0.Columns[tableInfo.Columns[0].ID].Buckets)) + require.Equal(t, 3, len(p0.Columns[tableInfo.Columns[2].ID].Buckets)) + require.Equal(t, 0, len(p1.Columns[tableInfo.Columns[0].ID].Buckets)) + require.Equal(t, 0, len(tbl.Columns[tableInfo.Columns[0].ID].Buckets)) + rs := tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[0].ID, 10)) + require.Equal(t, 1, len(rs.Rows())) + require.Equal(t, "3", rs.Rows()[0][0]) + require.Equal(t, "1", rs.Rows()[0][1]) + rs = tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[1].ID, 10)) + require.Equal(t, 0, len(rs.Rows())) + // The columns are: table_id, sample_num, sample_rate, buckets, topn, column_choice, column_ids. + rs = tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) + require.Equal(t, 1, len(rs.Rows())) + require.Equal(t, "0", rs.Rows()[0][0]) + + tk.MustExec("set @@session.tidb_partition_prune_mode = 'dynamic'") + + // analyze table in dynamic mode will ignore partition-level options and use default + tk.MustExec("analyze table t") + tk.MustQuery("select * from t where b > 1 and c > 1") + require.NoError(t, h.LoadNeededHistograms()) + tbl = h.GetTableStats(tableInfo) + require.Greater(t, tbl.Version, lastVersion) + lastVersion = tbl.Version + p0 = h.GetPartitionStats(tableInfo, pi.Definitions[0].ID) + p1 = h.GetPartitionStats(tableInfo, pi.Definitions[1].ID) + require.NotEqual(t, 3, len(p0.Columns[tableInfo.Columns[0].ID].Buckets)) + require.Equal(t, len(tbl.Columns[tableInfo.Columns[0].ID].Buckets), len(p0.Columns[tableInfo.Columns[0].ID].Buckets)) + require.Equal(t, len(tbl.Columns[tableInfo.Columns[0].ID].Buckets), len(p1.Columns[tableInfo.Columns[0].ID].Buckets)) + rs = tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[0].ID, 10)) + require.Equal(t, 1, len(rs.Rows())) + require.Equal(t, "3", rs.Rows()[0][0]) + require.Equal(t, "1", rs.Rows()[0][1]) + rs = tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[1].ID, 10)) + require.Equal(t, 0, len(rs.Rows())) + rs = tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) + require.Equal(t, 1, len(rs.Rows())) + require.Equal(t, "0", rs.Rows()[0][0]) + + // analyze table under dynamic mode with specified options with old partition-level options + tk.MustExec("analyze table t columns b,d with 2 topn, 2 buckets") + tk.MustQuery("select * from t where b > 1 and d > 1") + require.NoError(t, h.LoadNeededHistograms()) + tbl = h.GetTableStats(tableInfo) + require.Greater(t, tbl.Version, lastVersion) + lastVersion = tbl.Version + require.Equal(t, 2, len(tbl.Columns[tableInfo.Columns[1].ID].Buckets)) + require.Equal(t, 2, len(tbl.Columns[tableInfo.Columns[3].ID].Buckets)) + rs = tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[0].ID, 10)) + require.Equal(t, 1, len(rs.Rows())) + require.Equal(t, "3", rs.Rows()[0][0]) + require.Equal(t, "1", rs.Rows()[0][1]) + rs = tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[1].ID, 10)) + require.Equal(t, 0, len(rs.Rows())) + rs = tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) + require.Equal(t, 1, len(rs.Rows())) + require.Equal(t, "2", rs.Rows()[0][0]) + require.Equal(t, "2", rs.Rows()[0][1]) + + // analyze table under dynamic mode without options with old table-level & partition-level options + tk.MustExec("analyze table t") + tk.MustQuery("select * from t where b > 1 and d > 1") + require.NoError(t, h.LoadNeededHistograms()) + tbl = h.GetTableStats(tableInfo) + require.Greater(t, tbl.Version, lastVersion) + lastVersion = tbl.Version + require.Equal(t, 2, len(tbl.Columns[tableInfo.Columns[3].ID].Buckets)) + require.Equal(t, 2, len(tbl.Columns[tableInfo.Columns[3].ID].TopN.TopN)) + + // analyze table under dynamic mode with specified options with old table-level & partition-level options + tk.MustExec("analyze table t with 1 topn") + tk.MustQuery("select * from t where b > 1 and d > 1") + require.NoError(t, h.LoadNeededHistograms()) + tbl = h.GetTableStats(tableInfo) + require.Greater(t, tbl.Version, lastVersion) + require.Equal(t, 2, len(tbl.Columns[tableInfo.Columns[1].ID].Buckets)) + require.Equal(t, 2, len(tbl.Columns[tableInfo.Columns[3].ID].Buckets)) + require.Equal(t, 1, len(tbl.Columns[tableInfo.Columns[1].ID].TopN.TopN)) + require.Equal(t, 1, len(tbl.Columns[tableInfo.Columns[3].ID].TopN.TopN)) + rs = tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[0].ID, 10)) + require.Equal(t, 1, len(rs.Rows())) + require.Equal(t, "3", rs.Rows()[0][0]) + require.Equal(t, "1", rs.Rows()[0][1]) + rs = tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[1].ID, 10)) + require.Equal(t, 0, len(rs.Rows())) + rs = tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) + require.Equal(t, 1, len(rs.Rows())) + require.Equal(t, "2", rs.Rows()[0][0]) + require.Equal(t, "1", rs.Rows()[0][1]) +} + +func TestAnalyzePartitionUnderDynamic(t *testing.T) { + store, dom, clean := testkit.CreateMockStoreAndDomain(t) + defer clean() + tk := testkit.NewTestKit(t, store) + originalVal := tk.MustQuery("select @@tidb_persist_analyze_options").Rows()[0][0].(string) + defer func() { + tk.MustExec(fmt.Sprintf("set global tidb_persist_analyze_options = %v", originalVal)) + }() + tk.MustExec("set global tidb_persist_analyze_options = true") + + tk.MustExec("use test") + tk.MustExec("set @@session.tidb_analyze_version = 2") + tk.MustExec("set @@session.tidb_partition_prune_mode = 'dynamic'") + createTable := `CREATE TABLE t (a int, b int, c varchar(10), d int, primary key(a), index idx(b)) +PARTITION BY RANGE ( a ) ( + PARTITION p0 VALUES LESS THAN (10), + PARTITION p1 VALUES LESS THAN (20) +)` + tk.MustExec(createTable) + tk.MustExec("insert into t values (1,1,1,1),(2,1,2,2),(3,1,3,3),(4,1,4,4),(5,1,5,5),(6,1,6,6),(7,7,7,7),(8,8,8,8),(9,9,9,9)") + tk.MustExec("insert into t values (10,10,10,10),(11,11,11,11),(12,12,12,12),(13,13,13,13),(14,14,14,14)") + h := dom.StatsHandle() + oriLease := h.Lease() + h.SetLease(1) + defer func() { + h.SetLease(oriLease) + }() + is := dom.InfoSchema() + table, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) + require.NoError(t, err) + tableInfo := table.Meta() + pi := tableInfo.GetPartitionInfo() + require.NotNil(t, pi) + + // analyze partition with options under dynamic mode + tk.MustExec("analyze table t partition p0 columns a,b,c with 1 topn, 3 buckets") + tk.MustQuery("show warnings").Sort().Check(testkit.Rows( + "Note 1105 Analyze use auto adjusted sample rate 1.000000 for table test.t's partition p0", + "Warning 1105 Ignore columns and options when analyze partition in dynamic mode", + "Warning 8131 Build table: `t` global-level stats failed due to missing partition-level stats", + "Warning 8131 Build table: `t` index: `idx` global-level stats failed due to missing partition-level stats", + )) + tk.MustQuery("select * from t where a > 1 and b > 1 and c > 1 and d > 1") + require.NoError(t, h.LoadNeededHistograms()) + tbl := h.GetTableStats(tableInfo) + lastVersion := tbl.Version + require.NotEqual(t, 3, len(tbl.Columns[tableInfo.Columns[2].ID].Buckets)) + require.NotEqual(t, 3, len(tbl.Columns[tableInfo.Columns[3].ID].Buckets)) + + tk.MustExec("analyze table t partition p0") + tk.MustQuery("show warnings").Sort().Check(testkit.Rows( + "Note 1105 Analyze use auto adjusted sample rate 1.000000 for table test.t's partition p0", + "Warning 8131 Build table: `t` global-level stats failed due to missing partition-level stats", + "Warning 8131 Build table: `t` index: `idx` global-level stats failed due to missing partition-level stats", + )) + tbl = h.GetTableStats(tableInfo) + require.Equal(t, tbl.Version, lastVersion) // global stats not updated +} + +func TestAnalyzePartitionStaticToDynamic(t *testing.T) { + store, dom, clean := testkit.CreateMockStoreAndDomain(t) + defer clean() + tk := testkit.NewTestKit(t, store) + originalVal := tk.MustQuery("select @@tidb_persist_analyze_options").Rows()[0][0].(string) + defer func() { + tk.MustExec(fmt.Sprintf("set global tidb_persist_analyze_options = %v", originalVal)) + }() + + tk.MustExec("use test") + tk.MustExec("set @@session.tidb_analyze_version = 2") + createTable := `CREATE TABLE t (a int, b int, c varchar(10), d int, primary key(a), index idx(b)) +PARTITION BY RANGE ( a ) ( + PARTITION p0 VALUES LESS THAN (10), + PARTITION p1 VALUES LESS THAN (20) +)` + tk.MustExec(createTable) + tk.MustExec("insert into t values (1,1,1,1),(2,1,2,2),(3,1,3,3),(4,1,4,4),(5,1,5,5),(6,1,6,6),(7,7,7,7),(8,8,8,8),(9,9,9,9)") + tk.MustExec("insert into t values (10,10,10,10),(11,11,11,11),(12,12,12,12),(13,13,13,13),(14,14,14,14)") + h := dom.StatsHandle() + oriLease := h.Lease() + h.SetLease(1) + defer func() { + h.SetLease(oriLease) + }() + is := dom.InfoSchema() + table, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) + require.NoError(t, err) + tableInfo := table.Meta() + pi := tableInfo.GetPartitionInfo() + require.NotNil(t, pi) + + // generate old partition stats + tk.MustExec("set global tidb_persist_analyze_options = false") + tk.MustExec("set @@session.tidb_partition_prune_mode = 'static'") + tk.MustExec("analyze table t partition p0 columns a,c with 1 topn, 3 buckets") + tk.MustQuery("select * from t where a > 1 and b > 1 and c > 1 and d > 1") + require.NoError(t, h.LoadNeededHistograms()) + p0 := h.GetPartitionStats(tableInfo, pi.Definitions[0].ID) + require.Equal(t, 3, len(p0.Columns[tableInfo.Columns[2].ID].Buckets)) + + // analyze partition with existing stats of other partitions under dynamic + tk.MustExec("set @@session.tidb_partition_prune_mode = 'dynamic'") + tk.MustExec("analyze table t partition p1 columns a,b,d with 1 topn, 3 buckets") + tk.MustQuery("show warnings").Sort().Check(testkit.Rows( + "Note 1105 Analyze use auto adjusted sample rate 1.000000 for table test.t's partition p1", + "Warning 8244 Build table: `t` column: `d` global-level stats failed due to missing partition-level column stats, please run analyze table to refresh columns of all partitions", + )) + + // analyze partition with existing table-level options and existing partition stats under dynamic + tk.MustExec("insert into mysql.analyze_options values (?,?,?,?,?,?,?)", tableInfo.ID, 0, 0, 2, 2, "DEFAULT", "") + tk.MustExec("set global tidb_persist_analyze_options = true") + tk.MustExec("analyze table t partition p1 columns a,b,d with 1 topn, 3 buckets") + tk.MustQuery("show warnings").Sort().Check(testkit.Rows( + "Note 1105 Analyze use auto adjusted sample rate 1.000000 for table test.t's partition p1", + "Warning 1105 Ignore columns and options when analyze partition in dynamic mode", + "Warning 8244 Build table: `t` column: `d` global-level stats failed due to missing partition-level column stats, please run analyze table to refresh columns of all partitions", + )) + + // analyze partition with existing table-level & partition-level options and existing partition stats under dynamic + tk.MustExec("insert into mysql.analyze_options values (?,?,?,?,?,?,?)", pi.Definitions[1].ID, 0, 0, 1, 1, "DEFAULT", "") + tk.MustExec("analyze table t partition p1 columns a,b,d with 1 topn, 3 buckets") + tk.MustQuery("show warnings").Sort().Check(testkit.Rows( + "Note 1105 Analyze use auto adjusted sample rate 1.000000 for table test.t's partition p1", + "Warning 1105 Ignore columns and options when analyze partition in dynamic mode", + "Warning 8244 Build table: `t` column: `d` global-level stats failed due to missing partition-level column stats, please run analyze table to refresh columns of all partitions", + )) + tk.MustQuery("select * from t where a > 1 and b > 1 and c > 1 and d > 1") + require.NoError(t, h.LoadNeededHistograms()) + tbl := h.GetTableStats(tableInfo) + require.Equal(t, 0, len(tbl.Columns)) + + // ignore both p0's 3 buckets, persisted-partition-options' 1 bucket, just use table-level 2 buckets + tk.MustExec("analyze table t partition p0") + tk.MustQuery("select * from t where a > 1 and b > 1 and c > 1 and d > 1") + require.NoError(t, h.LoadNeededHistograms()) + tbl = h.GetTableStats(tableInfo) + require.Equal(t, 2, len(tbl.Columns[tableInfo.Columns[2].ID].Buckets)) +} + +func TestAnalyzePartitionUnderV1Dynamic(t *testing.T) { + store, dom, clean := testkit.CreateMockStoreAndDomain(t) + defer clean() + tk := testkit.NewTestKit(t, store) + originalVal := tk.MustQuery("select @@tidb_persist_analyze_options").Rows()[0][0].(string) + defer func() { + tk.MustExec(fmt.Sprintf("set global tidb_persist_analyze_options = %v", originalVal)) + }() + + tk.MustExec("use test") + tk.MustExec("set @@session.tidb_analyze_version = 1") + tk.MustExec("set @@session.tidb_partition_prune_mode = 'dynamic'") + createTable := `CREATE TABLE t (a int, b int, c varchar(10), d int, primary key(a), index idx(b)) +PARTITION BY RANGE ( a ) ( + PARTITION p0 VALUES LESS THAN (10), + PARTITION p1 VALUES LESS THAN (20) +)` + tk.MustExec(createTable) + tk.MustExec("insert into t values (1,1,1,1),(2,1,2,2),(3,1,3,3),(4,1,4,4),(5,1,5,5),(6,1,6,6),(7,7,7,7),(8,8,8,8),(9,9,9,9)") + tk.MustExec("insert into t values (10,10,10,10),(11,11,11,11),(12,12,12,12),(13,13,13,13),(14,14,14,14)") + h := dom.StatsHandle() + oriLease := h.Lease() + h.SetLease(1) + defer func() { + h.SetLease(oriLease) + }() + is := dom.InfoSchema() + table, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) + require.NoError(t, err) + tableInfo := table.Meta() + pi := tableInfo.GetPartitionInfo() + require.NotNil(t, pi) + + // analyze partition with index and with options are allowed under dynamic V1 + tk.MustExec("analyze table t partition p0 with 1 topn, 3 buckets") + tk.MustQuery("show warnings").Sort().Check(testkit.Rows( + "Warning 8131 Build table: `t` global-level stats failed due to missing partition-level stats", + "Warning 8131 Build table: `t` index: `idx` global-level stats failed due to missing partition-level stats", + )) + tk.MustExec("analyze table t partition p1 with 1 topn, 3 buckets") + tk.MustQuery("show warnings").Sort().Check(testkit.Rows()) + tk.MustQuery("select * from t where a > 1 and b > 1 and c > 1 and d > 1") + require.NoError(t, h.LoadNeededHistograms()) + tbl := h.GetTableStats(tableInfo) + lastVersion := tbl.Version + require.Equal(t, 3, len(tbl.Columns[tableInfo.Columns[2].ID].Buckets)) + require.Equal(t, 3, len(tbl.Columns[tableInfo.Columns[3].ID].Buckets)) + + tk.MustExec("analyze table t partition p1 index idx with 1 topn, 2 buckets") + tk.MustQuery("show warnings").Sort().Check(testkit.Rows()) + tbl = h.GetTableStats(tableInfo) + require.Greater(t, tbl.Version, lastVersion) + require.Equal(t, 2, len(tbl.Indices[tableInfo.Indices[0].ID].Buckets)) +} diff --git a/executor/show_test.go b/executor/show_test.go index 2d0fbdc05c3bd..e4e45bb4b9bb6 100644 --- a/executor/show_test.go +++ b/executor/show_test.go @@ -1588,11 +1588,6 @@ func TestShowVar(t *testing.T) { res = tk.MustQuery(showSQL) require.Len(t, res.Rows(), len(globalVars)) - // Test a known hidden variable. - res = tk.MustQuery("show variables like '" + variable.TiDBPartitionPruneMode + "'") - require.Len(t, res.Rows(), 0) - res = tk.MustQuery("show global variables like '" + variable.TiDBPartitionPruneMode + "'") - require.Len(t, res.Rows(), 0) // Test Hidden tx_read_ts res = tk.MustQuery("show variables like '%tx_read_ts'") require.Len(t, res.Rows(), 0) diff --git a/parser/model/model.go b/parser/model/model.go index 5da704cc01831..0fdfe50fd8dfb 100644 --- a/parser/model/model.go +++ b/parser/model/model.go @@ -299,6 +299,16 @@ func FindColumnInfoByID(cols []*ColumnInfo, id int64) *ColumnInfo { return nil } +// FindIndexInfoByID finds IndexInfo in indices by id. +func FindIndexInfoByID(indices []*IndexInfo, id int64) *IndexInfo { + for _, idx := range indices { + if idx.ID == id { + return idx + } + } + return nil +} + // ExtraHandleID is the column ID of column which we need to append to schema to occupy the handle's position // for use of execution phase. const ExtraHandleID = -1 @@ -1224,6 +1234,24 @@ func (t *TableInfo) FindConstraintInfoByName(constrName string) *ConstraintInfo return nil } +// FindIndexNameByID finds index name by id. +func (t *TableInfo) FindIndexNameByID(id int64) string { + indexInfo := FindIndexInfoByID(t.Indices, id) + if indexInfo != nil { + return indexInfo.Name.L + } + return "" +} + +// FindColumnNameByID finds column name by id. +func (t *TableInfo) FindColumnNameByID(id int64) string { + colInfo := FindColumnInfoByID(t.Columns, id) + if colInfo != nil { + return colInfo.Name.L + } + return "" +} + // FKInfo provides meta data describing a foreign key constraint. type FKInfo struct { ID int64 `json:"id"` diff --git a/planner/core/common_plans.go b/planner/core/common_plans.go index 4f189a642ee8a..8738776fa52b0 100644 --- a/planner/core/common_plans.go +++ b/planner/core/common_plans.go @@ -1117,11 +1117,12 @@ type AnalyzeInfo struct { // V2AnalyzeOptions is used to hold analyze options information. type V2AnalyzeOptions struct { - PhyTableID int64 - RawOpts map[ast.AnalyzeOptionType]uint64 - FilledOpts map[ast.AnalyzeOptionType]uint64 - ColChoice model.ColumnChoice - ColumnList []*model.ColumnInfo + PhyTableID int64 + RawOpts map[ast.AnalyzeOptionType]uint64 + FilledOpts map[ast.AnalyzeOptionType]uint64 + ColChoice model.ColumnChoice + ColumnList []*model.ColumnInfo + IsPartition bool } // AnalyzeColumnsTask is used for analyze columns. diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index 5dd27d8d407f6..4c83651d02bc4 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -2181,6 +2181,13 @@ func (b *PlanBuilder) genV2AnalyzeOptions( if !persist { return optionsMap, colsInfoMap, nil } + dynamicPrune := variable.PartitionPruneMode(b.ctx.GetSessionVars().PartitionPruneMode.Load()) == variable.Dynamic + if !isAnalyzeTable && dynamicPrune && (len(astOpts) > 0 || astColChoice != model.DefaultChoice) { + astOpts = make(map[ast.AnalyzeOptionType]uint64, 0) + astColChoice = model.DefaultChoice + astColList = make([]*model.ColumnInfo, 0) + b.ctx.GetSessionVars().StmtCtx.AppendWarning(errors.New("Ignore columns and options when analyze partition in dynamic mode")) + } tblSavedOpts, tblSavedColChoice, tblSavedColList, err := b.getSavedAnalyzeOpts(tbl.TableInfo.ID, tbl.TableInfo) if err != nil { return nil, nil, err @@ -2198,16 +2205,30 @@ func (b *PlanBuilder) genV2AnalyzeOptions( return nil, nil, err } tblAnalyzeOptions := V2AnalyzeOptions{ - PhyTableID: tbl.TableInfo.ID, - RawOpts: tblOpts, - FilledOpts: tblFilledOpts, - ColChoice: tblColChoice, - ColumnList: tblColList, + PhyTableID: tbl.TableInfo.ID, + RawOpts: tblOpts, + FilledOpts: tblFilledOpts, + ColChoice: tblColChoice, + ColumnList: tblColList, + IsPartition: false, } optionsMap[tbl.TableInfo.ID] = tblAnalyzeOptions colsInfoMap[tbl.TableInfo.ID] = tblColsInfo for _, id := range physicalIDs { if id != tbl.TableInfo.ID { + if dynamicPrune { + parV2Options := V2AnalyzeOptions{ + PhyTableID: id, + RawOpts: tblOpts, + FilledOpts: tblFilledOpts, + ColChoice: tblColChoice, + ColumnList: tblColList, + IsPartition: true, + } + optionsMap[id] = parV2Options + colsInfoMap[id] = tblColsInfo + continue + } parSavedOpts, parSavedColChoice, parSavedColList, err := b.getSavedAnalyzeOpts(id, tbl.TableInfo) if err != nil { return nil, nil, err diff --git a/sessionctx/variable/sysvar.go b/sessionctx/variable/sysvar.go index e780696f086aa..3587a343992d7 100644 --- a/sessionctx/variable/sysvar.go +++ b/sessionctx/variable/sysvar.go @@ -1359,14 +1359,26 @@ var defaultSysVars = []*SysVar{ s.EnableClusteredIndex = TiDBOptEnableClustered(val) return nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBPartitionPruneMode, Value: DefTiDBPartitionPruneMode, Hidden: true, Type: TypeStr, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { + {Scope: ScopeGlobal | ScopeSession, Name: TiDBPartitionPruneMode, Value: DefTiDBPartitionPruneMode, Type: TypeStr, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { mode := PartitionPruneMode(normalizedValue).Update() if !mode.Valid() { return normalizedValue, ErrWrongTypeForVar.GenWithStackByArgs(TiDBPartitionPruneMode) } return string(mode), nil + }, GetSession: func(s *SessionVars) (string, error) { + return s.PartitionPruneMode.Load(), nil }, SetSession: func(s *SessionVars, val string) error { - s.PartitionPruneMode.Store(strings.ToLower(strings.TrimSpace(val))) + newMode := strings.ToLower(strings.TrimSpace(val)) + if PartitionPruneMode(s.PartitionPruneMode.Load()) == Static && PartitionPruneMode(newMode) == Dynamic { + s.StmtCtx.AppendWarning(errors.New("Please analyze all partition tables again for consistency between partition and global stats")) + } + s.PartitionPruneMode.Store(newMode) + return nil + }, SetGlobal: func(s *SessionVars, val string) error { + newMode := strings.ToLower(strings.TrimSpace(val)) + if PartitionPruneMode(newMode) == Dynamic { + s.StmtCtx.AppendWarning(errors.New("Please analyze all partition tables again for consistency between partition and global stats")) + } return nil }}, {Scope: ScopeGlobal | ScopeSession, Name: TiDBRedactLog, Value: BoolToOnOff(DefTiDBRedactLog), Type: TypeBool, SetSession: func(s *SessionVars, val string) error { diff --git a/statistics/handle/handle.go b/statistics/handle/handle.go index 69e3a7d6290a7..e2aaf18fa324d 100644 --- a/statistics/handle/handle.go +++ b/statistics/handle/handle.go @@ -466,19 +466,24 @@ func (h *Handle) mergePartitionStats2GlobalStats(sc sessionctx.Context, opts map if isIndex == 0 { errMsg = fmt.Sprintf("`%s`", tableInfo.Name.L) } else { - indexName := "" - for _, idx := range tableInfo.Indices { - if idx.ID == histIDs[0] { - indexName = idx.Name.L - } - } - errMsg = fmt.Sprintf("`%s` index: `%s`", tableInfo.Name.L, indexName) + errMsg = fmt.Sprintf("`%s` index: `%s`", tableInfo.Name.L, tableInfo.FindIndexNameByID(histIDs[0])) } err = types.ErrPartitionStatsMissing.GenWithStackByArgs(errMsg) return } for i := 0; i < globalStats.Num; i++ { count, hg, cms, topN, fms := partitionStats.GetStatsInfo(histIDs[i], isIndex == 1) + // partition is not empty but column stats(hist, topn) is missing + if partitionStats.Count > 0 && (hg == nil || hg.TotalRowCount() <= 0) && (topN == nil || topN.TotalCount() <= 0) { + var errMsg string + if isIndex == 0 { + errMsg = fmt.Sprintf("`%s` column: `%s`", tableInfo.Name.L, tableInfo.FindColumnNameByID(histIDs[i])) + } else { + errMsg = fmt.Sprintf("`%s` index: `%s`", tableInfo.Name.L, tableInfo.FindIndexNameByID(histIDs[i])) + } + err = types.ErrPartitionColumnStatsMissing.GenWithStackByArgs(errMsg) + return + } if i == 0 { // In a partition, we will only update globalStats.Count once globalStats.Count += count diff --git a/statistics/handle/handle_test.go b/statistics/handle/handle_test.go index a84d2c71faa3d..d8b106c71e03f 100644 --- a/statistics/handle/handle_test.go +++ b/statistics/handle/handle_test.go @@ -1070,18 +1070,6 @@ partition by range (a) ( checkHealthy(60, 50, 66) } -func TestHideGlobalStatsSwitch(t *testing.T) { - // NOTICE: remove this test when this global-stats is GA. - store, clean := testkit.CreateMockStore(t) - defer clean() - tk := testkit.NewTestKit(t, store) - rs := tk.MustQuery("show variables").Rows() - for _, r := range rs { - require.NotEqual(t, "tidb_partition_prune_mode", strings.ToLower(r[0].(string))) - } - require.Len(t, tk.MustQuery("show variables where variable_name like '%tidb_partition_prune_mode%'").Rows(), 0) -} - func TestGlobalStatsData(t *testing.T) { store, dom, clean := testkit.CreateMockStoreAndDomain(t) defer clean() @@ -2220,6 +2208,7 @@ func TestFMSWithAnalyzePartition(t *testing.T) { tk.MustExec("analyze table t partition p0 with 1 topn, 2 buckets") tk.MustQuery("show warnings").Sort().Check(testkit.Rows( "Note 1105 Analyze use auto adjusted sample rate 1.000000 for table test.t's partition p0", + "Warning 1105 Ignore columns and options when analyze partition in dynamic mode", "Warning 8131 Build table: `t` global-level stats failed due to missing partition-level stats", "Warning 8131 Build table: `t` index: `a` global-level stats failed due to missing partition-level stats", )) diff --git a/types/errors.go b/types/errors.go index 4490ba03c3e9d..58cc5210a8b11 100644 --- a/types/errors.go +++ b/types/errors.go @@ -87,4 +87,7 @@ var ( // ErrPartitionStatsMissing is returned when the partition-level stats is missing and the build global-level stats fails. // Put this error here is to prevent `import cycle not allowed`. ErrPartitionStatsMissing = dbterror.ClassTypes.NewStd(mysql.ErrPartitionStatsMissing) + // ErrPartitionColumnStatsMissing is returned when the partition-level column stats is missing and the build global-level stats fails. + // Put this error here is to prevent `import cycle not allowed`. + ErrPartitionColumnStatsMissing = dbterror.ClassTypes.NewStd(mysql.ErrPartitionColumnStatsMissing) )