From eb786785aa77c0ec5ea92b40e86718bb8dedbeb3 Mon Sep 17 00:00:00 2001 From: wuxiaoju Date: Mon, 16 May 2022 16:34:58 +0800 Subject: [PATCH 01/11] global stats for dynamic prune --- errno/errcode.go | 1 + errno/errname.go | 7 +- executor/analyze.go | 104 +++++--- executor/analyze_test.go | 490 ++++++++++++++++++++++++++-------- planner/core/common_plans.go | 1 + planner/core/planbuilder.go | 67 +++-- sessionctx/variable/sysvar.go | 2 + statistics/handle/handle.go | 5 + types/errors.go | 3 + 9 files changed, 492 insertions(+), 188 deletions(-) diff --git a/errno/errcode.go b/errno/errcode.go index 0cb35c4cacae1..4296db33d3dd5 100644 --- a/errno/errcode.go +++ b/errno/errcode.go @@ -1067,6 +1067,7 @@ const ( ErrPlacementPolicyInUse = 8241 ErrOptOnCacheTable = 8242 ErrHTTPServiceError = 8243 + ErrPartitionColumnStatsMissing = 8244 // TiKV/PD/TiFlash errors. ErrPDServerTimeout = 9001 ErrTiKVServerTimeout = 9002 diff --git a/errno/errname.go b/errno/errname.go index d7925462dea21..3cf46e82fe6b5 100644 --- a/errno/errname.go +++ b/errno/errname.go @@ -1059,9 +1059,10 @@ var MySQLErrName = map[uint16]*mysql.ErrMessage{ ErrInvalidTableSample: mysql.Message("Invalid TABLESAMPLE: %s", nil), - ErrJSONObjectKeyTooLong: mysql.Message("TiDB does not yet support JSON objects with the key length >= 65536", nil), - ErrPartitionStatsMissing: mysql.Message("Build table: %s global-level stats failed due to missing partition-level stats", nil), - ErrNotSupportedWithSem: mysql.Message("Feature '%s' is not supported when security enhanced mode is enabled", nil), + ErrJSONObjectKeyTooLong: mysql.Message("TiDB does not yet support JSON objects with the key length >= 65536", nil), + ErrPartitionStatsMissing: mysql.Message("Build table: %s global-level stats failed due to missing partition-level stats", nil), + ErrPartitionColumnStatsMissing: mysql.Message("Build table: %s global-level stats failed due to missing partition-level column stats, please run analyze table to refresh columns of all partitions", nil), + ErrNotSupportedWithSem: mysql.Message("Feature '%s' is not supported when security enhanced mode is enabled", nil), ErrPlacementPolicyCheck: mysql.Message("Placement policy didn't meet the constraint, reason: %s", nil), ErrMultiStatementDisabled: mysql.Message("client has multi-statement capability disabled. Run SET GLOBAL tidb_multi_statement_mode='ON' after you understand the security risk", nil), diff --git a/executor/analyze.go b/executor/analyze.go index dd78388d2152f..8e88fe4cc6782 100644 --- a/executor/analyze.go +++ b/executor/analyze.go @@ -88,6 +88,19 @@ const ( maxSketchSize = 10000 ) +type globalStatsKey struct { + tableID int64 + indexID int64 +} + +type globalStatsInfo struct { + isIndex int + // When the `isIndex == 0`, histIDs will be the column IDs. + // Otherwise, histIDs will only contain the index ID. + histIDs []int64 + statsVersion int +} + // Next implements the Executor Next interface. func (e *AnalyzeExec) Next(ctx context.Context, req *chunk.Chunk) error { concurrency, err := getBuildStatsConcurrency(e.ctx) @@ -123,17 +136,6 @@ func (e *AnalyzeExec) Next(ctx context.Context, req *chunk.Chunk) error { pruneMode := variable.PartitionPruneMode(e.ctx.GetSessionVars().PartitionPruneMode.Load()) // needGlobalStats used to indicate whether we should merge the partition-level stats to global-level stats. needGlobalStats := pruneMode == variable.Dynamic - type globalStatsKey struct { - tableID int64 - indexID int64 - } - type globalStatsInfo struct { - isIndex int - // When the `isIndex == 0`, histIDs will be the column IDs. - // Otherwise, histIDs will only contain the index ID. - histIDs []int64 - statsVersion int - } // globalStatsMap is a map used to store which partition tables and the corresponding indexes need global-level stats. // The meaning of key in map is the structure that used to store the tableID and indexID. // The meaning of value in map is some additional information needed to build global-level stats. @@ -200,37 +202,9 @@ func (e *AnalyzeExec) Next(ctx context.Context, req *chunk.Chunk) error { return err } if needGlobalStats { - for globalStatsID, info := range globalStatsMap { - globalOpts := e.opts - if e.OptionsMap != nil { - if v2Options, ok := e.OptionsMap[globalStatsID.tableID]; ok { - globalOpts = v2Options.FilledOpts - } - } - globalStats, err := statsHandle.MergePartitionStats2GlobalStatsByTableID(e.ctx, globalOpts, e.ctx.GetInfoSchema().(infoschema.InfoSchema), globalStatsID.tableID, info.isIndex, info.histIDs) - if err != nil { - if types.ErrPartitionStatsMissing.Equal(err) { - // When we find some partition-level stats are missing, we need to report warning. - e.ctx.GetSessionVars().StmtCtx.AppendWarning(err) - continue - } - return err - } - for i := 0; i < globalStats.Num; i++ { - hg, cms, topN, fms := globalStats.Hg[i], globalStats.Cms[i], globalStats.TopN[i], globalStats.Fms[i] - // fms for global stats doesn't need to dump to kv. - err = statsHandle.SaveStatsToStorage(globalStatsID.tableID, globalStats.Count, info.isIndex, hg, cms, topN, fms, info.statsVersion, 1, false, true) - if err != nil { - logutil.Logger(ctx).Error("save global-level stats to storage failed", zap.Error(err)) - } - // Dump stats to historical storage. - if err := e.recordHistoricalStats(globalStatsID.tableID); err != nil { - logutil.BgLogger().Error("record historical stats failed", zap.Error(err)) - } - } - } + e.genGlobalStats(globalStatsMap) } - err = e.saveAnalyzeOptsV2() + err = e.saveV2AnalyzeOpts() if err != nil { e.ctx.GetSessionVars().StmtCtx.AppendWarning(err) } @@ -240,6 +214,40 @@ func (e *AnalyzeExec) Next(ctx context.Context, req *chunk.Chunk) error { return statsHandle.Update(e.ctx.GetInfoSchema().(infoschema.InfoSchema), handle.WithTableStatsByQuery()) } +func (e *AnalyzeExec) genGlobalStats(globalStatsMap map[globalStatsKey]globalStatsInfo) error { + statsHandle := domain.GetDomain(e.ctx).StatsHandle() + for globalStatsID, info := range globalStatsMap { + globalOpts := e.opts + if e.OptionsMap != nil { + if v2Options, ok := e.OptionsMap[globalStatsID.tableID]; ok { + globalOpts = v2Options.FilledOpts + } + } + globalStats, err := statsHandle.MergePartitionStats2GlobalStatsByTableID(e.ctx, globalOpts, e.ctx.GetInfoSchema().(infoschema.InfoSchema), globalStatsID.tableID, info.isIndex, info.histIDs) + if err != nil { + if types.ErrPartitionStatsMissing.Equal(err) || types.ErrPartitionColumnStatsMissing.Equal(err) { + // When we find some partition-level stats are missing, we need to report warning. + e.ctx.GetSessionVars().StmtCtx.AppendWarning(err) + continue + } + return err + } + for i := 0; i < globalStats.Num; i++ { + hg, cms, topN, fms := globalStats.Hg[i], globalStats.Cms[i], globalStats.TopN[i], globalStats.Fms[i] + // fms for global stats doesn't need to dump to kv. + err = statsHandle.SaveStatsToStorage(globalStatsID.tableID, globalStats.Count, info.isIndex, hg, cms, topN, fms, info.statsVersion, 1, false, true) + if err != nil { + logutil.BgLogger().Error("save global-level stats to storage failed", zap.Error(err)) + } + // Dump stats to historical storage. + if err := e.recordHistoricalStats(globalStatsID.tableID); err != nil { + logutil.BgLogger().Error("record historical stats failed", zap.Error(err)) + } + } + } + return nil +} + func finishJobWithLog(sctx sessionctx.Context, job *statistics.AnalyzeJob, analyzeErr error) { FinishAnalyzeJob(sctx, job, analyzeErr) if job != nil { @@ -258,14 +266,22 @@ func finishJobWithLog(sctx sessionctx.Context, job *statistics.AnalyzeJob, analy } } -func (e *AnalyzeExec) saveAnalyzeOptsV2() error { +func (e *AnalyzeExec) saveV2AnalyzeOpts() error { if !variable.PersistAnalyzeOptions.Load() || len(e.OptionsMap) == 0 { return nil } + // only to save table options if dynamic prune mode + dynamicPrune := variable.PartitionPruneMode(e.ctx.GetSessionVars().PartitionPruneMode.Load()) == variable.Dynamic + toSaveMap := make(map[int64]core.V2AnalyzeOptions) + for id, opts := range e.OptionsMap { + if opts.IsTable || !dynamicPrune { + toSaveMap[id] = opts + } + } sql := new(strings.Builder) sqlexec.MustFormatSQL(sql, "REPLACE INTO mysql.analyze_options (table_id,sample_num,sample_rate,buckets,topn,column_choice,column_ids) VALUES ") idx := 0 - for _, opts := range e.OptionsMap { + for _, opts := range toSaveMap { sampleNum := opts.RawOpts[ast.AnalyzeOptNumSamples] sampleRate := float64(0) if val, ok := opts.RawOpts[ast.AnalyzeOptSampleRate]; ok { @@ -283,7 +299,7 @@ func (e *AnalyzeExec) saveAnalyzeOptsV2() error { } colIDStrs := strings.Join(colIDs, ",") sqlexec.MustFormatSQL(sql, "(%?,%?,%?,%?,%?,%?,%?)", opts.PhyTableID, sampleNum, sampleRate, buckets, topn, colChoice, colIDStrs) - if idx < len(e.OptionsMap)-1 { + if idx < len(toSaveMap)-1 { sqlexec.MustFormatSQL(sql, ",") } idx += 1 diff --git a/executor/analyze_test.go b/executor/analyze_test.go index df657305edf5b..ec47f97a71db1 100644 --- a/executor/analyze_test.go +++ b/executor/analyze_test.go @@ -1543,124 +1543,6 @@ PARTITION BY RANGE ( a ) ( require.Equal(t, 0, len(rs.Rows())) } -func TestSavedPartitionAnalyzeOptionsDynamic(t *testing.T) { - store, dom, clean := testkit.CreateMockStoreAndDomain(t) - defer clean() - tk := testkit.NewTestKit(t, store) - originalVal := tk.MustQuery("select @@tidb_persist_analyze_options").Rows()[0][0].(string) - defer func() { - tk.MustExec(fmt.Sprintf("set global tidb_persist_analyze_options = %v", originalVal)) - }() - tk.MustExec("set global tidb_persist_analyze_options = true") - - tk.MustExec("use test") - tk.MustExec("set @@session.tidb_analyze_version = 2") - tk.MustExec("set @@session.tidb_partition_prune_mode = 'dynamic'") - createTable := `CREATE TABLE t (a int, b int, c varchar(10), primary key(a), index idx(b)) -PARTITION BY RANGE ( a ) ( - PARTITION p0 VALUES LESS THAN (10), - PARTITION p1 VALUES LESS THAN (20) -)` - tk.MustExec(createTable) - tk.MustExec("insert into t values (1,1,1),(2,1,2),(3,1,3),(4,1,4),(5,1,5),(6,1,6),(7,7,7),(8,8,8),(9,9,9),(10,10,10),(11,11,11),(12,12,12),(13,13,13),(14,14,14)") - - h := dom.StatsHandle() - oriLease := h.Lease() - h.SetLease(1) - defer func() { - h.SetLease(oriLease) - }() - is := dom.InfoSchema() - table, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) - require.NoError(t, err) - tableInfo := table.Meta() - pi := tableInfo.GetPartitionInfo() - require.NotNil(t, pi) - - // analyze partition only sets options of partition - tk.MustExec("analyze table t partition p0 with 1 topn, 3 buckets") - tk.MustQuery("select * from t where b > 1 and c > 1") - require.NoError(t, h.LoadNeededHistograms()) - tbl := h.GetTableStats(tableInfo) - lastVersion := tbl.Version - // no global stats since partition stats missing - require.Equal(t, 0, len(tbl.Columns[tableInfo.Columns[0].ID].Buckets)) - // The columns are: table_id, sample_num, sample_rate, buckets, topn, column_choice, column_ids. - rs := tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[0].ID, 10)) - require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "3", rs.Rows()[0][3]) - require.Equal(t, "1", rs.Rows()[0][4]) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) - require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "0", rs.Rows()[0][3]) - require.Equal(t, "-1", rs.Rows()[0][4]) - - // merge partition & table level options - tk.MustExec("analyze table t columns a,b with 0 topn, 2 buckets") - tk.MustQuery("select * from t where b > 1 and c > 1") - require.NoError(t, h.LoadNeededHistograms()) - tbl = h.GetTableStats(tableInfo) - require.Greater(t, tbl.Version, lastVersion) - lastVersion = tbl.Version - p0 := h.GetPartitionStats(tableInfo, pi.Definitions[0].ID) - p1 := h.GetPartitionStats(tableInfo, pi.Definitions[1].ID) - require.Equal(t, 2, len(p0.Columns[tableInfo.Columns[0].ID].Buckets)) - require.Equal(t, 2, len(p1.Columns[tableInfo.Columns[0].ID].Buckets)) - // merged global stats - require.Equal(t, 2, len(tbl.Columns[tableInfo.Columns[0].ID].Buckets)) - // check column c is not analyzed - require.Less(t, tbl.Columns[tableInfo.Columns[2].ID].LastUpdateVersion, tbl.Columns[tableInfo.Columns[0].ID].LastUpdateVersion) - require.Less(t, p0.Columns[tableInfo.Columns[2].ID].LastUpdateVersion, p0.Columns[tableInfo.Columns[0].ID].LastUpdateVersion) - require.Less(t, p1.Columns[tableInfo.Columns[2].ID].LastUpdateVersion, p1.Columns[tableInfo.Columns[0].ID].LastUpdateVersion) - // The columns are: table_id, sample_num, sample_rate, buckets, topn, column_choice, column_ids. - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(tbl.PhysicalID, 10)) - require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "0", rs.Rows()[0][2]) - require.Equal(t, "2", rs.Rows()[0][3]) - require.Equal(t, "0", rs.Rows()[0][4]) - require.Equal(t, "LIST", rs.Rows()[0][5]) - colIDStrs := strings.Join([]string{strconv.FormatInt(tableInfo.Columns[0].ID, 10), strconv.FormatInt(tableInfo.Columns[1].ID, 10)}, ",") - require.Equal(t, colIDStrs, rs.Rows()[0][6]) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(p0.PhysicalID, 10)) - require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "0", rs.Rows()[0][2]) - require.Equal(t, "2", rs.Rows()[0][3]) - require.Equal(t, "0", rs.Rows()[0][4]) - require.Equal(t, "LIST", rs.Rows()[0][5]) - require.Equal(t, colIDStrs, rs.Rows()[0][6]) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(p1.PhysicalID, 10)) - require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "0", rs.Rows()[0][2]) - require.Equal(t, "2", rs.Rows()[0][3]) - require.Equal(t, "0", rs.Rows()[0][4]) - require.Equal(t, "LIST", rs.Rows()[0][5]) - require.Equal(t, colIDStrs, rs.Rows()[0][6]) - - // analyze partition only updates this partition, and set different collist - tk.MustExec("analyze table t partition p1 columns a,c with 1 buckets") - tk.MustQuery("select * from t where b > 1 and c > 1") - require.NoError(t, h.LoadNeededHistograms()) - tbl = h.GetTableStats(tableInfo) - require.Greater(t, tbl.Version, lastVersion) - p0 = h.GetPartitionStats(tableInfo, pi.Definitions[0].ID) - p1 = h.GetPartitionStats(tableInfo, pi.Definitions[1].ID) - require.Equal(t, 1, len(p1.Columns[tableInfo.Columns[0].ID].Buckets)) - require.Equal(t, 2, len(tbl.Columns[tableInfo.Columns[0].ID].Buckets)) - // check column c is analyzed, but only global stats loaded - require.Equal(t, 0, len(p1.Columns[tableInfo.Columns[2].ID].Buckets)) - require.Equal(t, 2, len(tbl.Columns[tableInfo.Columns[2].ID].Buckets)) - // The columns are: table_id, sample_num, sample_rate, buckets, topn, column_choice, column_ids. - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(tbl.PhysicalID, 10)) - require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "2", rs.Rows()[0][3]) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(p1.PhysicalID, 10)) - require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "1", rs.Rows()[0][3]) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(p0.PhysicalID, 10)) - require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "2", rs.Rows()[0][3]) -} - func TestSavedAnalyzeOptionsForMultipleTables(t *testing.T) { store, dom, clean := testkit.CreateMockStoreAndDomain(t) defer clean() @@ -2986,3 +2868,375 @@ func TestShowAanalyzeStatusJobInfo(t *testing.T) { tk.MustExec("analyze table t") checkJobInfo("analyze table columns a, b, d with 3 buckets, 1 topn, 1 samplerate") } + +func TestAnalyzePartitionTableWithDynamicMode(t *testing.T) { + store, dom, clean := testkit.CreateMockStoreAndDomain(t) + defer clean() + tk := testkit.NewTestKit(t, store) + originalVal := tk.MustQuery("select @@tidb_persist_analyze_options").Rows()[0][0].(string) + defer func() { + tk.MustExec(fmt.Sprintf("set global tidb_persist_analyze_options = %v", originalVal)) + }() + tk.MustExec("set global tidb_persist_analyze_options = true") + + tk.MustExec("use test") + tk.MustExec("set @@session.tidb_analyze_version = 2") + tk.MustExec("set @@session.tidb_partition_prune_mode = 'dynamic'") + createTable := `CREATE TABLE t (a int, b int, c varchar(10), d int, primary key(a), index idx(b)) +PARTITION BY RANGE ( a ) ( + PARTITION p0 VALUES LESS THAN (10), + PARTITION p1 VALUES LESS THAN (20) +)` + tk.MustExec(createTable) + tk.MustExec("insert into t values (1,1,1,1),(2,1,2,2),(3,1,3,3),(4,1,4,4),(5,1,5,5),(6,1,6,6),(7,7,7,7),(8,8,8,8),(9,9,9,9)") + tk.MustExec("insert into t values (10,10,10,10),(11,11,11,11),(12,12,12,12),(13,13,13,13),(14,14,14,14)") + h := dom.StatsHandle() + oriLease := h.Lease() + h.SetLease(1) + defer func() { + h.SetLease(oriLease) + }() + is := dom.InfoSchema() + table, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) + require.NoError(t, err) + tableInfo := table.Meta() + pi := tableInfo.GetPartitionInfo() + require.NotNil(t, pi) + + // analyze table only sets table options and gen globalStats + tk.MustExec("analyze table t columns a,c with 1 topn, 3 buckets") + tk.MustQuery("select * from t where b > 1 and c > 1") + require.NoError(t, h.LoadNeededHistograms()) + tbl := h.GetTableStats(tableInfo) + lastVersion := tbl.Version + // both globalStats and partition stats generated and options saved for column a,c + require.Equal(t, 3, len(tbl.Columns[tableInfo.Columns[0].ID].Buckets)) + require.Equal(t, 1, len(tbl.Columns[tableInfo.Columns[0].ID].TopN.TopN)) + require.Equal(t, 3, len(tbl.Columns[tableInfo.Columns[2].ID].Buckets)) + require.Equal(t, 1, len(tbl.Columns[tableInfo.Columns[2].ID].TopN.TopN)) + rs := tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[0].ID, 10)) + require.Equal(t, 0, len(rs.Rows())) + rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[1].ID, 10)) + require.Equal(t, 0, len(rs.Rows())) + // The columns are: table_id, sample_num, sample_rate, buckets, topn, column_choice, column_ids. + rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) + require.Equal(t, 1, len(rs.Rows())) + require.Equal(t, "3", rs.Rows()[0][3]) + require.Equal(t, "1", rs.Rows()[0][4]) + + // analyze table with persisted table-level options + tk.MustExec("analyze table t") + tk.MustQuery("select * from t where b > 1 and c > 1") + require.NoError(t, h.LoadNeededHistograms()) + tbl = h.GetTableStats(tableInfo) + require.Greater(t, tbl.Version, lastVersion) + lastVersion = tbl.Version + require.Equal(t, 3, len(tbl.Columns[tableInfo.Columns[0].ID].Buckets)) + require.Equal(t, 1, len(tbl.Columns[tableInfo.Columns[0].ID].TopN.TopN)) + require.Equal(t, 3, len(tbl.Columns[tableInfo.Columns[2].ID].Buckets)) + require.Equal(t, 1, len(tbl.Columns[tableInfo.Columns[2].ID].TopN.TopN)) + rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[0].ID, 10)) + require.Equal(t, 0, len(rs.Rows())) + rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[1].ID, 10)) + require.Equal(t, 0, len(rs.Rows())) + // The columns are: table_id, sample_num, sample_rate, buckets, topn, column_choice, column_ids. + rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) + require.Equal(t, 1, len(rs.Rows())) + require.Equal(t, "3", rs.Rows()[0][3]) + require.Equal(t, "1", rs.Rows()[0][4]) + + // analyze table with merged table-level options + tk.MustExec("analyze table t with 2 topn, 2 buckets") + tk.MustQuery("select * from t where b > 1 and c > 1") + require.NoError(t, h.LoadNeededHistograms()) + tbl = h.GetTableStats(tableInfo) + lastVersion = tbl.Version + require.Equal(t, 2, len(tbl.Columns[tableInfo.Columns[0].ID].Buckets)) + require.Equal(t, 2, len(tbl.Columns[tableInfo.Columns[0].ID].TopN.TopN)) + require.Equal(t, 2, len(tbl.Columns[tableInfo.Columns[2].ID].Buckets)) + require.Equal(t, 2, len(tbl.Columns[tableInfo.Columns[2].ID].TopN.TopN)) + rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[0].ID, 10)) + require.Equal(t, 0, len(rs.Rows())) + rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[1].ID, 10)) + require.Equal(t, 0, len(rs.Rows())) + // The columns are: table_id, sample_num, sample_rate, buckets, topn, column_choice, column_ids. + rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) + require.Equal(t, 1, len(rs.Rows())) + require.Equal(t, "2", rs.Rows()[0][3]) + require.Equal(t, "2", rs.Rows()[0][4]) +} + +func TestAnalyzePartitionTableStaticToDynamic(t *testing.T) { + store, dom, clean := testkit.CreateMockStoreAndDomain(t) + defer clean() + tk := testkit.NewTestKit(t, store) + originalVal := tk.MustQuery("select @@tidb_persist_analyze_options").Rows()[0][0].(string) + defer func() { + tk.MustExec(fmt.Sprintf("set global tidb_persist_analyze_options = %v", originalVal)) + }() + tk.MustExec("set global tidb_persist_analyze_options = true") + + tk.MustExec("use test") + tk.MustExec("set @@session.tidb_analyze_version = 2") + tk.MustExec("set @@session.tidb_partition_prune_mode = 'static'") + createTable := `CREATE TABLE t (a int, b int, c varchar(10), d int, primary key(a), index idx(b)) +PARTITION BY RANGE ( a ) ( + PARTITION p0 VALUES LESS THAN (10), + PARTITION p1 VALUES LESS THAN (20) +)` + tk.MustExec(createTable) + tk.MustExec("insert into t values (1,1,1,1),(2,1,2,2),(3,1,3,3),(4,1,4,4),(5,1,5,5),(6,1,6,6),(7,7,7,7),(8,8,8,8),(9,9,9,9)") + tk.MustExec("insert into t values (10,10,10,10),(11,11,11,11),(12,12,12,12),(13,13,13,13),(14,14,14,14)") + h := dom.StatsHandle() + oriLease := h.Lease() + h.SetLease(1) + defer func() { + h.SetLease(oriLease) + }() + is := dom.InfoSchema() + table, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) + require.NoError(t, err) + tableInfo := table.Meta() + pi := tableInfo.GetPartitionInfo() + require.NotNil(t, pi) + + // analyze partition under static mode with options + tk.MustExec("analyze table t partition p0 columns a,c with 1 topn, 3 buckets") + tk.MustQuery("select * from t where b > 1 and c > 1") + require.NoError(t, h.LoadNeededHistograms()) + tbl := h.GetTableStats(tableInfo) + p0 := h.GetPartitionStats(tableInfo, pi.Definitions[0].ID) + p1 := h.GetPartitionStats(tableInfo, pi.Definitions[1].ID) + lastVersion := tbl.Version + require.Equal(t, 3, len(p0.Columns[tableInfo.Columns[0].ID].Buckets)) + require.Equal(t, 3, len(p0.Columns[tableInfo.Columns[2].ID].Buckets)) + require.Equal(t, 0, len(p1.Columns[tableInfo.Columns[0].ID].Buckets)) + require.Equal(t, 0, len(tbl.Columns[tableInfo.Columns[0].ID].Buckets)) + rs := tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[0].ID, 10)) + require.Equal(t, 1, len(rs.Rows())) + require.Equal(t, "3", rs.Rows()[0][3]) + require.Equal(t, "1", rs.Rows()[0][4]) + rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[1].ID, 10)) + require.Equal(t, 0, len(rs.Rows())) + // The columns are: table_id, sample_num, sample_rate, buckets, topn, column_choice, column_ids. + rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) + require.Equal(t, 1, len(rs.Rows())) + require.Equal(t, "0", rs.Rows()[0][3]) + + tk.MustExec("set @@session.tidb_partition_prune_mode = 'dynamic'") + + // analyze table and dynamic mode will ignore partition-level options and use default + tk.MustExec("analyze table t") + tk.MustQuery("select * from t where b > 1 and c > 1") + require.NoError(t, h.LoadNeededHistograms()) + tbl = h.GetTableStats(tableInfo) + require.Greater(t, tbl.Version, lastVersion) + lastVersion = tbl.Version + p0 = h.GetPartitionStats(tableInfo, pi.Definitions[0].ID) + p1 = h.GetPartitionStats(tableInfo, pi.Definitions[1].ID) + require.NotEqual(t, 3, len(p0.Columns[tableInfo.Columns[0].ID].Buckets)) + require.Equal(t, len(tbl.Columns[tableInfo.Columns[0].ID].Buckets), len(p0.Columns[tableInfo.Columns[0].ID].Buckets)) + require.Equal(t, len(tbl.Columns[tableInfo.Columns[0].ID].Buckets), len(p1.Columns[tableInfo.Columns[0].ID].Buckets)) + rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[0].ID, 10)) + require.Equal(t, 1, len(rs.Rows())) + require.Equal(t, "3", rs.Rows()[0][3]) + require.Equal(t, "1", rs.Rows()[0][4]) + rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[1].ID, 10)) + require.Equal(t, 0, len(rs.Rows())) + // The columns are: table_id, sample_num, sample_rate, buckets, topn, column_choice, column_ids. + rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) + require.Equal(t, 1, len(rs.Rows())) + require.Equal(t, "0", rs.Rows()[0][3]) + + // analyze table under dynamic mode with specified options with old partition-level options + tk.MustExec("analyze table t columns b,d with 2 topn, 2 buckets") + tk.MustQuery("select * from t where b > 1 and d > 1") + require.NoError(t, h.LoadNeededHistograms()) + tbl = h.GetTableStats(tableInfo) + require.Greater(t, tbl.Version, lastVersion) + lastVersion = tbl.Version + require.Equal(t, 2, len(tbl.Columns[tableInfo.Columns[1].ID].Buckets)) + require.Equal(t, 2, len(tbl.Columns[tableInfo.Columns[3].ID].Buckets)) + rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[0].ID, 10)) + require.Equal(t, 1, len(rs.Rows())) + require.Equal(t, "3", rs.Rows()[0][3]) + require.Equal(t, "1", rs.Rows()[0][4]) + rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[1].ID, 10)) + require.Equal(t, 0, len(rs.Rows())) + // The columns are: table_id, sample_num, sample_rate, buckets, topn, column_choice, column_ids. + rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) + require.Equal(t, 1, len(rs.Rows())) + require.Equal(t, "2", rs.Rows()[0][3]) + require.Equal(t, "2", rs.Rows()[0][4]) + + // analyze table under dynamic mode without options with old table-level & partition-level options + tk.MustExec("analyze table t") + tk.MustQuery("select * from t where b > 1 and d > 1") + require.NoError(t, h.LoadNeededHistograms()) + tbl = h.GetTableStats(tableInfo) + require.Greater(t, tbl.Version, lastVersion) + lastVersion = tbl.Version + require.Equal(t, 2, len(tbl.Columns[tableInfo.Columns[3].ID].Buckets)) + require.Equal(t, 2, len(tbl.Columns[tableInfo.Columns[3].ID].TopN.TopN)) + + // analyze table under dynamic mode with specified options with old table-level & partition-level options + tk.MustExec("analyze table t with 1 topn") + tk.MustQuery("select * from t where b > 1 and d > 1") + require.NoError(t, h.LoadNeededHistograms()) + tbl = h.GetTableStats(tableInfo) + require.Greater(t, tbl.Version, lastVersion) + lastVersion = tbl.Version + require.Equal(t, 2, len(tbl.Columns[tableInfo.Columns[1].ID].Buckets)) + require.Equal(t, 2, len(tbl.Columns[tableInfo.Columns[3].ID].Buckets)) + require.Equal(t, 1, len(tbl.Columns[tableInfo.Columns[1].ID].TopN.TopN)) + require.Equal(t, 1, len(tbl.Columns[tableInfo.Columns[3].ID].TopN.TopN)) + rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[0].ID, 10)) + require.Equal(t, 1, len(rs.Rows())) + require.Equal(t, "3", rs.Rows()[0][3]) + require.Equal(t, "1", rs.Rows()[0][4]) + rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[1].ID, 10)) + require.Equal(t, 0, len(rs.Rows())) + // The columns are: table_id, sample_num, sample_rate, buckets, topn, column_choice, column_ids. + rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) + require.Equal(t, 1, len(rs.Rows())) + require.Equal(t, "2", rs.Rows()[0][3]) + require.Equal(t, "1", rs.Rows()[0][4]) +} + +func TestAnalyzePartitionUnderDynamic(t *testing.T) { + store, dom, clean := testkit.CreateMockStoreAndDomain(t) + defer clean() + tk := testkit.NewTestKit(t, store) + originalVal := tk.MustQuery("select @@tidb_persist_analyze_options").Rows()[0][0].(string) + defer func() { + tk.MustExec(fmt.Sprintf("set global tidb_persist_analyze_options = %v", originalVal)) + }() + tk.MustExec("set global tidb_persist_analyze_options = true") + + tk.MustExec("use test") + tk.MustExec("set @@session.tidb_analyze_version = 2") + tk.MustExec("set @@session.tidb_partition_prune_mode = 'dynamic'") + createTable := `CREATE TABLE t (a int, b int, c varchar(10), d int, primary key(a), index idx(b)) +PARTITION BY RANGE ( a ) ( + PARTITION p0 VALUES LESS THAN (10), + PARTITION p1 VALUES LESS THAN (20) +)` + tk.MustExec(createTable) + tk.MustExec("insert into t values (1,1,1,1),(2,1,2,2),(3,1,3,3),(4,1,4,4),(5,1,5,5),(6,1,6,6),(7,7,7,7),(8,8,8,8),(9,9,9,9)") + tk.MustExec("insert into t values (10,10,10,10),(11,11,11,11),(12,12,12,12),(13,13,13,13),(14,14,14,14)") + h := dom.StatsHandle() + oriLease := h.Lease() + h.SetLease(1) + defer func() { + h.SetLease(oriLease) + }() + is := dom.InfoSchema() + table, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) + require.NoError(t, err) + tableInfo := table.Meta() + pi := tableInfo.GetPartitionInfo() + require.NotNil(t, pi) + + // analyze partition with options under dynamic mode + tk.MustExec("analyze table t partition p0 columns a,b,c with 1 topn, 3 buckets") + tk.MustQuery("show warnings").Sort().Check(testkit.Rows( + "Note 1105 Analyze use auto adjusted sample rate 1.000000 for table test.t's partition p0", + "Warning 1105 Ignore columns and options when analyze partition in dynamic mode", + "Warning 8131 Build table: `t` global-level stats failed due to missing partition-level stats", + "Warning 8131 Build table: `t` index: `idx` global-level stats failed due to missing partition-level stats", + )) + tk.MustQuery("select * from t where a > 1 and b > 1 and c > 1 and d > 1") + require.NoError(t, h.LoadNeededHistograms()) + tbl := h.GetTableStats(tableInfo) + lastVersion := tbl.Version + require.NotEqual(t, 3, len(tbl.Columns[tableInfo.Columns[2].ID].Buckets)) + require.NotEqual(t, 3, len(tbl.Columns[tableInfo.Columns[3].ID].Buckets)) + + tk.MustExec("analyze table t partition p0") + tk.MustQuery("show warnings").Sort().Check(testkit.Rows( + "Note 1105 Analyze use auto adjusted sample rate 1.000000 for table test.t's partition p0", + "Warning 8131 Build table: `t` global-level stats failed due to missing partition-level stats", + "Warning 8131 Build table: `t` index: `idx` global-level stats failed due to missing partition-level stats", + )) + tbl = h.GetTableStats(tableInfo) + require.Equal(t, tbl.Version, lastVersion) // global stats not updated +} + +func TestAnalyzePartitionStaticToDynamic(t *testing.T) { + store, dom, clean := testkit.CreateMockStoreAndDomain(t) + defer clean() + tk := testkit.NewTestKit(t, store) + originalVal := tk.MustQuery("select @@tidb_persist_analyze_options").Rows()[0][0].(string) + defer func() { + tk.MustExec(fmt.Sprintf("set global tidb_persist_analyze_options = %v", originalVal)) + }() + + tk.MustExec("use test") + tk.MustExec("set @@session.tidb_analyze_version = 2") + createTable := `CREATE TABLE t (a int, b int, c varchar(10), d int, primary key(a), index idx(b)) +PARTITION BY RANGE ( a ) ( + PARTITION p0 VALUES LESS THAN (10), + PARTITION p1 VALUES LESS THAN (20) +)` + tk.MustExec(createTable) + tk.MustExec("insert into t values (1,1,1,1),(2,1,2,2),(3,1,3,3),(4,1,4,4),(5,1,5,5),(6,1,6,6),(7,7,7,7),(8,8,8,8),(9,9,9,9)") + tk.MustExec("insert into t values (10,10,10,10),(11,11,11,11),(12,12,12,12),(13,13,13,13),(14,14,14,14)") + h := dom.StatsHandle() + oriLease := h.Lease() + h.SetLease(1) + defer func() { + h.SetLease(oriLease) + }() + is := dom.InfoSchema() + table, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) + require.NoError(t, err) + tableInfo := table.Meta() + pi := tableInfo.GetPartitionInfo() + require.NotNil(t, pi) + + // generate old partition stats + tk.MustExec("set global tidb_persist_analyze_options = false") + tk.MustExec("set @@session.tidb_partition_prune_mode = 'static'") + tk.MustExec("analyze table t partition p0 columns a,c with 1 topn, 3 buckets") + tk.MustQuery("select * from t where a > 1 and b > 1 and c > 1 and d > 1") + require.NoError(t, h.LoadNeededHistograms()) + p0 := h.GetPartitionStats(tableInfo, pi.Definitions[0].ID) + require.Equal(t, 3, len(p0.Columns[tableInfo.Columns[2].ID].Buckets)) + + // analyze partition with existing stats of other partitions + tk.MustExec("set @@session.tidb_partition_prune_mode = 'dynamic'") + tk.MustExec("analyze table t partition p1 columns a,b,d with 1 topn, 3 buckets") + tk.MustQuery("show warnings").Sort().Check(testkit.Rows( + "Note 1105 Analyze use auto adjusted sample rate 1.000000 for table test.t's partition p1", + "Warning 8244 Build table: `t` global-level stats failed due to missing partition-level column stats, please run analyze table to refresh columns of all partitions", + )) + + // insert table-level persisted analyze options + tk.MustExec("insert into mysql.analyze_options values (?,?,?,?,?,?,?)", tableInfo.ID, 0, 0, 2, 2, "DEFAULT", "") + tk.MustExec("analyze table t partition p1 columns a,b,d with 1 topn, 3 buckets") + tk.MustQuery("show warnings").Sort().Check(testkit.Rows( + "Note 1105 Analyze use auto adjusted sample rate 1.000000 for table test.t's partition p1", + "Warning 8244 Build table: `t` global-level stats failed due to missing partition-level column stats, please run analyze table to refresh columns of all partitions", + )) + + // insert partition-level persisted analyze options + tk.MustExec("insert into mysql.analyze_options values (?,?,?,?,?,?,?)", pi.Definitions[0].ID, 0, 0, 2, 2, "DEFAULT", "") + tk.MustExec("analyze table t partition p1 columns a,b,d with 1 topn, 3 buckets") + tk.MustQuery("show warnings").Sort().Check(testkit.Rows( + "Note 1105 Analyze use auto adjusted sample rate 1.000000 for table test.t's partition p1", + "Warning 8244 Build table: `t` global-level stats failed due to missing partition-level column stats, please run analyze table to refresh columns of all partitions", + )) + tk.MustQuery("select * from t where a > 1 and b > 1 and c > 1 and d > 1") + require.NoError(t, h.LoadNeededHistograms()) + tbl := h.GetTableStats(tableInfo) + require.Equal(t, 0, len(tbl.Columns)) + + tk.MustExec("analyze table t partition p1 columns a,b,c with 1 topn, 3 buckets") + tk.MustQuery("show warnings").Sort().Check(testkit.Rows( + "Note 1105 Analyze use auto adjusted sample rate 1.000000 for table test.t's partition p1", + )) + tk.MustQuery("select * from t where a > 1 and b > 1 and c > 1 and d > 1") + require.NoError(t, h.LoadNeededHistograms()) + tbl = h.GetTableStats(tableInfo) + require.Equal(t, 3, len(tbl.Columns[tableInfo.Columns[2].ID].Buckets)) +} diff --git a/planner/core/common_plans.go b/planner/core/common_plans.go index a33a0bd61087d..5a84f30f64032 100644 --- a/planner/core/common_plans.go +++ b/planner/core/common_plans.go @@ -1107,6 +1107,7 @@ type V2AnalyzeOptions struct { FilledOpts map[ast.AnalyzeOptionType]uint64 ColChoice model.ColumnChoice ColumnList []*model.ColumnInfo + IsTable bool } // AnalyzeColumnsTask is used for analyze columns. diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index e41731103b91e..a7c491c8d9c0e 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -2178,6 +2178,13 @@ func (b *PlanBuilder) genV2AnalyzeOptions( if !persist { return optionsMap, colsInfoMap, nil } + dynamicPrune := variable.PartitionPruneMode(b.ctx.GetSessionVars().PartitionPruneMode.Load()) == variable.Dynamic + if !isAnalyzeTable && dynamicPrune && (len(astOpts) > 0 || astColChoice != model.DefaultChoice) { + astOpts = make(map[ast.AnalyzeOptionType]uint64, 0) + astColChoice = model.DefaultChoice + astColList = make([]*model.ColumnInfo, 0) + b.ctx.GetSessionVars().StmtCtx.AppendWarning(errors.New("Ignore columns and options when analyze partition in dynamic mode")) + } tblSavedOpts, tblSavedColChoice, tblSavedColList, err := b.getSavedAnalyzeOpts(tbl.TableInfo.ID, tbl.TableInfo) if err != nil { return nil, nil, err @@ -2200,35 +2207,49 @@ func (b *PlanBuilder) genV2AnalyzeOptions( FilledOpts: tblFilledOpts, ColChoice: tblColChoice, ColumnList: tblColList, + IsTable: true, } optionsMap[tbl.TableInfo.ID] = tblAnalyzeOptions colsInfoMap[tbl.TableInfo.ID] = tblColsInfo for _, id := range physicalIDs { if id != tbl.TableInfo.ID { - parSavedOpts, parSavedColChoice, parSavedColList, err := b.getSavedAnalyzeOpts(id, tbl.TableInfo) - if err != nil { - return nil, nil, err - } - // merge partition level options with table level options firstly - savedOpts := mergeAnalyzeOptions(parSavedOpts, tblSavedOpts) - savedColChoice, savedColList := mergeColumnList(parSavedColChoice, parSavedColList, tblSavedColChoice, tblSavedColList) - // then merge statement level options - mergedOpts := mergeAnalyzeOptions(astOpts, savedOpts) - filledMergedOpts := fillAnalyzeOptionsV2(mergedOpts) - finalColChoice, mergedColList := mergeColumnList(astColChoice, astColList, savedColChoice, savedColList) - finalColsInfo, finalColList, err := b.getFullAnalyzeColumnsInfo(tbl, finalColChoice, mergedColList, predicateCols, mustAnalyzedCols, mustAllColumns, false) - if err != nil { - return nil, nil, err - } - parV2Options := V2AnalyzeOptions{ - PhyTableID: id, - RawOpts: mergedOpts, - FilledOpts: filledMergedOpts, - ColChoice: finalColChoice, - ColumnList: finalColList, + if !dynamicPrune { + parSavedOpts, parSavedColChoice, parSavedColList, err := b.getSavedAnalyzeOpts(id, tbl.TableInfo) + if err != nil { + return nil, nil, err + } + // merge partition level options with table level options firstly + savedOpts := mergeAnalyzeOptions(parSavedOpts, tblSavedOpts) + savedColChoice, savedColList := mergeColumnList(parSavedColChoice, parSavedColList, tblSavedColChoice, tblSavedColList) + // then merge statement level options + mergedOpts := mergeAnalyzeOptions(astOpts, savedOpts) + filledMergedOpts := fillAnalyzeOptionsV2(mergedOpts) + finalColChoice, mergedColList := mergeColumnList(astColChoice, astColList, savedColChoice, savedColList) + finalColsInfo, finalColList, err := b.getFullAnalyzeColumnsInfo(tbl, finalColChoice, mergedColList, predicateCols, mustAnalyzedCols, mustAllColumns, false) + if err != nil { + return nil, nil, err + } + parV2Options := V2AnalyzeOptions{ + PhyTableID: id, + RawOpts: mergedOpts, + FilledOpts: filledMergedOpts, + ColChoice: finalColChoice, + ColumnList: finalColList, + } + optionsMap[id] = parV2Options + colsInfoMap[id] = finalColsInfo + } else { + parV2Options := V2AnalyzeOptions{ + PhyTableID: id, + RawOpts: tblOpts, + FilledOpts: tblFilledOpts, + ColChoice: tblColChoice, + ColumnList: tblColList, + IsTable: false, + } + optionsMap[id] = parV2Options + colsInfoMap[id] = tblColsInfo } - optionsMap[id] = parV2Options - colsInfoMap[id] = finalColsInfo } } return optionsMap, colsInfoMap, nil diff --git a/sessionctx/variable/sysvar.go b/sessionctx/variable/sysvar.go index 9edfd0f0d2a7a..21448dd8fae02 100644 --- a/sessionctx/variable/sysvar.go +++ b/sessionctx/variable/sysvar.go @@ -1305,6 +1305,8 @@ var defaultSysVars = []*SysVar{ return normalizedValue, ErrWrongTypeForVar.GenWithStackByArgs(TiDBPartitionPruneMode) } return string(mode), nil + }, GetSession: func(s *SessionVars) (string, error) { + return s.PartitionPruneMode.Load(), nil }, SetSession: func(s *SessionVars, val string) error { s.PartitionPruneMode.Store(strings.ToLower(strings.TrimSpace(val))) return nil diff --git a/statistics/handle/handle.go b/statistics/handle/handle.go index fb75619ed4252..b454365c8c291 100644 --- a/statistics/handle/handle.go +++ b/statistics/handle/handle.go @@ -417,6 +417,11 @@ func (h *Handle) mergePartitionStats2GlobalStats(sc sessionctx.Context, opts map } for i := 0; i < globalStats.Num; i++ { count, hg, cms, topN, fms := partitionStats.GetStatsInfo(histIDs[i], isIndex == 1) + if (hg == nil || hg.TotalRowCount() <= 0) && (topN == nil || topN.TotalCount() <= 0) { + errMsg := fmt.Sprintf("`%s`", tableInfo.Name.L) + err = types.ErrPartitionColumnStatsMissing.GenWithStackByArgs(errMsg) + return + } if i == 0 { // In a partition, we will only update globalStats.Count once globalStats.Count += count diff --git a/types/errors.go b/types/errors.go index 4490ba03c3e9d..58cc5210a8b11 100644 --- a/types/errors.go +++ b/types/errors.go @@ -87,4 +87,7 @@ var ( // ErrPartitionStatsMissing is returned when the partition-level stats is missing and the build global-level stats fails. // Put this error here is to prevent `import cycle not allowed`. ErrPartitionStatsMissing = dbterror.ClassTypes.NewStd(mysql.ErrPartitionStatsMissing) + // ErrPartitionColumnStatsMissing is returned when the partition-level column stats is missing and the build global-level stats fails. + // Put this error here is to prevent `import cycle not allowed`. + ErrPartitionColumnStatsMissing = dbterror.ClassTypes.NewStd(mysql.ErrPartitionColumnStatsMissing) ) From 8f00715875d1af73051b3149cb8008363d82df09 Mon Sep 17 00:00:00 2001 From: wuxiaoju Date: Tue, 17 May 2022 14:35:50 +0800 Subject: [PATCH 02/11] fix for ut --- statistics/handle/handle.go | 3 ++- statistics/handle/handle_test.go | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/statistics/handle/handle.go b/statistics/handle/handle.go index 0047395b55501..fbc08186566a3 100644 --- a/statistics/handle/handle.go +++ b/statistics/handle/handle.go @@ -425,7 +425,8 @@ func (h *Handle) mergePartitionStats2GlobalStats(sc sessionctx.Context, opts map } for i := 0; i < globalStats.Num; i++ { count, hg, cms, topN, fms := partitionStats.GetStatsInfo(histIDs[i], isIndex == 1) - if (hg == nil || hg.TotalRowCount() <= 0) && (topN == nil || topN.TotalCount() <= 0) { + // partition is not empty but column stats(hist, topn) is missing + if partitionStats.Count > 0 && (hg == nil || hg.TotalRowCount() <= 0) && (topN == nil || topN.TotalCount() <= 0) { errMsg := fmt.Sprintf("`%s`", tableInfo.Name.L) err = types.ErrPartitionColumnStatsMissing.GenWithStackByArgs(errMsg) return diff --git a/statistics/handle/handle_test.go b/statistics/handle/handle_test.go index a84d2c71faa3d..ef2c662942ccd 100644 --- a/statistics/handle/handle_test.go +++ b/statistics/handle/handle_test.go @@ -2220,6 +2220,7 @@ func TestFMSWithAnalyzePartition(t *testing.T) { tk.MustExec("analyze table t partition p0 with 1 topn, 2 buckets") tk.MustQuery("show warnings").Sort().Check(testkit.Rows( "Note 1105 Analyze use auto adjusted sample rate 1.000000 for table test.t's partition p0", + "Warning 1105 Ignore columns and options when analyze partition in dynamic mode", "Warning 8131 Build table: `t` global-level stats failed due to missing partition-level stats", "Warning 8131 Build table: `t` index: `a` global-level stats failed due to missing partition-level stats", )) From 2c41cb5cdef4468165bbddd1f05e67cb220e7a51 Mon Sep 17 00:00:00 2001 From: wuxiaoju Date: Tue, 17 May 2022 15:19:14 +0800 Subject: [PATCH 03/11] do not hide partition_prune variable --- sessionctx/variable/sysvar.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sessionctx/variable/sysvar.go b/sessionctx/variable/sysvar.go index 196c7ed6f2f29..e63ff0403a903 100644 --- a/sessionctx/variable/sysvar.go +++ b/sessionctx/variable/sysvar.go @@ -1315,7 +1315,7 @@ var defaultSysVars = []*SysVar{ s.EnableClusteredIndex = TiDBOptEnableClustered(val) return nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBPartitionPruneMode, Value: DefTiDBPartitionPruneMode, Hidden: true, Type: TypeStr, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { + {Scope: ScopeGlobal | ScopeSession, Name: TiDBPartitionPruneMode, Value: DefTiDBPartitionPruneMode, Type: TypeStr, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) { mode := PartitionPruneMode(normalizedValue).Update() if !mode.Valid() { return normalizedValue, ErrWrongTypeForVar.GenWithStackByArgs(TiDBPartitionPruneMode) From 0c836ac6c443e26eb4eae102ef12ee335575b2e5 Mon Sep 17 00:00:00 2001 From: wuxiaoju Date: Tue, 17 May 2022 16:48:39 +0800 Subject: [PATCH 04/11] warnings when set dynamic prune mode --- sessionctx/variable/sysvar.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/sessionctx/variable/sysvar.go b/sessionctx/variable/sysvar.go index e63ff0403a903..9a3773326832c 100644 --- a/sessionctx/variable/sysvar.go +++ b/sessionctx/variable/sysvar.go @@ -1324,7 +1324,17 @@ var defaultSysVars = []*SysVar{ }, GetSession: func(s *SessionVars) (string, error) { return s.PartitionPruneMode.Load(), nil }, SetSession: func(s *SessionVars, val string) error { - s.PartitionPruneMode.Store(strings.ToLower(strings.TrimSpace(val))) + newMode := strings.ToLower(strings.TrimSpace(val)) + if PartitionPruneMode(s.PartitionPruneMode.Load()) == Static && PartitionPruneMode(newMode) == Dynamic { + s.StmtCtx.AppendWarning(errors.New("Please analyze all partition tables again for consistency between partition and global stats")) + } + s.PartitionPruneMode.Store(newMode) + return nil + }, SetGlobal: func(s *SessionVars, val string) error { + newMode := strings.ToLower(strings.TrimSpace(val)) + if PartitionPruneMode(newMode) == Dynamic { + s.StmtCtx.AppendWarning(errors.New("Please analyze all partition tables again for consistency between partition and global stats")) + } return nil }}, {Scope: ScopeGlobal | ScopeSession, Name: TiDBRedactLog, Value: BoolToOnOff(DefTiDBRedactLog), Type: TypeBool, SetSession: func(s *SessionVars, val string) error { From 2b2761f8be0ae947504e97c99ce04780b1cc7f22 Mon Sep 17 00:00:00 2001 From: wuxiaoju Date: Tue, 17 May 2022 16:54:00 +0800 Subject: [PATCH 05/11] fix --- executor/analyze.go | 88 ++++++++++++++++++---------------------- executor/analyze_test.go | 3 +- 2 files changed, 41 insertions(+), 50 deletions(-) diff --git a/executor/analyze.go b/executor/analyze.go index 8e88fe4cc6782..a6f095b37d3de 100644 --- a/executor/analyze.go +++ b/executor/analyze.go @@ -88,19 +88,6 @@ const ( maxSketchSize = 10000 ) -type globalStatsKey struct { - tableID int64 - indexID int64 -} - -type globalStatsInfo struct { - isIndex int - // When the `isIndex == 0`, histIDs will be the column IDs. - // Otherwise, histIDs will only contain the index ID. - histIDs []int64 - statsVersion int -} - // Next implements the Executor Next interface. func (e *AnalyzeExec) Next(ctx context.Context, req *chunk.Chunk) error { concurrency, err := getBuildStatsConcurrency(e.ctx) @@ -136,6 +123,17 @@ func (e *AnalyzeExec) Next(ctx context.Context, req *chunk.Chunk) error { pruneMode := variable.PartitionPruneMode(e.ctx.GetSessionVars().PartitionPruneMode.Load()) // needGlobalStats used to indicate whether we should merge the partition-level stats to global-level stats. needGlobalStats := pruneMode == variable.Dynamic + type globalStatsKey struct { + tableID int64 + indexID int64 + } + type globalStatsInfo struct { + isIndex int + // When the `isIndex == 0`, histIDs will be the column IDs. + // Otherwise, histIDs will only contain the index ID. + histIDs []int64 + statsVersion int + } // globalStatsMap is a map used to store which partition tables and the corresponding indexes need global-level stats. // The meaning of key in map is the structure that used to store the tableID and indexID. // The meaning of value in map is some additional information needed to build global-level stats. @@ -202,7 +200,35 @@ func (e *AnalyzeExec) Next(ctx context.Context, req *chunk.Chunk) error { return err } if needGlobalStats { - e.genGlobalStats(globalStatsMap) + for globalStatsID, info := range globalStatsMap { + globalOpts := e.opts + if e.OptionsMap != nil { + if v2Options, ok := e.OptionsMap[globalStatsID.tableID]; ok { + globalOpts = v2Options.FilledOpts + } + } + globalStats, err := statsHandle.MergePartitionStats2GlobalStatsByTableID(e.ctx, globalOpts, e.ctx.GetInfoSchema().(infoschema.InfoSchema), globalStatsID.tableID, info.isIndex, info.histIDs) + if err != nil { + if types.ErrPartitionStatsMissing.Equal(err) || types.ErrPartitionColumnStatsMissing.Equal(err) { + // When we find some partition-level stats are missing, we need to report warning. + e.ctx.GetSessionVars().StmtCtx.AppendWarning(err) + continue + } + return err + } + for i := 0; i < globalStats.Num; i++ { + hg, cms, topN, fms := globalStats.Hg[i], globalStats.Cms[i], globalStats.TopN[i], globalStats.Fms[i] + // fms for global stats doesn't need to dump to kv. + err = statsHandle.SaveStatsToStorage(globalStatsID.tableID, globalStats.Count, info.isIndex, hg, cms, topN, fms, info.statsVersion, 1, false, true) + if err != nil { + logutil.Logger(ctx).Error("save global-level stats to storage failed", zap.Error(err)) + } + // Dump stats to historical storage. + if err := e.recordHistoricalStats(globalStatsID.tableID); err != nil { + logutil.BgLogger().Error("record historical stats failed", zap.Error(err)) + } + } + } } err = e.saveV2AnalyzeOpts() if err != nil { @@ -214,40 +240,6 @@ func (e *AnalyzeExec) Next(ctx context.Context, req *chunk.Chunk) error { return statsHandle.Update(e.ctx.GetInfoSchema().(infoschema.InfoSchema), handle.WithTableStatsByQuery()) } -func (e *AnalyzeExec) genGlobalStats(globalStatsMap map[globalStatsKey]globalStatsInfo) error { - statsHandle := domain.GetDomain(e.ctx).StatsHandle() - for globalStatsID, info := range globalStatsMap { - globalOpts := e.opts - if e.OptionsMap != nil { - if v2Options, ok := e.OptionsMap[globalStatsID.tableID]; ok { - globalOpts = v2Options.FilledOpts - } - } - globalStats, err := statsHandle.MergePartitionStats2GlobalStatsByTableID(e.ctx, globalOpts, e.ctx.GetInfoSchema().(infoschema.InfoSchema), globalStatsID.tableID, info.isIndex, info.histIDs) - if err != nil { - if types.ErrPartitionStatsMissing.Equal(err) || types.ErrPartitionColumnStatsMissing.Equal(err) { - // When we find some partition-level stats are missing, we need to report warning. - e.ctx.GetSessionVars().StmtCtx.AppendWarning(err) - continue - } - return err - } - for i := 0; i < globalStats.Num; i++ { - hg, cms, topN, fms := globalStats.Hg[i], globalStats.Cms[i], globalStats.TopN[i], globalStats.Fms[i] - // fms for global stats doesn't need to dump to kv. - err = statsHandle.SaveStatsToStorage(globalStatsID.tableID, globalStats.Count, info.isIndex, hg, cms, topN, fms, info.statsVersion, 1, false, true) - if err != nil { - logutil.BgLogger().Error("save global-level stats to storage failed", zap.Error(err)) - } - // Dump stats to historical storage. - if err := e.recordHistoricalStats(globalStatsID.tableID); err != nil { - logutil.BgLogger().Error("record historical stats failed", zap.Error(err)) - } - } - } - return nil -} - func finishJobWithLog(sctx sessionctx.Context, job *statistics.AnalyzeJob, analyzeErr error) { FinishAnalyzeJob(sctx, job, analyzeErr) if job != nil { diff --git a/executor/analyze_test.go b/executor/analyze_test.go index ec47f97a71db1..eb37798391248 100644 --- a/executor/analyze_test.go +++ b/executor/analyze_test.go @@ -2950,7 +2950,7 @@ PARTITION BY RANGE ( a ) ( tk.MustQuery("select * from t where b > 1 and c > 1") require.NoError(t, h.LoadNeededHistograms()) tbl = h.GetTableStats(tableInfo) - lastVersion = tbl.Version + require.Greater(t, tbl.Version, lastVersion) require.Equal(t, 2, len(tbl.Columns[tableInfo.Columns[0].ID].Buckets)) require.Equal(t, 2, len(tbl.Columns[tableInfo.Columns[0].ID].TopN.TopN)) require.Equal(t, 2, len(tbl.Columns[tableInfo.Columns[2].ID].Buckets)) @@ -3085,7 +3085,6 @@ PARTITION BY RANGE ( a ) ( require.NoError(t, h.LoadNeededHistograms()) tbl = h.GetTableStats(tableInfo) require.Greater(t, tbl.Version, lastVersion) - lastVersion = tbl.Version require.Equal(t, 2, len(tbl.Columns[tableInfo.Columns[1].ID].Buckets)) require.Equal(t, 2, len(tbl.Columns[tableInfo.Columns[3].ID].Buckets)) require.Equal(t, 1, len(tbl.Columns[tableInfo.Columns[1].ID].TopN.TopN)) From 89a6d1a57ad0b450959a2eecc043bd973e00d6f3 Mon Sep 17 00:00:00 2001 From: wuxiaoju Date: Tue, 17 May 2022 16:57:34 +0800 Subject: [PATCH 06/11] fix --- planner/core/planbuilder.go | 52 ++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index e52705103023c..4ddebde07191e 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -2214,32 +2214,7 @@ func (b *PlanBuilder) genV2AnalyzeOptions( colsInfoMap[tbl.TableInfo.ID] = tblColsInfo for _, id := range physicalIDs { if id != tbl.TableInfo.ID { - if !dynamicPrune { - parSavedOpts, parSavedColChoice, parSavedColList, err := b.getSavedAnalyzeOpts(id, tbl.TableInfo) - if err != nil { - return nil, nil, err - } - // merge partition level options with table level options firstly - savedOpts := mergeAnalyzeOptions(parSavedOpts, tblSavedOpts) - savedColChoice, savedColList := mergeColumnList(parSavedColChoice, parSavedColList, tblSavedColChoice, tblSavedColList) - // then merge statement level options - mergedOpts := mergeAnalyzeOptions(astOpts, savedOpts) - filledMergedOpts := fillAnalyzeOptionsV2(mergedOpts) - finalColChoice, mergedColList := mergeColumnList(astColChoice, astColList, savedColChoice, savedColList) - finalColsInfo, finalColList, err := b.getFullAnalyzeColumnsInfo(tbl, finalColChoice, mergedColList, predicateCols, mustAnalyzedCols, mustAllColumns, false) - if err != nil { - return nil, nil, err - } - parV2Options := V2AnalyzeOptions{ - PhyTableID: id, - RawOpts: mergedOpts, - FilledOpts: filledMergedOpts, - ColChoice: finalColChoice, - ColumnList: finalColList, - } - optionsMap[id] = parV2Options - colsInfoMap[id] = finalColsInfo - } else { + if dynamicPrune { parV2Options := V2AnalyzeOptions{ PhyTableID: id, RawOpts: tblOpts, @@ -2250,7 +2225,32 @@ func (b *PlanBuilder) genV2AnalyzeOptions( } optionsMap[id] = parV2Options colsInfoMap[id] = tblColsInfo + continue + } + parSavedOpts, parSavedColChoice, parSavedColList, err := b.getSavedAnalyzeOpts(id, tbl.TableInfo) + if err != nil { + return nil, nil, err + } + // merge partition level options with table level options firstly + savedOpts := mergeAnalyzeOptions(parSavedOpts, tblSavedOpts) + savedColChoice, savedColList := mergeColumnList(parSavedColChoice, parSavedColList, tblSavedColChoice, tblSavedColList) + // then merge statement level options + mergedOpts := mergeAnalyzeOptions(astOpts, savedOpts) + filledMergedOpts := fillAnalyzeOptionsV2(mergedOpts) + finalColChoice, mergedColList := mergeColumnList(astColChoice, astColList, savedColChoice, savedColList) + finalColsInfo, finalColList, err := b.getFullAnalyzeColumnsInfo(tbl, finalColChoice, mergedColList, predicateCols, mustAnalyzedCols, mustAllColumns, false) + if err != nil { + return nil, nil, err + } + parV2Options := V2AnalyzeOptions{ + PhyTableID: id, + RawOpts: mergedOpts, + FilledOpts: filledMergedOpts, + ColChoice: finalColChoice, + ColumnList: finalColList, } + optionsMap[id] = parV2Options + colsInfoMap[id] = finalColsInfo } } return optionsMap, colsInfoMap, nil From 3109a51cf93ffa164a9183e0b59557ddd5f45120 Mon Sep 17 00:00:00 2001 From: wuxiaoju Date: Tue, 17 May 2022 18:07:07 +0800 Subject: [PATCH 07/11] revert tests for hidden variable --- executor/show_test.go | 5 ----- statistics/handle/handle_test.go | 12 ------------ 2 files changed, 17 deletions(-) diff --git a/executor/show_test.go b/executor/show_test.go index f8772d3a4b50a..8772a4f732711 100644 --- a/executor/show_test.go +++ b/executor/show_test.go @@ -1588,11 +1588,6 @@ func TestShowVar(t *testing.T) { res = tk.MustQuery(showSQL) require.Len(t, res.Rows(), len(globalVars)) - // Test a known hidden variable. - res = tk.MustQuery("show variables like '" + variable.TiDBPartitionPruneMode + "'") - require.Len(t, res.Rows(), 0) - res = tk.MustQuery("show global variables like '" + variable.TiDBPartitionPruneMode + "'") - require.Len(t, res.Rows(), 0) // Test Hidden tx_read_ts res = tk.MustQuery("show variables like '%tx_read_ts'") require.Len(t, res.Rows(), 0) diff --git a/statistics/handle/handle_test.go b/statistics/handle/handle_test.go index ef2c662942ccd..d8b106c71e03f 100644 --- a/statistics/handle/handle_test.go +++ b/statistics/handle/handle_test.go @@ -1070,18 +1070,6 @@ partition by range (a) ( checkHealthy(60, 50, 66) } -func TestHideGlobalStatsSwitch(t *testing.T) { - // NOTICE: remove this test when this global-stats is GA. - store, clean := testkit.CreateMockStore(t) - defer clean() - tk := testkit.NewTestKit(t, store) - rs := tk.MustQuery("show variables").Rows() - for _, r := range rs { - require.NotEqual(t, "tidb_partition_prune_mode", strings.ToLower(r[0].(string))) - } - require.Len(t, tk.MustQuery("show variables where variable_name like '%tidb_partition_prune_mode%'").Rows(), 0) -} - func TestGlobalStatsData(t *testing.T) { store, dom, clean := testkit.CreateMockStoreAndDomain(t) defer clean() From 8c0e4babc80ecd777e64aabfd5b671f67d20e9e6 Mon Sep 17 00:00:00 2001 From: wuxiaoju Date: Wed, 18 May 2022 00:38:10 +0800 Subject: [PATCH 08/11] add ut for dynamic v1 --- executor/analyze_test.go | 55 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/executor/analyze_test.go b/executor/analyze_test.go index eb37798391248..9cb7f144e317f 100644 --- a/executor/analyze_test.go +++ b/executor/analyze_test.go @@ -3239,3 +3239,58 @@ PARTITION BY RANGE ( a ) ( tbl = h.GetTableStats(tableInfo) require.Equal(t, 3, len(tbl.Columns[tableInfo.Columns[2].ID].Buckets)) } + +func TestAnalyzePartitionUnderV1Dynamic(t *testing.T) { + store, dom, clean := testkit.CreateMockStoreAndDomain(t) + defer clean() + tk := testkit.NewTestKit(t, store) + originalVal := tk.MustQuery("select @@tidb_persist_analyze_options").Rows()[0][0].(string) + defer func() { + tk.MustExec(fmt.Sprintf("set global tidb_persist_analyze_options = %v", originalVal)) + }() + + tk.MustExec("use test") + tk.MustExec("set @@session.tidb_analyze_version = 1") + tk.MustExec("set @@session.tidb_partition_prune_mode = 'dynamic'") + createTable := `CREATE TABLE t (a int, b int, c varchar(10), d int, primary key(a), index idx(b)) +PARTITION BY RANGE ( a ) ( + PARTITION p0 VALUES LESS THAN (10), + PARTITION p1 VALUES LESS THAN (20) +)` + tk.MustExec(createTable) + tk.MustExec("insert into t values (1,1,1,1),(2,1,2,2),(3,1,3,3),(4,1,4,4),(5,1,5,5),(6,1,6,6),(7,7,7,7),(8,8,8,8),(9,9,9,9)") + tk.MustExec("insert into t values (10,10,10,10),(11,11,11,11),(12,12,12,12),(13,13,13,13),(14,14,14,14)") + h := dom.StatsHandle() + oriLease := h.Lease() + h.SetLease(1) + defer func() { + h.SetLease(oriLease) + }() + is := dom.InfoSchema() + table, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) + require.NoError(t, err) + tableInfo := table.Meta() + pi := tableInfo.GetPartitionInfo() + require.NotNil(t, pi) + + // analyze partition with index and with options are allowed under dynamic V1 + tk.MustExec("analyze table t partition p0 with 1 topn, 3 buckets") + tk.MustQuery("show warnings").Sort().Check(testkit.Rows( + "Warning 8131 Build table: `t` global-level stats failed due to missing partition-level stats", + "Warning 8131 Build table: `t` index: `idx` global-level stats failed due to missing partition-level stats", + )) + tk.MustExec("analyze table t partition p1 with 1 topn, 3 buckets") + tk.MustQuery("show warnings").Sort().Check(testkit.Rows()) + tk.MustQuery("select * from t where a > 1 and b > 1 and c > 1 and d > 1") + require.NoError(t, h.LoadNeededHistograms()) + tbl := h.GetTableStats(tableInfo) + lastVersion := tbl.Version + require.Equal(t, 3, len(tbl.Columns[tableInfo.Columns[2].ID].Buckets)) + require.Equal(t, 3, len(tbl.Columns[tableInfo.Columns[3].ID].Buckets)) + + tk.MustExec("analyze table t partition p1 index idx with 1 topn, 2 buckets") + tk.MustQuery("show warnings").Sort().Check(testkit.Rows()) + tbl = h.GetTableStats(tableInfo) + require.Greater(t, tbl.Version, lastVersion) + require.Equal(t, 2, len(tbl.Indices[tableInfo.Indices[0].ID].Buckets)) +} From 4ed825d66ac43da185e6cd2cbc503260f444278f Mon Sep 17 00:00:00 2001 From: wuxiaoju Date: Wed, 18 May 2022 11:38:23 +0800 Subject: [PATCH 09/11] fix errors.toml --- errors.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/errors.toml b/errors.toml index 405b7aab6d34f..d8112d297e6fe 100755 --- a/errors.toml +++ b/errors.toml @@ -2586,6 +2586,11 @@ error = ''' Build table: %s global-level stats failed due to missing partition-level stats ''' +["types:8244"] +error = ''' +Build table: %s global-level stats failed due to missing partition-level column stats, please run analyze table to refresh columns of all partitions +''' + ["variable:1193"] error = ''' Unknown system variable '%-.64s' From 5d3b3843c30315c8d6554d9b50fe76d7b6fd83e8 Mon Sep 17 00:00:00 2001 From: Xiaoju Wu Date: Wed, 18 May 2022 21:32:46 +0800 Subject: [PATCH 10/11] Update executor/analyze_test.go Co-authored-by: Song Gao --- executor/analyze_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/executor/analyze_test.go b/executor/analyze_test.go index 9cb7f144e317f..35d16097ff3c7 100644 --- a/executor/analyze_test.go +++ b/executor/analyze_test.go @@ -3025,7 +3025,7 @@ PARTITION BY RANGE ( a ) ( tk.MustExec("set @@session.tidb_partition_prune_mode = 'dynamic'") - // analyze table and dynamic mode will ignore partition-level options and use default + // analyze table in dynamic mode will ignore partition-level options and use default tk.MustExec("analyze table t") tk.MustQuery("select * from t where b > 1 and c > 1") require.NoError(t, h.LoadNeededHistograms()) From fca3ecc7e79f3df39d6c12f3859bdbe1d4c4add3 Mon Sep 17 00:00:00 2001 From: wuxiaoju Date: Thu, 19 May 2022 00:16:29 +0800 Subject: [PATCH 11/11] address comments --- executor/analyze.go | 2 +- executor/analyze_test.go | 263 +++++++++++++++++------------------ parser/model/model.go | 28 ++++ planner/core/common_plans.go | 12 +- planner/core/planbuilder.go | 24 ++-- statistics/handle/handle.go | 15 +- 6 files changed, 179 insertions(+), 165 deletions(-) diff --git a/executor/analyze.go b/executor/analyze.go index a13e4570e2c70..69cbd76ef1d37 100644 --- a/executor/analyze.go +++ b/executor/analyze.go @@ -266,7 +266,7 @@ func (e *AnalyzeExec) saveV2AnalyzeOpts() error { dynamicPrune := variable.PartitionPruneMode(e.ctx.GetSessionVars().PartitionPruneMode.Load()) == variable.Dynamic toSaveMap := make(map[int64]core.V2AnalyzeOptions) for id, opts := range e.OptionsMap { - if opts.IsTable || !dynamicPrune { + if !opts.IsPartition || !dynamicPrune { toSaveMap[id] = opts } } diff --git a/executor/analyze_test.go b/executor/analyze_test.go index 35d16097ff3c7..10c9471110823 100644 --- a/executor/analyze_test.go +++ b/executor/analyze_test.go @@ -1276,11 +1276,10 @@ func TestSavedAnalyzeOptions(t *testing.T) { require.Equal(t, 2, len(col1.Buckets)) col2 := tbl.Columns[tableInfo.Columns[2].ID] require.Equal(t, 2, len(col2.Buckets)) - // The columns are: table_id, sample_num, sample_rate, buckets, topn, column_choice, column_ids. - rs := tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(tbl.PhysicalID, 10)) + rs := tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(tbl.PhysicalID, 10)) require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "2", rs.Rows()[0][3]) - require.Equal(t, "1", rs.Rows()[0][4]) + require.Equal(t, "2", rs.Rows()[0][0]) + require.Equal(t, "1", rs.Rows()[0][1]) // auto-analyze uses the table-level options tk.MustExec("insert into t values (10,10,10)") @@ -1306,15 +1305,14 @@ func TestSavedAnalyzeOptions(t *testing.T) { require.Equal(t, 1, len(col1.TopN.TopN)) col2 = tbl.Columns[tableInfo.Columns[2].ID] require.Less(t, col2.LastUpdateVersion, col0.LastUpdateVersion) // not updated since removed from list - // The columns are: table_id, sample_num, sample_rate, buckets, topn, column_choice, column_ids. - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(tbl.PhysicalID, 10)) + rs = tk.MustQuery("select sample_rate,buckets,topn,column_choice,column_ids from mysql.analyze_options where table_id=" + strconv.FormatInt(tbl.PhysicalID, 10)) require.Equal(t, 1, len(rs.Rows())) + require.Equal(t, "1", rs.Rows()[0][0]) + require.Equal(t, "3", rs.Rows()[0][1]) require.Equal(t, "1", rs.Rows()[0][2]) - require.Equal(t, "3", rs.Rows()[0][3]) - require.Equal(t, "1", rs.Rows()[0][4]) - require.Equal(t, "LIST", rs.Rows()[0][5]) + require.Equal(t, "LIST", rs.Rows()[0][3]) colIDStrs := strings.Join([]string{strconv.FormatInt(tableInfo.Columns[0].ID, 10), strconv.FormatInt(tableInfo.Columns[1].ID, 10)}, ",") - require.Equal(t, colIDStrs, rs.Rows()[0][6]) + require.Equal(t, colIDStrs, rs.Rows()[0][4]) // disable option persistence tk.MustExec("set global tidb_persist_analyze_options = false") @@ -1324,10 +1322,9 @@ func TestSavedAnalyzeOptions(t *testing.T) { require.Greater(t, tbl.Version, lastVersion) col0 = tbl.Columns[tableInfo.Columns[0].ID] require.NotEqual(t, 3, len(col0.Buckets)) - // The columns are: table_id, sample_num, sample_rate, buckets, topn, column_choice, column_ids. - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(tbl.PhysicalID, 10)) + rs = tk.MustQuery("select topn from mysql.analyze_options where table_id=" + strconv.FormatInt(tbl.PhysicalID, 10)) require.Equal(t, 1, len(rs.Rows())) - require.NotEqual(t, "2", rs.Rows()[0][4]) + require.NotEqual(t, "2", rs.Rows()[0][0]) } func TestSavedPartitionAnalyzeOptions(t *testing.T) { @@ -1370,15 +1367,14 @@ PARTITION BY RANGE ( a ) ( p0 := h.GetPartitionStats(tableInfo, pi.Definitions[0].ID) lastVersion := p0.Version require.Equal(t, 3, len(p0.Columns[tableInfo.Columns[0].ID].Buckets)) - // The columns are: table_id, sample_num, sample_rate, buckets, topn, column_choice, column_ids. - rs := tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(p0.PhysicalID, 10)) + rs := tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(p0.PhysicalID, 10)) require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "3", rs.Rows()[0][3]) - require.Equal(t, "1", rs.Rows()[0][4]) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) + require.Equal(t, "3", rs.Rows()[0][0]) + require.Equal(t, "1", rs.Rows()[0][1]) + rs = tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "0", rs.Rows()[0][3]) - require.Equal(t, "-1", rs.Rows()[0][4]) + require.Equal(t, "0", rs.Rows()[0][0]) + require.Equal(t, "-1", rs.Rows()[0][1]) // merge partition & table level options tk.MustExec("analyze table t columns a,b with 0 topn, 2 buckets") @@ -1393,29 +1389,28 @@ PARTITION BY RANGE ( a ) ( // check column c is not analyzed require.Less(t, p0.Columns[tableInfo.Columns[2].ID].LastUpdateVersion, p0.Columns[tableInfo.Columns[0].ID].LastUpdateVersion) require.Less(t, p1.Columns[tableInfo.Columns[2].ID].LastUpdateVersion, p1.Columns[tableInfo.Columns[0].ID].LastUpdateVersion) - // The columns are: table_id, sample_num, sample_rate, buckets, topn, column_choice, column_ids. - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) + rs = tk.MustQuery("select sample_rate,buckets,topn,column_choice,column_ids from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) require.Equal(t, 1, len(rs.Rows())) + require.Equal(t, "0", rs.Rows()[0][0]) + require.Equal(t, "2", rs.Rows()[0][1]) require.Equal(t, "0", rs.Rows()[0][2]) - require.Equal(t, "2", rs.Rows()[0][3]) - require.Equal(t, "0", rs.Rows()[0][4]) - require.Equal(t, "LIST", rs.Rows()[0][5]) + require.Equal(t, "LIST", rs.Rows()[0][3]) colIDStrsAB := strings.Join([]string{strconv.FormatInt(tableInfo.Columns[0].ID, 10), strconv.FormatInt(tableInfo.Columns[1].ID, 10)}, ",") - require.Equal(t, colIDStrsAB, rs.Rows()[0][6]) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(p0.PhysicalID, 10)) + require.Equal(t, colIDStrsAB, rs.Rows()[0][4]) + rs = tk.MustQuery("select sample_rate,buckets,topn,column_choice,column_ids from mysql.analyze_options where table_id=" + strconv.FormatInt(p0.PhysicalID, 10)) require.Equal(t, 1, len(rs.Rows())) + require.Equal(t, "0", rs.Rows()[0][0]) + require.Equal(t, "2", rs.Rows()[0][1]) require.Equal(t, "0", rs.Rows()[0][2]) - require.Equal(t, "2", rs.Rows()[0][3]) - require.Equal(t, "0", rs.Rows()[0][4]) - require.Equal(t, "LIST", rs.Rows()[0][5]) - require.Equal(t, colIDStrsAB, rs.Rows()[0][6]) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(p1.PhysicalID, 10)) + require.Equal(t, "LIST", rs.Rows()[0][3]) + require.Equal(t, colIDStrsAB, rs.Rows()[0][4]) + rs = tk.MustQuery("select sample_rate,buckets,topn,column_choice,column_ids from mysql.analyze_options where table_id=" + strconv.FormatInt(p1.PhysicalID, 10)) require.Equal(t, 1, len(rs.Rows())) + require.Equal(t, "0", rs.Rows()[0][0]) + require.Equal(t, "2", rs.Rows()[0][1]) require.Equal(t, "0", rs.Rows()[0][2]) - require.Equal(t, "2", rs.Rows()[0][3]) - require.Equal(t, "0", rs.Rows()[0][4]) - require.Equal(t, "LIST", rs.Rows()[0][5]) - require.Equal(t, colIDStrsAB, rs.Rows()[0][6]) + require.Equal(t, "LIST", rs.Rows()[0][3]) + require.Equal(t, colIDStrsAB, rs.Rows()[0][4]) // analyze partition only updates this partition, and set different collist tk.MustExec("analyze table t partition p1 columns a,c with 1 buckets") @@ -1432,19 +1427,18 @@ PARTITION BY RANGE ( a ) ( require.Equal(t, 1, len(p1.Columns[tableInfo.Columns[2].ID].Buckets)) require.NotEqual(t, 1, len(p0.Columns[tableInfo.Columns[2].ID].Buckets)) colIDStrsABC := strings.Join([]string{strconv.FormatInt(tableInfo.Columns[0].ID, 10), strconv.FormatInt(tableInfo.Columns[1].ID, 10), strconv.FormatInt(tableInfo.Columns[2].ID, 10)}, ",") - // The columns are: table_id, sample_num, sample_rate, buckets, topn, column_choice, column_ids. - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) + rs = tk.MustQuery("select buckets,column_ids from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "2", rs.Rows()[0][3]) - require.Equal(t, colIDStrsAB, rs.Rows()[0][6]) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(p1.PhysicalID, 10)) + require.Equal(t, "2", rs.Rows()[0][0]) + require.Equal(t, colIDStrsAB, rs.Rows()[0][1]) + rs = tk.MustQuery("select buckets,column_ids from mysql.analyze_options where table_id=" + strconv.FormatInt(p1.PhysicalID, 10)) require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "1", rs.Rows()[0][3]) - require.Equal(t, colIDStrsABC, rs.Rows()[0][6]) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(p0.PhysicalID, 10)) + require.Equal(t, "1", rs.Rows()[0][0]) + require.Equal(t, colIDStrsABC, rs.Rows()[0][1]) + rs = tk.MustQuery("select buckets,column_ids from mysql.analyze_options where table_id=" + strconv.FormatInt(p0.PhysicalID, 10)) require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "2", rs.Rows()[0][3]) - require.Equal(t, colIDStrsAB, rs.Rows()[0][6]) + require.Equal(t, "2", rs.Rows()[0][0]) + require.Equal(t, colIDStrsAB, rs.Rows()[0][1]) // analyze partition without options uses saved partition options tk.MustExec("analyze table t partition p0") @@ -1452,26 +1446,25 @@ PARTITION BY RANGE ( a ) ( require.Greater(t, p0.Version, lastVersion) lastVersion = p0.Version require.Equal(t, 2, len(p0.Columns[tableInfo.Columns[0].ID].Buckets)) - // The columns are: table_id, sample_num, sample_rate, buckets, topn, column_choice, column_ids. - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) + rs = tk.MustQuery("select buckets from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "2", rs.Rows()[0][3]) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(p0.PhysicalID, 10)) + require.Equal(t, "2", rs.Rows()[0][0]) + rs = tk.MustQuery("select buckets from mysql.analyze_options where table_id=" + strconv.FormatInt(p0.PhysicalID, 10)) require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "2", rs.Rows()[0][3]) + require.Equal(t, "2", rs.Rows()[0][0]) // merge options of statement's, partition's and table's tk.MustExec("analyze table t partition p0 with 3 buckets") p0 = h.GetPartitionStats(tableInfo, pi.Definitions[0].ID) require.Greater(t, p0.Version, lastVersion) require.Equal(t, 3, len(p0.Columns[tableInfo.Columns[0].ID].Buckets)) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(p0.PhysicalID, 10)) + rs = tk.MustQuery("select sample_rate,buckets,topn,column_choice,column_ids from mysql.analyze_options where table_id=" + strconv.FormatInt(p0.PhysicalID, 10)) require.Equal(t, 1, len(rs.Rows())) + require.Equal(t, "0", rs.Rows()[0][0]) + require.Equal(t, "3", rs.Rows()[0][1]) require.Equal(t, "0", rs.Rows()[0][2]) - require.Equal(t, "3", rs.Rows()[0][3]) - require.Equal(t, "0", rs.Rows()[0][4]) - require.Equal(t, "LIST", rs.Rows()[0][5]) - require.Equal(t, colIDStrsAB, rs.Rows()[0][6]) + require.Equal(t, "LIST", rs.Rows()[0][3]) + require.Equal(t, colIDStrsAB, rs.Rows()[0][4]) // add new partitions, use table options as default tk.MustExec("ALTER TABLE t ADD PARTITION (PARTITION p2 VALUES LESS THAN (30))") @@ -1484,20 +1477,20 @@ PARTITION BY RANGE ( a ) ( pi = tableInfo.GetPartitionInfo() p2 := h.GetPartitionStats(tableInfo, pi.Definitions[2].ID) require.Equal(t, 2, len(p2.Columns[tableInfo.Columns[0].ID].Buckets)) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(p2.PhysicalID, 10)) + rs = tk.MustQuery("select sample_rate,buckets,topn,column_choice,column_ids from mysql.analyze_options where table_id=" + strconv.FormatInt(p2.PhysicalID, 10)) require.Equal(t, 1, len(rs.Rows())) + require.Equal(t, "0", rs.Rows()[0][0]) + require.Equal(t, "2", rs.Rows()[0][1]) require.Equal(t, "0", rs.Rows()[0][2]) - require.Equal(t, "2", rs.Rows()[0][3]) - require.Equal(t, "0", rs.Rows()[0][4]) - require.Equal(t, "LIST", rs.Rows()[0][5]) - require.Equal(t, colIDStrsAB, rs.Rows()[0][6]) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) + require.Equal(t, "LIST", rs.Rows()[0][3]) + require.Equal(t, colIDStrsAB, rs.Rows()[0][4]) + rs = tk.MustQuery("select sample_rate,buckets,topn,column_choice,column_ids from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) require.Equal(t, 1, len(rs.Rows())) + require.Equal(t, "0", rs.Rows()[0][0]) + require.Equal(t, "2", rs.Rows()[0][1]) require.Equal(t, "0", rs.Rows()[0][2]) - require.Equal(t, "2", rs.Rows()[0][3]) - require.Equal(t, "0", rs.Rows()[0][4]) - require.Equal(t, "LIST", rs.Rows()[0][5]) - require.Equal(t, colIDStrsAB, rs.Rows()[0][6]) + require.Equal(t, "LIST", rs.Rows()[0][3]) + require.Equal(t, colIDStrsAB, rs.Rows()[0][4]) // set analyze version back to 1, will not use persisted tk.MustExec("set @@session.tidb_analyze_version = 1") @@ -1512,15 +1505,15 @@ PARTITION BY RANGE ( a ) ( tk.MustExec("analyze table t") colIDStrsA := strings.Join([]string{strconv.FormatInt(tableInfo.Columns[0].ID, 10)}, ",") colIDStrsAC := strings.Join([]string{strconv.FormatInt(tableInfo.Columns[0].ID, 10), strconv.FormatInt(tableInfo.Columns[2].ID, 10)}, ",") - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) + rs = tk.MustQuery("select column_ids from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, colIDStrsA, rs.Rows()[0][6]) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(p0.PhysicalID, 10)) + require.Equal(t, colIDStrsA, rs.Rows()[0][0]) + rs = tk.MustQuery("select column_ids from mysql.analyze_options where table_id=" + strconv.FormatInt(p0.PhysicalID, 10)) require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, colIDStrsA, rs.Rows()[0][6]) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(p1.PhysicalID, 10)) + require.Equal(t, colIDStrsA, rs.Rows()[0][0]) + rs = tk.MustQuery("select column_ids from mysql.analyze_options where table_id=" + strconv.FormatInt(p1.PhysicalID, 10)) require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, colIDStrsAC, rs.Rows()[0][6]) + require.Equal(t, colIDStrsAC, rs.Rows()[0][0]) // drop partition tk.MustExec("alter table t drop partition p1") @@ -1584,15 +1577,14 @@ func TestSavedAnalyzeOptionsForMultipleTables(t *testing.T) { tbl2Col0 := tblStats2.Columns[tableInfo2.Columns[0].ID] require.Equal(t, 3, len(tbl1Col0.Buckets)) require.Equal(t, 2, len(tbl2Col0.Buckets)) - // The columns are: table_id, sample_num, sample_rate, buckets, topn, column_choice, column_ids. - rs := tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo1.ID, 10)) + rs := tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo1.ID, 10)) require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "3", rs.Rows()[0][3]) - require.Equal(t, "2", rs.Rows()[0][4]) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo2.ID, 10)) + require.Equal(t, "3", rs.Rows()[0][0]) + require.Equal(t, "2", rs.Rows()[0][1]) + rs = tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo2.ID, 10)) require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "2", rs.Rows()[0][3]) - require.Equal(t, "2", rs.Rows()[0][4]) + require.Equal(t, "2", rs.Rows()[0][0]) + require.Equal(t, "2", rs.Rows()[0][1]) } func TestSavedAnalyzeColumnOptions(t *testing.T) { @@ -2914,15 +2906,14 @@ PARTITION BY RANGE ( a ) ( require.Equal(t, 1, len(tbl.Columns[tableInfo.Columns[0].ID].TopN.TopN)) require.Equal(t, 3, len(tbl.Columns[tableInfo.Columns[2].ID].Buckets)) require.Equal(t, 1, len(tbl.Columns[tableInfo.Columns[2].ID].TopN.TopN)) - rs := tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[0].ID, 10)) + rs := tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[0].ID, 10)) require.Equal(t, 0, len(rs.Rows())) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[1].ID, 10)) + rs = tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[1].ID, 10)) require.Equal(t, 0, len(rs.Rows())) - // The columns are: table_id, sample_num, sample_rate, buckets, topn, column_choice, column_ids. - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) + rs = tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "3", rs.Rows()[0][3]) - require.Equal(t, "1", rs.Rows()[0][4]) + require.Equal(t, "3", rs.Rows()[0][0]) + require.Equal(t, "1", rs.Rows()[0][1]) // analyze table with persisted table-level options tk.MustExec("analyze table t") @@ -2935,15 +2926,14 @@ PARTITION BY RANGE ( a ) ( require.Equal(t, 1, len(tbl.Columns[tableInfo.Columns[0].ID].TopN.TopN)) require.Equal(t, 3, len(tbl.Columns[tableInfo.Columns[2].ID].Buckets)) require.Equal(t, 1, len(tbl.Columns[tableInfo.Columns[2].ID].TopN.TopN)) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[0].ID, 10)) + rs = tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[0].ID, 10)) require.Equal(t, 0, len(rs.Rows())) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[1].ID, 10)) + rs = tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[1].ID, 10)) require.Equal(t, 0, len(rs.Rows())) - // The columns are: table_id, sample_num, sample_rate, buckets, topn, column_choice, column_ids. - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) + rs = tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "3", rs.Rows()[0][3]) - require.Equal(t, "1", rs.Rows()[0][4]) + require.Equal(t, "3", rs.Rows()[0][0]) + require.Equal(t, "1", rs.Rows()[0][1]) // analyze table with merged table-level options tk.MustExec("analyze table t with 2 topn, 2 buckets") @@ -2955,15 +2945,14 @@ PARTITION BY RANGE ( a ) ( require.Equal(t, 2, len(tbl.Columns[tableInfo.Columns[0].ID].TopN.TopN)) require.Equal(t, 2, len(tbl.Columns[tableInfo.Columns[2].ID].Buckets)) require.Equal(t, 2, len(tbl.Columns[tableInfo.Columns[2].ID].TopN.TopN)) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[0].ID, 10)) + rs = tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[0].ID, 10)) require.Equal(t, 0, len(rs.Rows())) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[1].ID, 10)) + rs = tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[1].ID, 10)) require.Equal(t, 0, len(rs.Rows())) - // The columns are: table_id, sample_num, sample_rate, buckets, topn, column_choice, column_ids. - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) + rs = tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "2", rs.Rows()[0][3]) - require.Equal(t, "2", rs.Rows()[0][4]) + require.Equal(t, "2", rs.Rows()[0][0]) + require.Equal(t, "2", rs.Rows()[0][1]) } func TestAnalyzePartitionTableStaticToDynamic(t *testing.T) { @@ -3012,16 +3001,16 @@ PARTITION BY RANGE ( a ) ( require.Equal(t, 3, len(p0.Columns[tableInfo.Columns[2].ID].Buckets)) require.Equal(t, 0, len(p1.Columns[tableInfo.Columns[0].ID].Buckets)) require.Equal(t, 0, len(tbl.Columns[tableInfo.Columns[0].ID].Buckets)) - rs := tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[0].ID, 10)) + rs := tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[0].ID, 10)) require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "3", rs.Rows()[0][3]) - require.Equal(t, "1", rs.Rows()[0][4]) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[1].ID, 10)) + require.Equal(t, "3", rs.Rows()[0][0]) + require.Equal(t, "1", rs.Rows()[0][1]) + rs = tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[1].ID, 10)) require.Equal(t, 0, len(rs.Rows())) // The columns are: table_id, sample_num, sample_rate, buckets, topn, column_choice, column_ids. - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) + rs = tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "0", rs.Rows()[0][3]) + require.Equal(t, "0", rs.Rows()[0][0]) tk.MustExec("set @@session.tidb_partition_prune_mode = 'dynamic'") @@ -3037,16 +3026,15 @@ PARTITION BY RANGE ( a ) ( require.NotEqual(t, 3, len(p0.Columns[tableInfo.Columns[0].ID].Buckets)) require.Equal(t, len(tbl.Columns[tableInfo.Columns[0].ID].Buckets), len(p0.Columns[tableInfo.Columns[0].ID].Buckets)) require.Equal(t, len(tbl.Columns[tableInfo.Columns[0].ID].Buckets), len(p1.Columns[tableInfo.Columns[0].ID].Buckets)) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[0].ID, 10)) + rs = tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[0].ID, 10)) require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "3", rs.Rows()[0][3]) - require.Equal(t, "1", rs.Rows()[0][4]) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[1].ID, 10)) + require.Equal(t, "3", rs.Rows()[0][0]) + require.Equal(t, "1", rs.Rows()[0][1]) + rs = tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[1].ID, 10)) require.Equal(t, 0, len(rs.Rows())) - // The columns are: table_id, sample_num, sample_rate, buckets, topn, column_choice, column_ids. - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) + rs = tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "0", rs.Rows()[0][3]) + require.Equal(t, "0", rs.Rows()[0][0]) // analyze table under dynamic mode with specified options with old partition-level options tk.MustExec("analyze table t columns b,d with 2 topn, 2 buckets") @@ -3057,17 +3045,16 @@ PARTITION BY RANGE ( a ) ( lastVersion = tbl.Version require.Equal(t, 2, len(tbl.Columns[tableInfo.Columns[1].ID].Buckets)) require.Equal(t, 2, len(tbl.Columns[tableInfo.Columns[3].ID].Buckets)) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[0].ID, 10)) + rs = tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[0].ID, 10)) require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "3", rs.Rows()[0][3]) - require.Equal(t, "1", rs.Rows()[0][4]) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[1].ID, 10)) + require.Equal(t, "3", rs.Rows()[0][0]) + require.Equal(t, "1", rs.Rows()[0][1]) + rs = tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[1].ID, 10)) require.Equal(t, 0, len(rs.Rows())) - // The columns are: table_id, sample_num, sample_rate, buckets, topn, column_choice, column_ids. - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) + rs = tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "2", rs.Rows()[0][3]) - require.Equal(t, "2", rs.Rows()[0][4]) + require.Equal(t, "2", rs.Rows()[0][0]) + require.Equal(t, "2", rs.Rows()[0][1]) // analyze table under dynamic mode without options with old table-level & partition-level options tk.MustExec("analyze table t") @@ -3089,17 +3076,16 @@ PARTITION BY RANGE ( a ) ( require.Equal(t, 2, len(tbl.Columns[tableInfo.Columns[3].ID].Buckets)) require.Equal(t, 1, len(tbl.Columns[tableInfo.Columns[1].ID].TopN.TopN)) require.Equal(t, 1, len(tbl.Columns[tableInfo.Columns[3].ID].TopN.TopN)) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[0].ID, 10)) + rs = tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[0].ID, 10)) require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "3", rs.Rows()[0][3]) - require.Equal(t, "1", rs.Rows()[0][4]) - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[1].ID, 10)) + require.Equal(t, "3", rs.Rows()[0][0]) + require.Equal(t, "1", rs.Rows()[0][1]) + rs = tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(pi.Definitions[1].ID, 10)) require.Equal(t, 0, len(rs.Rows())) - // The columns are: table_id, sample_num, sample_rate, buckets, topn, column_choice, column_ids. - rs = tk.MustQuery("select * from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) + rs = tk.MustQuery("select buckets,topn from mysql.analyze_options where table_id=" + strconv.FormatInt(tableInfo.ID, 10)) require.Equal(t, 1, len(rs.Rows())) - require.Equal(t, "2", rs.Rows()[0][3]) - require.Equal(t, "1", rs.Rows()[0][4]) + require.Equal(t, "2", rs.Rows()[0][0]) + require.Equal(t, "1", rs.Rows()[0][1]) } func TestAnalyzePartitionUnderDynamic(t *testing.T) { @@ -3202,42 +3188,43 @@ PARTITION BY RANGE ( a ) ( p0 := h.GetPartitionStats(tableInfo, pi.Definitions[0].ID) require.Equal(t, 3, len(p0.Columns[tableInfo.Columns[2].ID].Buckets)) - // analyze partition with existing stats of other partitions + // analyze partition with existing stats of other partitions under dynamic tk.MustExec("set @@session.tidb_partition_prune_mode = 'dynamic'") tk.MustExec("analyze table t partition p1 columns a,b,d with 1 topn, 3 buckets") tk.MustQuery("show warnings").Sort().Check(testkit.Rows( "Note 1105 Analyze use auto adjusted sample rate 1.000000 for table test.t's partition p1", - "Warning 8244 Build table: `t` global-level stats failed due to missing partition-level column stats, please run analyze table to refresh columns of all partitions", + "Warning 8244 Build table: `t` column: `d` global-level stats failed due to missing partition-level column stats, please run analyze table to refresh columns of all partitions", )) - // insert table-level persisted analyze options + // analyze partition with existing table-level options and existing partition stats under dynamic tk.MustExec("insert into mysql.analyze_options values (?,?,?,?,?,?,?)", tableInfo.ID, 0, 0, 2, 2, "DEFAULT", "") + tk.MustExec("set global tidb_persist_analyze_options = true") tk.MustExec("analyze table t partition p1 columns a,b,d with 1 topn, 3 buckets") tk.MustQuery("show warnings").Sort().Check(testkit.Rows( "Note 1105 Analyze use auto adjusted sample rate 1.000000 for table test.t's partition p1", - "Warning 8244 Build table: `t` global-level stats failed due to missing partition-level column stats, please run analyze table to refresh columns of all partitions", + "Warning 1105 Ignore columns and options when analyze partition in dynamic mode", + "Warning 8244 Build table: `t` column: `d` global-level stats failed due to missing partition-level column stats, please run analyze table to refresh columns of all partitions", )) - // insert partition-level persisted analyze options - tk.MustExec("insert into mysql.analyze_options values (?,?,?,?,?,?,?)", pi.Definitions[0].ID, 0, 0, 2, 2, "DEFAULT", "") + // analyze partition with existing table-level & partition-level options and existing partition stats under dynamic + tk.MustExec("insert into mysql.analyze_options values (?,?,?,?,?,?,?)", pi.Definitions[1].ID, 0, 0, 1, 1, "DEFAULT", "") tk.MustExec("analyze table t partition p1 columns a,b,d with 1 topn, 3 buckets") tk.MustQuery("show warnings").Sort().Check(testkit.Rows( "Note 1105 Analyze use auto adjusted sample rate 1.000000 for table test.t's partition p1", - "Warning 8244 Build table: `t` global-level stats failed due to missing partition-level column stats, please run analyze table to refresh columns of all partitions", + "Warning 1105 Ignore columns and options when analyze partition in dynamic mode", + "Warning 8244 Build table: `t` column: `d` global-level stats failed due to missing partition-level column stats, please run analyze table to refresh columns of all partitions", )) tk.MustQuery("select * from t where a > 1 and b > 1 and c > 1 and d > 1") require.NoError(t, h.LoadNeededHistograms()) tbl := h.GetTableStats(tableInfo) require.Equal(t, 0, len(tbl.Columns)) - tk.MustExec("analyze table t partition p1 columns a,b,c with 1 topn, 3 buckets") - tk.MustQuery("show warnings").Sort().Check(testkit.Rows( - "Note 1105 Analyze use auto adjusted sample rate 1.000000 for table test.t's partition p1", - )) + // ignore both p0's 3 buckets, persisted-partition-options' 1 bucket, just use table-level 2 buckets + tk.MustExec("analyze table t partition p0") tk.MustQuery("select * from t where a > 1 and b > 1 and c > 1 and d > 1") require.NoError(t, h.LoadNeededHistograms()) tbl = h.GetTableStats(tableInfo) - require.Equal(t, 3, len(tbl.Columns[tableInfo.Columns[2].ID].Buckets)) + require.Equal(t, 2, len(tbl.Columns[tableInfo.Columns[2].ID].Buckets)) } func TestAnalyzePartitionUnderV1Dynamic(t *testing.T) { diff --git a/parser/model/model.go b/parser/model/model.go index 5da704cc01831..0fdfe50fd8dfb 100644 --- a/parser/model/model.go +++ b/parser/model/model.go @@ -299,6 +299,16 @@ func FindColumnInfoByID(cols []*ColumnInfo, id int64) *ColumnInfo { return nil } +// FindIndexInfoByID finds IndexInfo in indices by id. +func FindIndexInfoByID(indices []*IndexInfo, id int64) *IndexInfo { + for _, idx := range indices { + if idx.ID == id { + return idx + } + } + return nil +} + // ExtraHandleID is the column ID of column which we need to append to schema to occupy the handle's position // for use of execution phase. const ExtraHandleID = -1 @@ -1224,6 +1234,24 @@ func (t *TableInfo) FindConstraintInfoByName(constrName string) *ConstraintInfo return nil } +// FindIndexNameByID finds index name by id. +func (t *TableInfo) FindIndexNameByID(id int64) string { + indexInfo := FindIndexInfoByID(t.Indices, id) + if indexInfo != nil { + return indexInfo.Name.L + } + return "" +} + +// FindColumnNameByID finds column name by id. +func (t *TableInfo) FindColumnNameByID(id int64) string { + colInfo := FindColumnInfoByID(t.Columns, id) + if colInfo != nil { + return colInfo.Name.L + } + return "" +} + // FKInfo provides meta data describing a foreign key constraint. type FKInfo struct { ID int64 `json:"id"` diff --git a/planner/core/common_plans.go b/planner/core/common_plans.go index 049b601470415..bd1accf2d4a8d 100644 --- a/planner/core/common_plans.go +++ b/planner/core/common_plans.go @@ -1115,12 +1115,12 @@ type AnalyzeInfo struct { // V2AnalyzeOptions is used to hold analyze options information. type V2AnalyzeOptions struct { - PhyTableID int64 - RawOpts map[ast.AnalyzeOptionType]uint64 - FilledOpts map[ast.AnalyzeOptionType]uint64 - ColChoice model.ColumnChoice - ColumnList []*model.ColumnInfo - IsTable bool + PhyTableID int64 + RawOpts map[ast.AnalyzeOptionType]uint64 + FilledOpts map[ast.AnalyzeOptionType]uint64 + ColChoice model.ColumnChoice + ColumnList []*model.ColumnInfo + IsPartition bool } // AnalyzeColumnsTask is used for analyze columns. diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index 4ddebde07191e..9d67a6499a656 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -2203,12 +2203,12 @@ func (b *PlanBuilder) genV2AnalyzeOptions( return nil, nil, err } tblAnalyzeOptions := V2AnalyzeOptions{ - PhyTableID: tbl.TableInfo.ID, - RawOpts: tblOpts, - FilledOpts: tblFilledOpts, - ColChoice: tblColChoice, - ColumnList: tblColList, - IsTable: true, + PhyTableID: tbl.TableInfo.ID, + RawOpts: tblOpts, + FilledOpts: tblFilledOpts, + ColChoice: tblColChoice, + ColumnList: tblColList, + IsPartition: false, } optionsMap[tbl.TableInfo.ID] = tblAnalyzeOptions colsInfoMap[tbl.TableInfo.ID] = tblColsInfo @@ -2216,12 +2216,12 @@ func (b *PlanBuilder) genV2AnalyzeOptions( if id != tbl.TableInfo.ID { if dynamicPrune { parV2Options := V2AnalyzeOptions{ - PhyTableID: id, - RawOpts: tblOpts, - FilledOpts: tblFilledOpts, - ColChoice: tblColChoice, - ColumnList: tblColList, - IsTable: false, + PhyTableID: id, + RawOpts: tblOpts, + FilledOpts: tblFilledOpts, + ColChoice: tblColChoice, + ColumnList: tblColList, + IsPartition: true, } optionsMap[id] = parV2Options colsInfoMap[id] = tblColsInfo diff --git a/statistics/handle/handle.go b/statistics/handle/handle.go index b2b5eb4068341..e2aaf18fa324d 100644 --- a/statistics/handle/handle.go +++ b/statistics/handle/handle.go @@ -466,13 +466,7 @@ func (h *Handle) mergePartitionStats2GlobalStats(sc sessionctx.Context, opts map if isIndex == 0 { errMsg = fmt.Sprintf("`%s`", tableInfo.Name.L) } else { - indexName := "" - for _, idx := range tableInfo.Indices { - if idx.ID == histIDs[0] { - indexName = idx.Name.L - } - } - errMsg = fmt.Sprintf("`%s` index: `%s`", tableInfo.Name.L, indexName) + errMsg = fmt.Sprintf("`%s` index: `%s`", tableInfo.Name.L, tableInfo.FindIndexNameByID(histIDs[0])) } err = types.ErrPartitionStatsMissing.GenWithStackByArgs(errMsg) return @@ -481,7 +475,12 @@ func (h *Handle) mergePartitionStats2GlobalStats(sc sessionctx.Context, opts map count, hg, cms, topN, fms := partitionStats.GetStatsInfo(histIDs[i], isIndex == 1) // partition is not empty but column stats(hist, topn) is missing if partitionStats.Count > 0 && (hg == nil || hg.TotalRowCount() <= 0) && (topN == nil || topN.TotalCount() <= 0) { - errMsg := fmt.Sprintf("`%s`", tableInfo.Name.L) + var errMsg string + if isIndex == 0 { + errMsg = fmt.Sprintf("`%s` column: `%s`", tableInfo.Name.L, tableInfo.FindColumnNameByID(histIDs[i])) + } else { + errMsg = fmt.Sprintf("`%s` index: `%s`", tableInfo.Name.L, tableInfo.FindIndexNameByID(histIDs[i])) + } err = types.ErrPartitionColumnStatsMissing.GenWithStackByArgs(errMsg) return }