*: global stats consistency for dynamic prune

errno/errcode.go

@@ -1067,6 +1067,7 @@ const (
 	ErrPlacementPolicyInUse               = 8241
 	ErrOptOnCacheTable                    = 8242
 	ErrHTTPServiceError                   = 8243
+	ErrPartitionColumnStatsMissing        = 8244
 	// TiKV/PD/TiFlash errors.
 	ErrPDServerTimeout           = 9001
 	ErrTiKVServerTimeout         = 9002

errno/errname.go

@@ -1059,9 +1059,10 @@ var MySQLErrName = map[uint16]*mysql.ErrMessage{
 
 	ErrInvalidTableSample: mysql.Message("Invalid TABLESAMPLE: %s", nil),
 
-	ErrJSONObjectKeyTooLong:  mysql.Message("TiDB does not yet support JSON objects with the key length >= 65536", nil),
-	ErrPartitionStatsMissing: mysql.Message("Build table: %s global-level stats failed due to missing partition-level stats", nil),
-	ErrNotSupportedWithSem:   mysql.Message("Feature '%s' is not supported when security enhanced mode is enabled", nil),
+	ErrJSONObjectKeyTooLong:        mysql.Message("TiDB does not yet support JSON objects with the key length >= 65536", nil),
+	ErrPartitionStatsMissing:       mysql.Message("Build table: %s global-level stats failed due to missing partition-level stats", nil),
+	ErrPartitionColumnStatsMissing: mysql.Message("Build table: %s global-level stats failed due to missing partition-level column stats, please run analyze table to refresh columns of all partitions", nil),
+	ErrNotSupportedWithSem:         mysql.Message("Feature '%s' is not supported when security enhanced mode is enabled", nil),
 
 	ErrPlacementPolicyCheck:            mysql.Message("Placement policy didn't meet the constraint, reason: %s", nil),
 	ErrMultiStatementDisabled:          mysql.Message("client has multi-statement capability disabled. Run SET GLOBAL tidb_multi_statement_mode='ON' after you understand the security risk", nil),

errors.toml

@@ -2586,6 +2586,11 @@ error = '''
 Build table: %s global-level stats failed due to missing partition-level stats
 '''
 
+["types:8244"]
+error = '''
+Build table: %s global-level stats failed due to missing partition-level column stats, please run analyze table to refresh columns of all partitions
+'''
+
 ["variable:1193"]
 error = '''
 Unknown system variable '%-.64s'

executor/analyze.go

@@ -209,7 +209,7 @@ func (e *AnalyzeExec) Next(ctx context.Context, req *chunk.Chunk) error {
 			}
 			globalStats, err := statsHandle.MergePartitionStats2GlobalStatsByTableID(e.ctx, globalOpts, e.ctx.GetInfoSchema().(infoschema.InfoSchema), globalStatsID.tableID, info.isIndex, info.histIDs)
 			if err != nil {
-				if types.ErrPartitionStatsMissing.Equal(err) {
+				if types.ErrPartitionStatsMissing.Equal(err) || types.ErrPartitionColumnStatsMissing.Equal(err) {
 					// When we find some partition-level stats are missing, we need to report warning.
 					e.ctx.GetSessionVars().StmtCtx.AppendWarning(err)
 					continue
@@ -230,7 +230,7 @@ func (e *AnalyzeExec) Next(ctx context.Context, req *chunk.Chunk) error {
 			}
 		}
 	}
-	err = e.saveAnalyzeOptsV2()
+	err = e.saveV2AnalyzeOpts()
 	if err != nil {
 		e.ctx.GetSessionVars().StmtCtx.AppendWarning(err)
 	}
@@ -258,14 +258,22 @@ func finishJobWithLog(sctx sessionctx.Context, job *statistics.AnalyzeJob, analy
 	}
 }
 
-func (e *AnalyzeExec) saveAnalyzeOptsV2() error {
+func (e *AnalyzeExec) saveV2AnalyzeOpts() error {
 	if !variable.PersistAnalyzeOptions.Load() || len(e.OptionsMap) == 0 {
 		return nil
 	}
+	// only to save table options if dynamic prune mode
+	dynamicPrune := variable.PartitionPruneMode(e.ctx.GetSessionVars().PartitionPruneMode.Load()) == variable.Dynamic
+	toSaveMap := make(map[int64]core.V2AnalyzeOptions)
+	for id, opts := range e.OptionsMap {
+		if opts.IsTable || !dynamicPrune {
+			toSaveMap[id] = opts
+		}
+	}
 	sql := new(strings.Builder)
 	sqlexec.MustFormatSQL(sql, "REPLACE INTO mysql.analyze_options (table_id,sample_num,sample_rate,buckets,topn,column_choice,column_ids) VALUES ")
 	idx := 0
-	for _, opts := range e.OptionsMap {
+	for _, opts := range toSaveMap {
 		sampleNum := opts.RawOpts[ast.AnalyzeOptNumSamples]
 		sampleRate := float64(0)
 		if val, ok := opts.RawOpts[ast.AnalyzeOptSampleRate]; ok {
@@ -283,7 +291,7 @@ func (e *AnalyzeExec) saveAnalyzeOptsV2() error {
 		}
 		colIDStrs := strings.Join(colIDs, ",")
 		sqlexec.MustFormatSQL(sql, "(%?,%?,%?,%?,%?,%?,%?)", opts.PhyTableID, sampleNum, sampleRate, buckets, topn, colChoice, colIDStrs)
-		if idx < len(e.OptionsMap)-1 {
+		if idx < len(toSaveMap)-1 {
 			sqlexec.MustFormatSQL(sql, ",")
 		}
 		idx += 1

executor/show_test.go

@@ -1588,11 +1588,6 @@ func TestShowVar(t *testing.T) {
 	res = tk.MustQuery(showSQL)
 	require.Len(t, res.Rows(), len(globalVars))
 
-	// Test a known hidden variable.
-	res = tk.MustQuery("show variables like '" + variable.TiDBPartitionPruneMode + "'")
-	require.Len(t, res.Rows(), 0)
-	res = tk.MustQuery("show global variables like '" + variable.TiDBPartitionPruneMode + "'")
-	require.Len(t, res.Rows(), 0)
 	// Test Hidden tx_read_ts
 	res = tk.MustQuery("show variables like '%tx_read_ts'")
 	require.Len(t, res.Rows(), 0)

planner/core/common_plans.go

@@ -1120,6 +1120,7 @@ type V2AnalyzeOptions struct {
 	FilledOpts map[ast.AnalyzeOptionType]uint64
 	ColChoice  model.ColumnChoice
 	ColumnList []*model.ColumnInfo
+	IsTable    bool
 }
 
 // AnalyzeColumnsTask is used for analyze columns.

planner/core/planbuilder.go

@@ -2179,6 +2179,13 @@ func (b *PlanBuilder) genV2AnalyzeOptions(
 	if !persist {
 		return optionsMap, colsInfoMap, nil
 	}
+	dynamicPrune := variable.PartitionPruneMode(b.ctx.GetSessionVars().PartitionPruneMode.Load()) == variable.Dynamic
+	if !isAnalyzeTable && dynamicPrune && (len(astOpts) > 0 || astColChoice != model.DefaultChoice) {
+		astOpts = make(map[ast.AnalyzeOptionType]uint64, 0)
+		astColChoice = model.DefaultChoice
+		astColList = make([]*model.ColumnInfo, 0)
+		b.ctx.GetSessionVars().StmtCtx.AppendWarning(errors.New("Ignore columns and options when analyze partition in dynamic mode"))
+	}
 	tblSavedOpts, tblSavedColChoice, tblSavedColList, err := b.getSavedAnalyzeOpts(tbl.TableInfo.ID, tbl.TableInfo)
 	if err != nil {
 		return nil, nil, err
@@ -2201,11 +2208,25 @@ func (b *PlanBuilder) genV2AnalyzeOptions(
 		FilledOpts: tblFilledOpts,
 		ColChoice:  tblColChoice,
 		ColumnList: tblColList,
+		IsTable:    true,
 	}
 	optionsMap[tbl.TableInfo.ID] = tblAnalyzeOptions
 	colsInfoMap[tbl.TableInfo.ID] = tblColsInfo
 	for _, id := range physicalIDs {
 		if id != tbl.TableInfo.ID {
+			if dynamicPrune {
+				parV2Options := V2AnalyzeOptions{
+					PhyTableID: id,
+					RawOpts:    tblOpts,
+					FilledOpts: tblFilledOpts,
+					ColChoice:  tblColChoice,
+					ColumnList: tblColList,
+					IsTable:    false,
+				}
+				optionsMap[id] = parV2Options
+				colsInfoMap[id] = tblColsInfo
+				continue
+			}
 			parSavedOpts, parSavedColChoice, parSavedColList, err := b.getSavedAnalyzeOpts(id, tbl.TableInfo)
 			if err != nil {
 				return nil, nil, err

sessionctx/variable/sysvar.go

@@ -1338,14 +1338,26 @@ var defaultSysVars = []*SysVar{
 		s.EnableClusteredIndex = TiDBOptEnableClustered(val)
 		return nil
 	}},
-	{Scope: ScopeGlobal | ScopeSession, Name: TiDBPartitionPruneMode, Value: DefTiDBPartitionPruneMode, Hidden: true, Type: TypeStr, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) {
+	{Scope: ScopeGlobal | ScopeSession, Name: TiDBPartitionPruneMode, Value: DefTiDBPartitionPruneMode, Type: TypeStr, Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope ScopeFlag) (string, error) {
 		mode := PartitionPruneMode(normalizedValue).Update()
 		if !mode.Valid() {
 			return normalizedValue, ErrWrongTypeForVar.GenWithStackByArgs(TiDBPartitionPruneMode)
 		}
 		return string(mode), nil
+	}, GetSession: func(s *SessionVars) (string, error) {
+		return s.PartitionPruneMode.Load(), nil
 	}, SetSession: func(s *SessionVars, val string) error {
-		s.PartitionPruneMode.Store(strings.ToLower(strings.TrimSpace(val)))
+		newMode := strings.ToLower(strings.TrimSpace(val))
+		if PartitionPruneMode(s.PartitionPruneMode.Load()) == Static && PartitionPruneMode(newMode) == Dynamic {
+			s.StmtCtx.AppendWarning(errors.New("Please analyze all partition tables again for consistency between partition and global stats"))
+		}
+		s.PartitionPruneMode.Store(newMode)
+		return nil
+	}, SetGlobal: func(s *SessionVars, val string) error {
+		newMode := strings.ToLower(strings.TrimSpace(val))
+		if PartitionPruneMode(newMode) == Dynamic {
+			s.StmtCtx.AppendWarning(errors.New("Please analyze all partition tables again for consistency between partition and global stats"))
+		}
 		return nil
 	}},
 	{Scope: ScopeGlobal | ScopeSession, Name: TiDBRedactLog, Value: BoolToOnOff(DefTiDBRedactLog), Type: TypeBool, SetSession: func(s *SessionVars, val string) error {

statistics/handle/handle.go

@@ -479,6 +479,12 @@ func (h *Handle) mergePartitionStats2GlobalStats(sc sessionctx.Context, opts map
 		}
 		for i := 0; i < globalStats.Num; i++ {
 			count, hg, cms, topN, fms := partitionStats.GetStatsInfo(histIDs[i], isIndex == 1)
+			// partition is not empty but column stats(hist, topn) is missing
+			if partitionStats.Count > 0 && (hg == nil || hg.TotalRowCount() <= 0) && (topN == nil || topN.TotalCount() <= 0) {
+				errMsg := fmt.Sprintf("`%s`", tableInfo.Name.L)
+				err = types.ErrPartitionColumnStatsMissing.GenWithStackByArgs(errMsg)
+				return
+			}
 			if i == 0 {
 				// In a partition, we will only update globalStats.Count once
 				globalStats.Count += count

statistics/handle/handle_test.go

@@ -1070,18 +1070,6 @@ partition by range (a) (
 	checkHealthy(60, 50, 66)
 }
 
-func TestHideGlobalStatsSwitch(t *testing.T) {
-	// NOTICE: remove this test when this global-stats is GA.
-	store, clean := testkit.CreateMockStore(t)
-	defer clean()
-	tk := testkit.NewTestKit(t, store)
-	rs := tk.MustQuery("show variables").Rows()
-	for _, r := range rs {
-		require.NotEqual(t, "tidb_partition_prune_mode", strings.ToLower(r[0].(string)))
-	}
-	require.Len(t, tk.MustQuery("show variables where variable_name like '%tidb_partition_prune_mode%'").Rows(), 0)
-}
-
 func TestGlobalStatsData(t *testing.T) {
 	store, dom, clean := testkit.CreateMockStoreAndDomain(t)
 	defer clean()
@@ -2220,6 +2208,7 @@ func TestFMSWithAnalyzePartition(t *testing.T) {
 	tk.MustExec("analyze table t partition p0 with 1 topn, 2 buckets")
 	tk.MustQuery("show warnings").Sort().Check(testkit.Rows(
 		"Note 1105 Analyze use auto adjusted sample rate 1.000000 for table test.t's partition p0",
+		"Warning 1105 Ignore columns and options when analyze partition in dynamic mode",
 		"Warning 8131 Build table: `t` global-level stats failed due to missing partition-level stats",
 		"Warning 8131 Build table: `t` index: `a` global-level stats failed due to missing partition-level stats",
 	))

types/errors.go

@@ -87,4 +87,7 @@ var (
 	// ErrPartitionStatsMissing is returned when the partition-level stats is missing and the build global-level stats fails.
 	// Put this error here is to prevent `import cycle not allowed`.
 	ErrPartitionStatsMissing = dbterror.ClassTypes.NewStd(mysql.ErrPartitionStatsMissing)
+	// ErrPartitionColumnStatsMissing is returned when the partition-level column stats is missing and the build global-level stats fails.
+	// Put this error here is to prevent `import cycle not allowed`.
+	ErrPartitionColumnStatsMissing = dbterror.ClassTypes.NewStd(mysql.ErrPartitionColumnStatsMissing)
 )