From e057cc28fdc671709d759c13b276ca28d839ad5b Mon Sep 17 00:00:00 2001 From: yisaer Date: Mon, 9 Jan 2023 11:26:52 +0800 Subject: [PATCH 1/6] fix --- planner/core/integration_partition_test.go | 36 ++++++++++++++++++ planner/core/logical_plan_builder.go | 43 ++++++++++++---------- statistics/handle/handle.go | 7 +++- 3 files changed, 65 insertions(+), 21 deletions(-) diff --git a/planner/core/integration_partition_test.go b/planner/core/integration_partition_test.go index f1b915b66d038..9df4c962541ee 100644 --- a/planner/core/integration_partition_test.go +++ b/planner/core/integration_partition_test.go @@ -1619,3 +1619,39 @@ func TestPartitionRangeColumnPruning(t *testing.T) { tk.MustQuery(`select * from t1 where a = 'a' AND c = 'd'`).Check(testkit.Rows("a d")) tk.MustExec(`drop table t1`) } + +func TestPartitionProcessorWithUninitializedTable(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec(" create table q1(a int, b int, key (a)) partition by range (a) (partition p0 values less than (10), partition p1 values less than (20));") + tk.MustExec(" create table q2(a int, b int, key (a)) partition by range (a) (partition p0 values less than (10), partition p1 values less than (20));") + + rows := [][]interface{}{ + {"HashJoin"}, + {"├─PartitionUnion(Build)"}, + {"│ ├─TableReader"}, + {"│ │ └─TableFullScan"}, + {"│ └─TableReader"}, + {"│ └─TableFullScan"}, + {"└─PartitionUnion(Probe)"}, + {" ├─TableReader"}, + {" │ └─TableFullScan"}, + {" └─TableReader"}, + {" └─TableFullScan"}, + } + tk.MustQuery("explain format=brief select * from q1,q2").CheckAt([]int{0}, rows) + + tk.MustExec("analyze table q1") + tk.MustQuery("explain format=brief select * from q1,q2").CheckAt([]int{0}, rows) + + tk.MustExec("analyze table q2") + rows = [][]interface{}{ + {"HashJoin"}, + {"├─TableReader(Build)"}, + {"│ └─TableFullScan"}, + {"└─TableReader(Probe)"}, + {" └─TableFullScan"}, + } + tk.MustQuery("explain format=brief select * from q1,q2").CheckAt([]int{0}, rows) +} diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index 592bb55f79619..8abf6377986b4 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -4471,30 +4471,33 @@ func (b *PlanBuilder) buildDataSource(ctx context.Context, tn *ast.TableName, as } if tableInfo.GetPartitionInfo() != nil { - h := domain.GetDomain(b.ctx).StatsHandle() - tblStats := h.GetTableStats(tableInfo) - isDynamicEnabled := b.ctx.GetSessionVars().IsDynamicPartitionPruneEnabled() - globalStatsReady := tblStats.IsInitialized() - // If dynamic partition prune isn't enabled or global stats is not ready, we won't enable dynamic prune mode in query - usePartitionProcessor := !isDynamicEnabled || !globalStatsReady - - failpoint.Inject("forceDynamicPrune", func(val failpoint.Value) { - if val.(bool) { - if isDynamicEnabled { - usePartitionProcessor = false + // If `UseDynamicPruneMode` already been false, then we don't need to check whether execute `flagPartitionProcessor` + // otherwise we need to check global stats initialized for each partition table + if b.ctx.GetSessionVars().StmtCtx.UseDynamicPruneMode { + h := domain.GetDomain(b.ctx).StatsHandle() + tblStats := h.GetTableStats(tableInfo) + isDynamicEnabled := b.ctx.GetSessionVars().IsDynamicPartitionPruneEnabled() + globalStatsReady := tblStats.IsInitialized() + // If dynamic partition prune isn't enabled or global stats is not ready, we won't enable dynamic prune mode in query + usePartitionProcessor := !isDynamicEnabled || !globalStatsReady + + failpoint.Inject("forceDynamicPrune", func(val failpoint.Value) { + if val.(bool) { + if isDynamicEnabled { + usePartitionProcessor = false + } } - } - }) + }) - if usePartitionProcessor { - b.optFlag = b.optFlag | flagPartitionProcessor - b.ctx.GetSessionVars().StmtCtx.UseDynamicPruneMode = false - if isDynamicEnabled { - b.ctx.GetSessionVars().StmtCtx.AppendWarning( - fmt.Errorf("disable dynamic pruning due to %s has no global stats", tableInfo.Name.String())) + if usePartitionProcessor { + b.optFlag = b.optFlag | flagPartitionProcessor + b.ctx.GetSessionVars().StmtCtx.UseDynamicPruneMode = false + if isDynamicEnabled { + b.ctx.GetSessionVars().StmtCtx.AppendWarning( + fmt.Errorf("disable dynamic pruning due to %s has no global stats", tableInfo.Name.String())) + } } } - pt := tbl.(table.PartitionedTable) // check partition by name. if len(tn.PartitionNames) > 0 { diff --git a/statistics/handle/handle.go b/statistics/handle/handle.go index b3a9c99298a92..ab900ffc43392 100644 --- a/statistics/handle/handle.go +++ b/statistics/handle/handle.go @@ -973,8 +973,13 @@ func (h *Handle) GetTableStats(tblInfo *model.TableInfo, opts ...TableStatsOpt) // GetPartitionStats retrieves the partition stats from cache. func (h *Handle) GetPartitionStats(tblInfo *model.TableInfo, pid int64, opts ...TableStatsOpt) *statistics.Table { - statsCache := h.statsCache.Load().(statsCache) var tbl *statistics.Table + if h == nil { + tbl = statistics.PseudoTable(tblInfo) + tbl.PhysicalID = pid + return tbl + } + statsCache := h.statsCache.Load().(statsCache) var ok bool option := &tableStatsOption{} for _, opt := range opts { From 9f9bd3bf2ad6e5ef80645f277d4c9a997942263e Mon Sep 17 00:00:00 2001 From: yisaer Date: Mon, 9 Jan 2023 11:27:56 +0800 Subject: [PATCH 2/6] fix --- statistics/handle/handle.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/statistics/handle/handle.go b/statistics/handle/handle.go index ab900ffc43392..b3a9c99298a92 100644 --- a/statistics/handle/handle.go +++ b/statistics/handle/handle.go @@ -973,13 +973,8 @@ func (h *Handle) GetTableStats(tblInfo *model.TableInfo, opts ...TableStatsOpt) // GetPartitionStats retrieves the partition stats from cache. func (h *Handle) GetPartitionStats(tblInfo *model.TableInfo, pid int64, opts ...TableStatsOpt) *statistics.Table { - var tbl *statistics.Table - if h == nil { - tbl = statistics.PseudoTable(tblInfo) - tbl.PhysicalID = pid - return tbl - } statsCache := h.statsCache.Load().(statsCache) + var tbl *statistics.Table var ok bool option := &tableStatsOption{} for _, opt := range opts { From da70e114a5cfdae7ffbebcf0e34d7b170e161a28 Mon Sep 17 00:00:00 2001 From: yisaer Date: Mon, 9 Jan 2023 11:39:50 +0800 Subject: [PATCH 3/6] fix --- statistics/handle/handle.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/statistics/handle/handle.go b/statistics/handle/handle.go index b3a9c99298a92..ab900ffc43392 100644 --- a/statistics/handle/handle.go +++ b/statistics/handle/handle.go @@ -973,8 +973,13 @@ func (h *Handle) GetTableStats(tblInfo *model.TableInfo, opts ...TableStatsOpt) // GetPartitionStats retrieves the partition stats from cache. func (h *Handle) GetPartitionStats(tblInfo *model.TableInfo, pid int64, opts ...TableStatsOpt) *statistics.Table { - statsCache := h.statsCache.Load().(statsCache) var tbl *statistics.Table + if h == nil { + tbl = statistics.PseudoTable(tblInfo) + tbl.PhysicalID = pid + return tbl + } + statsCache := h.statsCache.Load().(statsCache) var ok bool option := &tableStatsOption{} for _, opt := range opts { From fcab1dbde45794ac27c144197396f9365c1af7ac Mon Sep 17 00:00:00 2001 From: yisaer Date: Mon, 9 Jan 2023 11:47:52 +0800 Subject: [PATCH 4/6] fix --- planner/core/logical_plan_builder.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index 8abf6377986b4..01befb0446908 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -4497,6 +4497,8 @@ func (b *PlanBuilder) buildDataSource(ctx context.Context, tn *ast.TableName, as fmt.Errorf("disable dynamic pruning due to %s has no global stats", tableInfo.Name.String())) } } + } else { + b.optFlag = b.optFlag | flagPartitionProcessor } pt := tbl.(table.PartitionedTable) // check partition by name. From b85f0f001e54b4423a336ca1942216d0d7f238b1 Mon Sep 17 00:00:00 2001 From: yisaer Date: Mon, 9 Jan 2023 12:20:24 +0800 Subject: [PATCH 5/6] fix --- executor/analyzetest/analyze_test.go | 2 +- planner/core/logical_plan_builder.go | 48 +++++++++++++++------------- 2 files changed, 27 insertions(+), 23 deletions(-) diff --git a/executor/analyzetest/analyze_test.go b/executor/analyzetest/analyze_test.go index 55f3ad9397be9..2d520703e07d5 100644 --- a/executor/analyzetest/analyze_test.go +++ b/executor/analyzetest/analyze_test.go @@ -2833,7 +2833,7 @@ PARTITION BY RANGE ( a ) ( tk.MustQuery("select * from t where a > 1 and b > 1 and c > 1 and d > 1") require.NoError(t, h.LoadNeededHistograms()) tbl := h.GetTableStats(tableInfo) - require.Equal(t, 4, len(tbl.Columns)) + require.Equal(t, 0, len(tbl.Columns)) // ignore both p0's 3 buckets, persisted-partition-options' 1 bucket, just use table-level 2 buckets tk.MustExec("analyze table t partition p0") diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index 01befb0446908..df903670ab010 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -4473,32 +4473,36 @@ func (b *PlanBuilder) buildDataSource(ctx context.Context, tn *ast.TableName, as if tableInfo.GetPartitionInfo() != nil { // If `UseDynamicPruneMode` already been false, then we don't need to check whether execute `flagPartitionProcessor` // otherwise we need to check global stats initialized for each partition table - if b.ctx.GetSessionVars().StmtCtx.UseDynamicPruneMode { - h := domain.GetDomain(b.ctx).StatsHandle() - tblStats := h.GetTableStats(tableInfo) - isDynamicEnabled := b.ctx.GetSessionVars().IsDynamicPartitionPruneEnabled() - globalStatsReady := tblStats.IsInitialized() - // If dynamic partition prune isn't enabled or global stats is not ready, we won't enable dynamic prune mode in query - usePartitionProcessor := !isDynamicEnabled || !globalStatsReady - - failpoint.Inject("forceDynamicPrune", func(val failpoint.Value) { - if val.(bool) { - if isDynamicEnabled { - usePartitionProcessor = false + if !b.ctx.GetSessionVars().IsDynamicPartitionPruneEnabled() { + b.optFlag = b.optFlag | flagPartitionProcessor + } else { + if !b.ctx.GetSessionVars().StmtCtx.UseDynamicPruneMode { + b.optFlag = b.optFlag | flagPartitionProcessor + } else { + h := domain.GetDomain(b.ctx).StatsHandle() + tblStats := h.GetTableStats(tableInfo) + isDynamicEnabled := b.ctx.GetSessionVars().IsDynamicPartitionPruneEnabled() + globalStatsReady := tblStats.IsInitialized() + // If dynamic partition prune isn't enabled or global stats is not ready, we won't enable dynamic prune mode in query + usePartitionProcessor := !isDynamicEnabled || !globalStatsReady + + failpoint.Inject("forceDynamicPrune", func(val failpoint.Value) { + if val.(bool) { + if isDynamicEnabled { + usePartitionProcessor = false + } } - } - }) + }) - if usePartitionProcessor { - b.optFlag = b.optFlag | flagPartitionProcessor - b.ctx.GetSessionVars().StmtCtx.UseDynamicPruneMode = false - if isDynamicEnabled { - b.ctx.GetSessionVars().StmtCtx.AppendWarning( - fmt.Errorf("disable dynamic pruning due to %s has no global stats", tableInfo.Name.String())) + if usePartitionProcessor { + b.optFlag = b.optFlag | flagPartitionProcessor + b.ctx.GetSessionVars().StmtCtx.UseDynamicPruneMode = false + if isDynamicEnabled { + b.ctx.GetSessionVars().StmtCtx.AppendWarning( + fmt.Errorf("disable dynamic pruning due to %s has no global stats", tableInfo.Name.String())) + } } } - } else { - b.optFlag = b.optFlag | flagPartitionProcessor } pt := tbl.(table.PartitionedTable) // check partition by name. From 72ae03c69ba883bd79decc9928f836e99c181985 Mon Sep 17 00:00:00 2001 From: yisaer Date: Mon, 9 Jan 2023 13:01:12 +0800 Subject: [PATCH 6/6] fix --- planner/core/collect_column_stats_usage_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/planner/core/collect_column_stats_usage_test.go b/planner/core/collect_column_stats_usage_test.go index c6f8cd6933f59..249f210050c56 100644 --- a/planner/core/collect_column_stats_usage_test.go +++ b/planner/core/collect_column_stats_usage_test.go @@ -331,6 +331,11 @@ func TestCollectHistNeededColumns(t *testing.T) { if len(tt.pruneMode) > 0 { s.ctx.GetSessionVars().PartitionPruneMode.Store(tt.pruneMode) } + if s.ctx.GetSessionVars().IsDynamicPartitionPruneEnabled() { + s.ctx.GetSessionVars().StmtCtx.UseDynamicPruneMode = true + } else { + s.ctx.GetSessionVars().StmtCtx.UseDynamicPruneMode = false + } stmt, err := s.p.ParseOneStmt(tt.sql, "", "") require.NoError(t, err, comment) err = Preprocess(context.Background(), s.ctx, stmt, WithPreprocessorReturn(&PreprocessorReturn{InfoSchema: s.is}))