From 79b4e1d7936557409ce7ecd8906fc41cca836db1 Mon Sep 17 00:00:00 2001 From: elsa0520 Date: Thu, 1 Dec 2022 20:30:25 +0800 Subject: [PATCH 1/8] planner: change the stage of count star rewriter rule from logical to post --- planner/core/logical_plan_builder.go | 2 +- planner/core/optimizer.go | 124 +++++++++++++++++++++- planner/core/physical_plans.go | 8 ++ planner/core/rule_column_pruning.go | 13 +-- planner/core/testdata/plan_suite_in.json | 12 +-- planner/core/testdata/plan_suite_out.json | 46 ++++---- 6 files changed, 167 insertions(+), 38 deletions(-) diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index ebabe1ba95795..6f0352395a8ed 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -204,7 +204,7 @@ func (b *PlanBuilder) buildAggregation(ctx context.Context, p LogicalPlan, aggFu b.optFlag |= flagPredicatePushDown b.optFlag |= flagEliminateAgg b.optFlag |= flagEliminateProjection - b.optFlag |= flagCountStarRewriter +/* b.optFlag |= flagCountStarRewriter*/ if b.ctx.GetSessionVars().EnableSkewDistinctAgg { b.optFlag |= flagSkewDistinctAgg diff --git a/planner/core/optimizer.go b/planner/core/optimizer.go index d96f70462cc33..96ce047f65a3d 100644 --- a/planner/core/optimizer.go +++ b/planner/core/optimizer.go @@ -16,11 +16,13 @@ package core import ( "context" + "fmt" "math" "github.com/pingcap/errors" "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/expression" + "github.com/pingcap/tidb/expression/aggregation" "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/lock" @@ -72,7 +74,7 @@ const ( flagSyncWaitStatsLoadPoint flagJoinReOrder flagPrunColumnsAgain - flagCountStarRewriter + /* flagCountStarRewriter*/ ) var optRuleList = []logicalOptRule{ @@ -95,7 +97,7 @@ var optRuleList = []logicalOptRule{ &syncWaitStatsLoadPoint{}, &joinReOrderSolver{}, &columnPruner{}, // column pruning again at last, note it will mess up the results of buildKeySolver - &countStarRewriter{}, + /* &countStarRewriter{},*/ } type logicalOptimizeOp struct { @@ -392,6 +394,7 @@ func postOptimize(sctx sessionctx.Context, plan PhysicalPlan) (PhysicalPlan, err handleFineGrainedShuffle(sctx, plan) checkPlanCacheable(sctx, plan) propagateProbeParents(plan, nil) + countStarRewrite(plan) return plan, nil } @@ -533,6 +536,123 @@ func prunePhysicalColumnsInternal(sctx sessionctx.Context, plan PhysicalPlan) er return nil } +/* +* +The countStarRewriter is used to rewrite + + count(*) -> count(not null column) + +**Only for TiFlash** +Attention: +Since count(*) is directly translated into count(1) during grammar parsing, +the rewritten pattern actually matches count(constant) + +Pattern: +PhysicalAggregation: count(constant) + + | + TableFullScan: TiFlash + +Optimize: +Table + + + +Query: select count(*) from table +ColumnPruningRule: datasource pick row_id +countStarRewrite: datasource pick k1 instead of row_id + + rewrite count(*) -> count(k1) + +Rewritten Query: select count(k1) from table +*/ +func countStarRewrite(plan PhysicalPlan) { + countStarRewriteInternal(plan) + if tableReader, ok := plan.(*PhysicalTableReader); ok { + countStarRewrite(tableReader.tablePlan) + } else { + for _, child := range plan.Children() { + countStarRewrite(child) + } + } + +} + +func countStarRewriteInternal(plan PhysicalPlan) { + // match pattern any agg(count(constant)) -> tablefullscan(tiflash) + if !plan.SCtx().GetSessionVars().InRestrictedSQL { + fmt.Println("") + } + var physicalAgg *basePhysicalAgg + switch x := plan.(type) { + case *PhysicalHashAgg: + physicalAgg = x.getPointer() + case *PhysicalStreamAgg: + physicalAgg = x.getPointer() + default: + return + } + if len(physicalAgg.GroupByItems) > 0 || len(physicalAgg.children) != 1 { + return + } + for _, aggFunc := range physicalAgg.AggFuncs { + if aggFunc.Name != "count" || len(aggFunc.Args) != 1 || aggFunc.HasDistinct { + return + } + if _, ok := aggFunc.Args[0].(*expression.Constant); !ok { + return + } + } + physicalTableScan, ok := physicalAgg.Children()[0].(*PhysicalTableScan) + if !ok || !physicalTableScan.isFullScan() || physicalTableScan.StoreType != kv.TiFlash || len(physicalTableScan.schema.Columns) != 1 { + return + } + // rewrite datasource and agg args + rewriteTableScanAndAggArgs(physicalTableScan, physicalAgg.AggFuncs) +} + +// Pick the narrowest and not null column from table +// If there is no not null column in Data Source, the row_id or pk column will be retained +func rewriteTableScanAndAggArgs(physicalTableScan *PhysicalTableScan, aggFuncs []*aggregation.AggFuncDesc) { + var resultColumnInfo *model.ColumnInfo + var resultColumn *expression.Column + + resultColumnInfo = physicalTableScan.Columns[0] + resultColumn = physicalTableScan.schema.Columns[0] + // prefer not null column from table + for _, columnInfo := range physicalTableScan.Table.Columns { + if columnInfo.FieldType.IsVarLengthType() { + continue + } + if mysql.HasNotNullFlag(columnInfo.GetFlag()) { + if columnInfo.GetFlen() < resultColumnInfo.GetFlen() { + resultColumnInfo = columnInfo + resultColumn = &expression.Column{ + UniqueID: physicalTableScan.ctx.GetSessionVars().AllocPlanColumnID(), + ID: resultColumnInfo.ID, + RetType: resultColumnInfo.FieldType.Clone(), + OrigName: fmt.Sprintf("%s.%s.%s", physicalTableScan.DBName.L, physicalTableScan.Table.Name.L, resultColumnInfo.Name), + } + } + } + } + // table scan (row_id) -> (not null column) + physicalTableScan.Columns[0] = resultColumnInfo + physicalTableScan.schema.Columns[0] = resultColumn + // agg args count(1) -> count(not null column) + for _, aggFunc := range aggFuncs { + constExpr, ok := aggFunc.Args[0].(*expression.Constant) + if !ok { + return + } + // count(null) shouldn't be rewritten + if constExpr.Value.IsNull() { + continue + } + aggFunc.Args[0] = resultColumn + } +} + // Only for MPP(Window<-[Sort]<-ExchangeReceiver<-ExchangeSender). // TiFlashFineGrainedShuffleStreamCount: // < 0: fine grained shuffle is disabled. diff --git a/planner/core/physical_plans.go b/planner/core/physical_plans.go index ced23204f639d..537b0826eb381 100644 --- a/planner/core/physical_plans.go +++ b/planner/core/physical_plans.go @@ -1753,6 +1753,10 @@ type PhysicalHashAgg struct { basePhysicalAgg } +func (p *PhysicalHashAgg) getPointer() *basePhysicalAgg { + return &p.basePhysicalAgg +} + // Clone implements PhysicalPlan interface. func (p *PhysicalHashAgg) Clone() (PhysicalPlan, error) { cloned := new(PhysicalHashAgg) @@ -1787,6 +1791,10 @@ type PhysicalStreamAgg struct { basePhysicalAgg } +func (p *PhysicalStreamAgg) getPointer() *basePhysicalAgg { + return &p.basePhysicalAgg +} + // Clone implements PhysicalPlan interface. func (p *PhysicalStreamAgg) Clone() (PhysicalPlan, error) { cloned := new(PhysicalStreamAgg) diff --git a/planner/core/rule_column_pruning.go b/planner/core/rule_column_pruning.go index 0769a473a6f34..2b38d6b74c5aa 100644 --- a/planner/core/rule_column_pruning.go +++ b/planner/core/rule_column_pruning.go @@ -348,12 +348,13 @@ func (ds *DataSource) PruneColumns(parentUsedCols []*expression.Column, opt *log var handleCol *expression.Column var handleColInfo *model.ColumnInfo // case 1: tiflash - if ds.tableInfo.TiFlashReplica != nil { - handleCol, handleColInfo = preferNotNullColumnFromTable(ds) - } else { - // case 2: tikv - handleCol, handleColInfo = preferKeyColumnFromTable(ds, originSchemaColumns, originColumns) - } + /* if ds.tableInfo.TiFlashReplica != nil { + handleCol, handleColInfo = preferNotNullColumnFromTable(ds) + } else { + // case 2: tikv + handleCol, handleColInfo = preferKeyColumnFromTable(ds, originSchemaColumns, originColumns) + }*/ + handleCol, handleColInfo = preferKeyColumnFromTable(ds, originSchemaColumns, originColumns) ds.Columns = append(ds.Columns, handleColInfo) ds.schema.Append(handleCol) } diff --git a/planner/core/testdata/plan_suite_in.json b/planner/core/testdata/plan_suite_in.json index 47feb2ca0e437..c38250c802454 100644 --- a/planner/core/testdata/plan_suite_in.json +++ b/planner/core/testdata/plan_suite_in.json @@ -1099,14 +1099,14 @@ "name": "TestCountStarForTikv", "cases": [ "select count(*) from t", - "select count(1), count(3.1415), count(0), count(null) from t -- be rewritten count(row_id) but count(null)", + "select count(1), count(3.1415), count(0), count(null) from t -- shouldn't be rewritten", "select count(*) from t where a=1", "select count(*) from t_pick_row_id", - "select t.b, t.c from (select count(*) as c from t) a, t where a.c=t.a -- test recursive, be rewritten count(row_id)", - "select * from t out where out.a > (select count(*) from t inn where inn.a = out.b) -- shouldn't be rewritten for correlated sub query", - "select count(*) from t t1, t t2 where t1.a=t2.e -- shouldn't be rewritten when join under agg", + "select t.b, t.c from (select count(*) as c from t) a, t where a.c=t.a -- shouldn't be rewritten", + "select * from t out where out.a > (select count(*) from t inn where inn.a = out.b) -- shouldn't be rewritten", + "select count(*) from t t1, t t2 where t1.a=t2.e -- shouldn't be rewritten", "select count(distinct 1) from t -- shouldn't be rewritten", - "select count(1), count(a), count(b) from t -- count(1) to count(a)", + "select count(1), count(a), count(b) from t -- shouldn't be rewritten", "select a, count(*) from t group by a -- shouldn't be rewritten", "select sum(a) from t -- sum shouldn't be rewritten" ] @@ -1122,7 +1122,7 @@ "select * from t out where out.a > (select count(*) from t inn where inn.a = out.b) -- shouldn't be rewritten for correlated sub query", "select count(*) from t t1, t t2 where t1.a=t2.e -- shouldn't be rewritten when join under agg", "select count(distinct 1) from t -- shouldn't be rewritten", - "select count(1), count(a), count(b) from t -- count(1) to count(a)", + "select count(1), count(a), count(b) from t -- keep count(1)", "select a, count(*) from t group by a -- shouldn't be rewritten", "select sum(a) from t -- sum shouldn't be rewritten" ] diff --git a/planner/core/testdata/plan_suite_out.json b/planner/core/testdata/plan_suite_out.json index 6b8612f70992d..b3a7664b2b2fd 100644 --- a/planner/core/testdata/plan_suite_out.json +++ b/planner/core/testdata/plan_suite_out.json @@ -6447,17 +6447,17 @@ "Plan": [ "StreamAgg 1.00 root funcs:count(Column#12)->Column#10", "└─TableReader 1.00 root data:StreamAgg", - " └─StreamAgg 1.00 cop[tikv] funcs:count(test.t._tidb_rowid)->Column#12", + " └─StreamAgg 1.00 cop[tikv] funcs:count(1)->Column#12", " └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" ], "Warning": null }, { - "SQL": "select count(1), count(3.1415), count(0), count(null) from t -- be rewritten count(row_id) but count(null)", + "SQL": "select count(1), count(3.1415), count(0), count(null) from t -- shouldn't be rewritten", "Plan": [ "StreamAgg 1.00 root funcs:count(Column#18)->Column#10, funcs:count(Column#19)->Column#11, funcs:count(Column#20)->Column#12, funcs:count(Column#21)->Column#13", "└─TableReader 1.00 root data:StreamAgg", - " └─StreamAgg 1.00 cop[tikv] funcs:count(test.t._tidb_rowid)->Column#18, funcs:count(test.t._tidb_rowid)->Column#19, funcs:count(test.t._tidb_rowid)->Column#20, funcs:count(NULL)->Column#21", + " └─StreamAgg 1.00 cop[tikv] funcs:count(1)->Column#18, funcs:count(3.1415)->Column#19, funcs:count(0)->Column#20, funcs:count(NULL)->Column#21", " └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" ], "Warning": null @@ -6467,7 +6467,7 @@ "Plan": [ "StreamAgg 1.00 root funcs:count(Column#12)->Column#10", "└─TableReader 1.00 root data:StreamAgg", - " └─StreamAgg 1.00 cop[tikv] funcs:count(test.t.a)->Column#12", + " └─StreamAgg 1.00 cop[tikv] funcs:count(1)->Column#12", " └─Selection 10.00 cop[tikv] eq(test.t.a, 1)", " └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" ], @@ -6478,18 +6478,18 @@ "Plan": [ "StreamAgg 1.00 root funcs:count(Column#5)->Column#3", "└─TableReader 1.00 root data:StreamAgg", - " └─StreamAgg 1.00 cop[tikv] funcs:count(test.t_pick_row_id._tidb_rowid)->Column#5", + " └─StreamAgg 1.00 cop[tikv] funcs:count(1)->Column#5", " └─TableFullScan 10000.00 cop[tikv] table:t_pick_row_id keep order:false, stats:pseudo" ], "Warning": null }, { - "SQL": "select t.b, t.c from (select count(*) as c from t) a, t where a.c=t.a -- test recursive, be rewritten count(row_id)", + "SQL": "select t.b, t.c from (select count(*) as c from t) a, t where a.c=t.a -- shouldn't be rewritten", "Plan": [ "HashJoin 1.25 root inner join, equal:[eq(test.t.a, Column#10)]", "├─StreamAgg(Build) 1.00 root funcs:count(Column#21)->Column#10", "│ └─TableReader 1.00 root data:StreamAgg", - "│ └─StreamAgg 1.00 cop[tikv] funcs:count(test.t._tidb_rowid)->Column#21", + "│ └─StreamAgg 1.00 cop[tikv] funcs:count(1)->Column#21", "│ └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo", "└─TableReader(Probe) 10000.00 root data:TableFullScan", " └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" @@ -6497,7 +6497,7 @@ "Warning": null }, { - "SQL": "select * from t out where out.a > (select count(*) from t inn where inn.a = out.b) -- shouldn't be rewritten for correlated sub query", + "SQL": "select * from t out where out.a > (select count(*) from t inn where inn.a = out.b) -- shouldn't be rewritten", "Plan": [ "Projection 10000.00 root test.t.a, test.t.b, test.t.c, test.t.d, test.t.e, test.t.f, test.t.g, test.t.h", "└─Apply 10000.00 root CARTESIAN inner join, other cond:gt(test.t.a, Column#19)", @@ -6505,14 +6505,14 @@ " │ └─TableFullScan 10000.00 cop[tikv] table:out keep order:false, stats:pseudo", " └─StreamAgg(Probe) 10000.00 root funcs:count(Column#21)->Column#19", " └─TableReader 10000.00 root data:StreamAgg", - " └─StreamAgg 10000.00 cop[tikv] funcs:count(test.t.a)->Column#21", + " └─StreamAgg 10000.00 cop[tikv] funcs:count(1)->Column#21", " └─Selection 80000000.00 cop[tikv] eq(cast(test.t.a, double BINARY), cast(test.t.b, double BINARY))", " └─TableFullScan 100000000.00 cop[tikv] table:inn keep order:false, stats:pseudo" ], "Warning": null }, { - "SQL": "select count(*) from t t1, t t2 where t1.a=t2.e -- shouldn't be rewritten when join under agg", + "SQL": "select count(*) from t t1, t t2 where t1.a=t2.e -- shouldn't be rewritten", "Plan": [ "HashAgg 1.00 root funcs:count(1)->Column#19", "└─HashJoin 12500.00 root inner join, equal:[eq(test.t.a, test.t.e)]", @@ -6533,11 +6533,11 @@ "Warning": null }, { - "SQL": "select count(1), count(a), count(b) from t -- count(1) to count(a)", + "SQL": "select count(1), count(a), count(b) from t -- shouldn't be rewritten", "Plan": [ "StreamAgg 1.00 root funcs:count(Column#16)->Column#10, funcs:count(Column#17)->Column#11, funcs:count(Column#18)->Column#12", "└─TableReader 1.00 root data:StreamAgg", - " └─StreamAgg 1.00 cop[tikv] funcs:count(test.t.a)->Column#16, funcs:count(test.t.a)->Column#17, funcs:count(test.t.b)->Column#18", + " └─StreamAgg 1.00 cop[tikv] funcs:count(1)->Column#16, funcs:count(test.t.a)->Column#17, funcs:count(test.t.b)->Column#18", " └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" ], "Warning": null @@ -6570,10 +6570,10 @@ { "SQL": "select count(*) from t", "Plan": [ - "HashAgg 1.00 root funcs:count(Column#19)->Column#10", + "HashAgg 1.00 root funcs:count(Column#12)->Column#10", "└─TableReader 1.00 root data:ExchangeSender", " └─ExchangeSender 1.00 mpp[tiflash] ExchangeType: PassThrough", - " └─HashAgg 1.00 mpp[tiflash] funcs:count(test.t.d)->Column#19", + " └─HashAgg 1.00 mpp[tiflash] funcs:count(test.t.d)->Column#12", " └─TableFullScan 10000.00 mpp[tiflash] table:t keep order:false, stats:pseudo" ], "Warning": null @@ -6581,10 +6581,10 @@ { "SQL": "select count(1), count(3.1415), count(0), count(null) from t -- every count but count(null) can be rewritten", "Plan": [ - "HashAgg 1.00 root funcs:count(Column#25)->Column#10, funcs:count(Column#26)->Column#11, funcs:count(Column#27)->Column#12, funcs:count(Column#28)->Column#13", + "HashAgg 1.00 root funcs:count(Column#18)->Column#10, funcs:count(Column#19)->Column#11, funcs:count(Column#20)->Column#12, funcs:count(Column#21)->Column#13", "└─TableReader 1.00 root data:ExchangeSender", " └─ExchangeSender 1.00 mpp[tiflash] ExchangeType: PassThrough", - " └─HashAgg 1.00 mpp[tiflash] funcs:count(test.t.d)->Column#25, funcs:count(test.t.d)->Column#26, funcs:count(test.t.d)->Column#27, funcs:count(NULL)->Column#28", + " └─HashAgg 1.00 mpp[tiflash] funcs:count(test.t.d)->Column#18, funcs:count(test.t.d)->Column#19, funcs:count(test.t.d)->Column#20, funcs:count(NULL)->Column#21", " └─TableFullScan 10000.00 mpp[tiflash] table:t keep order:false, stats:pseudo" ], "Warning": null @@ -6595,7 +6595,7 @@ "HashAgg 1.00 root funcs:count(Column#12)->Column#10", "└─TableReader 1.00 root data:ExchangeSender", " └─ExchangeSender 1.00 mpp[tiflash] ExchangeType: PassThrough", - " └─HashAgg 1.00 mpp[tiflash] funcs:count(test.t.a)->Column#12", + " └─HashAgg 1.00 mpp[tiflash] funcs:count(1)->Column#12", " └─Selection 10.00 mpp[tiflash] eq(test.t.a, 1)", " └─TableFullScan 10000.00 mpp[tiflash] table:t keep order:false, stats:pseudo" ], @@ -6616,10 +6616,10 @@ "SQL": "select t.b, t.c from (select count(*) as c from t) a, t where a.c=t.a -- test recursive", "Plan": [ "HashJoin 1.25 root inner join, equal:[eq(test.t.a, Column#10)]", - "├─HashAgg(Build) 1.00 root funcs:count(Column#29)->Column#10", + "├─HashAgg(Build) 1.00 root funcs:count(Column#22)->Column#10", "│ └─TableReader 1.00 root data:ExchangeSender", "│ └─ExchangeSender 1.00 mpp[tiflash] ExchangeType: PassThrough", - "│ └─HashAgg 1.00 mpp[tiflash] funcs:count(test.t.d)->Column#29", + "│ └─HashAgg 1.00 mpp[tiflash] funcs:count(test.t.d)->Column#22", "│ └─TableFullScan 10000.00 mpp[tiflash] table:t keep order:false, stats:pseudo", "└─TableReader(Probe) 10000.00 root data:TableFullScan", " └─TableFullScan 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" @@ -6636,7 +6636,7 @@ " └─HashAgg(Probe) 10000.00 root funcs:count(Column#21)->Column#19", " └─TableReader 10000.00 root data:ExchangeSender", " └─ExchangeSender 10000.00 mpp[tiflash] ExchangeType: PassThrough", - " └─HashAgg 10000.00 mpp[tiflash] funcs:count(test.t.a)->Column#21", + " └─HashAgg 10000.00 mpp[tiflash] funcs:count(1)->Column#21", " └─Selection 80000000.00 mpp[tiflash] eq(cast(test.t.a, double BINARY), cast(test.t.b, double BINARY))", " └─TableFullScan 100000000.00 mpp[tiflash] table:inn keep order:false, stats:pseudo" ], @@ -6663,7 +6663,7 @@ "TableReader 1.00 root data:ExchangeSender", "└─ExchangeSender 1.00 mpp[tiflash] ExchangeType: PassThrough", " └─Projection 1.00 mpp[tiflash] Column#10", - " └─HashAgg 1.00 mpp[tiflash] funcs:count(distinct Column#19)->Column#10", + " └─HashAgg 1.00 mpp[tiflash] funcs:count(distinct Column#12)->Column#10", " └─ExchangeReceiver 1.00 mpp[tiflash] ", " └─ExchangeSender 1.00 mpp[tiflash] ExchangeType: PassThrough", " └─HashAgg 1.00 mpp[tiflash] group by:1, ", @@ -6672,12 +6672,12 @@ "Warning": null }, { - "SQL": "select count(1), count(a), count(b) from t -- count(1) to count(a)", + "SQL": "select count(1), count(a), count(b) from t -- keep count(1)", "Plan": [ "HashAgg 1.00 root funcs:count(Column#16)->Column#10, funcs:count(Column#17)->Column#11, funcs:count(Column#18)->Column#12", "└─TableReader 1.00 root data:ExchangeSender", " └─ExchangeSender 1.00 mpp[tiflash] ExchangeType: PassThrough", - " └─HashAgg 1.00 mpp[tiflash] funcs:count(test.t.a)->Column#16, funcs:count(test.t.a)->Column#17, funcs:count(test.t.b)->Column#18", + " └─HashAgg 1.00 mpp[tiflash] funcs:count(1)->Column#16, funcs:count(test.t.a)->Column#17, funcs:count(test.t.b)->Column#18", " └─TableFullScan 10000.00 mpp[tiflash] table:t keep order:false, stats:pseudo" ], "Warning": null From 233ca375114c8eb2136c3d053659c09e117019cd Mon Sep 17 00:00:00 2001 From: elsa0520 Date: Thu, 1 Dec 2022 20:36:42 +0800 Subject: [PATCH 2/8] remove old logical rule --- planner/core/logical_plan_builder.go | 1 - planner/core/optimizer.go | 2 - planner/core/rule_column_pruning.go | 47 --------- planner/core/rule_count_star_rewriter.go | 121 ----------------------- 4 files changed, 171 deletions(-) delete mode 100644 planner/core/rule_count_star_rewriter.go diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index 6f0352395a8ed..ad28e951ccdba 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -204,7 +204,6 @@ func (b *PlanBuilder) buildAggregation(ctx context.Context, p LogicalPlan, aggFu b.optFlag |= flagPredicatePushDown b.optFlag |= flagEliminateAgg b.optFlag |= flagEliminateProjection -/* b.optFlag |= flagCountStarRewriter*/ if b.ctx.GetSessionVars().EnableSkewDistinctAgg { b.optFlag |= flagSkewDistinctAgg diff --git a/planner/core/optimizer.go b/planner/core/optimizer.go index 96ce047f65a3d..13ea4f72cd2e2 100644 --- a/planner/core/optimizer.go +++ b/planner/core/optimizer.go @@ -74,7 +74,6 @@ const ( flagSyncWaitStatsLoadPoint flagJoinReOrder flagPrunColumnsAgain - /* flagCountStarRewriter*/ ) var optRuleList = []logicalOptRule{ @@ -97,7 +96,6 @@ var optRuleList = []logicalOptRule{ &syncWaitStatsLoadPoint{}, &joinReOrderSolver{}, &columnPruner{}, // column pruning again at last, note it will mess up the results of buildKeySolver - /* &countStarRewriter{},*/ } type logicalOptimizeOp struct { diff --git a/planner/core/rule_column_pruning.go b/planner/core/rule_column_pruning.go index 2b38d6b74c5aa..34a5259abbd32 100644 --- a/planner/core/rule_column_pruning.go +++ b/planner/core/rule_column_pruning.go @@ -336,24 +336,9 @@ func (ds *DataSource) PruneColumns(parentUsedCols []*expression.Column, opt *log appendColumnPruneTraceStep(ds, prunedColumns, opt) // For SQL like `select 1 from t`, tikv's response will be empty if no column is in schema. // So we'll force to push one if schema doesn't have any column. - // There are two situations - // case 1: tiflash. Select a non-empty and narrowest column. - // The main reason is that tiflash is a column storage structure, - // and choosing the narrowest column can reduce the amount of data read as much as possible, - // making the reading efficiency the best. - // case 2: tikv. Select row_id or pk column. - // The main reason is that tikv is a kv store. - // Select the key column for the best read efficiency. if ds.schema.Len() == 0 { var handleCol *expression.Column var handleColInfo *model.ColumnInfo - // case 1: tiflash - /* if ds.tableInfo.TiFlashReplica != nil { - handleCol, handleColInfo = preferNotNullColumnFromTable(ds) - } else { - // case 2: tikv - handleCol, handleColInfo = preferKeyColumnFromTable(ds, originSchemaColumns, originColumns) - }*/ handleCol, handleColInfo = preferKeyColumnFromTable(ds, originSchemaColumns, originColumns) ds.Columns = append(ds.Columns, handleColInfo) ds.schema.Append(handleCol) @@ -662,38 +647,6 @@ func appendItemPruneTraceStep(p LogicalPlan, itemType string, prunedObjects []fm opt.appendStepToCurrent(p.ID(), p.TP(), reason, action) } -// pick a not null and narrowest column from table -func preferNotNullColumnFromTable(dataSource *DataSource) (*expression.Column, *model.ColumnInfo) { - var resultColumnInfo *model.ColumnInfo - var resultColumn *expression.Column - - if dataSource.handleCols != nil { - resultColumn = dataSource.handleCols.GetCol(0) - resultColumnInfo = resultColumn.ToInfo() - } else { - resultColumn = dataSource.newExtraHandleSchemaCol() - resultColumnInfo = model.NewExtraHandleColInfo() - } - - for _, columnInfo := range dataSource.tableInfo.Columns { - if columnInfo.FieldType.IsVarLengthType() { - continue - } - if mysql.HasNotNullFlag(columnInfo.GetFlag()) { - if columnInfo.GetFlen() < resultColumnInfo.GetFlen() { - resultColumnInfo = columnInfo - resultColumn = &expression.Column{ - UniqueID: dataSource.ctx.GetSessionVars().AllocPlanColumnID(), - ID: resultColumnInfo.ID, - RetType: resultColumnInfo.FieldType.Clone(), - OrigName: fmt.Sprintf("%s.%s.%s", dataSource.DBName.L, dataSource.tableInfo.Name.L, resultColumnInfo.Name), - } - } - } - } - return resultColumn, resultColumnInfo -} - func preferKeyColumnFromTable(dataSource *DataSource, originColumns []*expression.Column, originSchemaColumns []*model.ColumnInfo) (*expression.Column, *model.ColumnInfo) { var resultColumnInfo *model.ColumnInfo diff --git a/planner/core/rule_count_star_rewriter.go b/planner/core/rule_count_star_rewriter.go deleted file mode 100644 index 97835b311d8ff..0000000000000 --- a/planner/core/rule_count_star_rewriter.go +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright 2022 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package core - -import ( - "context" - - "github.com/pingcap/tidb/expression" - "github.com/pingcap/tidb/expression/aggregation" - "github.com/pingcap/tidb/parser/mysql" -) - -/** -The countStarRewriter is used to rewrite - count(*) -> count(not null column) -Attention: -Since count(*) is directly translated into count(1) during grammar parsing, -the rewritten pattern actually matches count(constant) - -Pattern: -LogcialAggregation: count(constant) - | - DataSource - -Optimize: -Table - - -Case1 there are columns from datasource -Query: select count(*) from table where k3=1 -CountStarRewriterRule: pick the narrowest not null column from datasource -Rewritten Query: select count(k3) from table where k3=1 - -Case2 there is no columns from datasource -Query: select count(*) from table -ColumnPruningRule: pick k1 as the narrowest not null column from origin table @Function.preferNotNullColumnFromTable - datasource.columns: k1 -CountStarRewriterRule: rewrite count(*) -> count(k1) -Rewritten Query: select count(k1) from table - -*/ - -type countStarRewriter struct { -} - -func (c *countStarRewriter) optimize(ctx context.Context, p LogicalPlan, opt *logicalOptimizeOp) (LogicalPlan, error) { - return c.countStarRewriter(p, opt) -} - -func (c *countStarRewriter) countStarRewriter(p LogicalPlan, opt *logicalOptimizeOp) (LogicalPlan, error) { - c.matchPatternAndRewrite(p) - newChildren := make([]LogicalPlan, 0, len(p.Children())) - for _, child := range p.Children() { - newChild, err := c.countStarRewriter(child, opt) - if err != nil { - return nil, err - } - newChildren = append(newChildren, newChild) - } - p.SetChildren(newChildren...) - return p, nil -} - -func (c *countStarRewriter) matchPatternAndRewrite(p LogicalPlan) { - // match pattern agg(count(constant)) -> datasource - agg, ok := p.(*LogicalAggregation) - if !ok || len(agg.GroupByItems) > 0 { - return - } - dataSource, ok := agg.Children()[0].(*DataSource) - if !ok { - return - } - for _, aggFunc := range agg.AggFuncs { - if aggFunc.Name != "count" || len(aggFunc.Args) != 1 || aggFunc.HasDistinct { - continue - } - constExpr, ok := aggFunc.Args[0].(*expression.Constant) - if !ok || constExpr.Value.IsNull() || len(dataSource.Columns) == 0 { - continue - } - // rewrite - rewriteCountConstantToCountColumn(dataSource, aggFunc) - } -} - -// Pick the narrowest and not null column from Data Source -// If there is no not null column in Data Source, the count(constant) will not be rewritten. -func rewriteCountConstantToCountColumn(dataSource *DataSource, aggFunc *aggregation.AggFuncDesc) { - var newAggColumn *expression.Column - for _, columnFromDataSource := range dataSource.schema.Columns { - if columnFromDataSource.GetType().IsVarLengthType() { - continue - } - if mysql.HasNotNullFlag(columnFromDataSource.GetType().GetFlag()) { - if newAggColumn == nil || columnFromDataSource.GetType().GetFlen() < newAggColumn.GetType().GetFlen() { - newAggColumn = columnFromDataSource - } - } - } - if newAggColumn != nil { - // update count(1) -> count(newAggColumn) - aggFunc.Args[0] = newAggColumn - } -} - -func (*countStarRewriter) name() string { - return "count_star_rewriter" -} From 8885d6c5da443876e6efe01c865034b9d814aa81 Mon Sep 17 00:00:00 2001 From: elsa0520 Date: Thu, 1 Dec 2022 21:06:21 +0800 Subject: [PATCH 3/8] fix ut --- cmd/explaintest/r/access_path_selection.result | 2 +- cmd/explaintest/r/clustered_index.result | 16 ++++++++-------- cmd/explaintest/r/explain_easy.result | 4 ++-- cmd/explaintest/r/index_merge.result | 10 +++++----- planner/core/BUILD.bazel | 1 - planner/core/partition_pruner_test.go | 4 ++-- planner/core/testdata/enforce_mpp_suite_out.json | 16 ++++++++-------- planner/core/testdata/integration_suite_out.json | 16 ++++++++-------- 8 files changed, 34 insertions(+), 35 deletions(-) diff --git a/cmd/explaintest/r/access_path_selection.result b/cmd/explaintest/r/access_path_selection.result index 75ede06c71dd3..cafdc72269eed 100644 --- a/cmd/explaintest/r/access_path_selection.result +++ b/cmd/explaintest/r/access_path_selection.result @@ -41,5 +41,5 @@ explain format = 'brief' select count(1) from access_path_selection; id estRows task access object operator info StreamAgg 1.00 root funcs:count(Column#18)->Column#4 └─TableReader 1.00 root data:StreamAgg - └─StreamAgg 1.00 cop[tikv] funcs:count(test.access_path_selection._tidb_rowid)->Column#18 + └─StreamAgg 1.00 cop[tikv] funcs:count(1)->Column#18 └─TableFullScan 10000.00 cop[tikv] table:access_path_selection keep order:false, stats:pseudo diff --git a/cmd/explaintest/r/clustered_index.result b/cmd/explaintest/r/clustered_index.result index d3e131e6a5a65..bd1824d07edb3 100644 --- a/cmd/explaintest/r/clustered_index.result +++ b/cmd/explaintest/r/clustered_index.result @@ -30,25 +30,25 @@ explain select count(*) from with_cluster_index.tbl_0 where col_0 < 5429 ; id estRows task access object operator info StreamAgg_17 1.00 root funcs:count(Column#8)->Column#6 └─IndexReader_18 1.00 root index:StreamAgg_9 - └─StreamAgg_9 1.00 cop[tikv] funcs:count(with_cluster_index.tbl_0.col_0)->Column#8 + └─StreamAgg_9 1.00 cop[tikv] funcs:count(1)->Column#8 └─IndexRangeScan_16 798.90 cop[tikv] table:tbl_0, index:idx_3(col_0) range:[-inf,5429), keep order:false explain select count(*) from wout_cluster_index.tbl_0 where col_0 < 5429 ; id estRows task access object operator info StreamAgg_17 1.00 root funcs:count(Column#9)->Column#7 └─IndexReader_18 1.00 root index:StreamAgg_9 - └─StreamAgg_9 1.00 cop[tikv] funcs:count(wout_cluster_index.tbl_0.col_0)->Column#9 + └─StreamAgg_9 1.00 cop[tikv] funcs:count(1)->Column#9 └─IndexRangeScan_16 798.90 cop[tikv] table:tbl_0, index:idx_3(col_0) range:[-inf,5429), keep order:false explain select count(*) from with_cluster_index.tbl_0 where col_0 < 41 ; id estRows task access object operator info StreamAgg_17 1.00 root funcs:count(Column#8)->Column#6 └─IndexReader_18 1.00 root index:StreamAgg_9 - └─StreamAgg_9 1.00 cop[tikv] funcs:count(with_cluster_index.tbl_0.col_0)->Column#8 + └─StreamAgg_9 1.00 cop[tikv] funcs:count(1)->Column#8 └─IndexRangeScan_16 41.00 cop[tikv] table:tbl_0, index:idx_3(col_0) range:[-inf,41), keep order:false explain select count(*) from wout_cluster_index.tbl_0 where col_0 < 41 ; id estRows task access object operator info StreamAgg_17 1.00 root funcs:count(Column#9)->Column#7 └─IndexReader_18 1.00 root index:StreamAgg_9 - └─StreamAgg_9 1.00 cop[tikv] funcs:count(wout_cluster_index.tbl_0.col_0)->Column#9 + └─StreamAgg_9 1.00 cop[tikv] funcs:count(1)->Column#9 └─IndexRangeScan_16 41.00 cop[tikv] table:tbl_0, index:idx_3(col_0) range:[-inf,41), keep order:false explain select col_14 from with_cluster_index.tbl_2 where col_11 <> '2013-11-01' ; id estRows task access object operator info @@ -109,24 +109,24 @@ explain select count(*) from with_cluster_index.tbl_0 where col_0 <= 0 ; id estRows task access object operator info StreamAgg_16 1.00 root funcs:count(Column#8)->Column#6 └─IndexReader_17 1.00 root index:StreamAgg_9 - └─StreamAgg_9 1.00 cop[tikv] funcs:count(with_cluster_index.tbl_0.col_0)->Column#8 + └─StreamAgg_9 1.00 cop[tikv] funcs:count(1)->Column#8 └─IndexRangeScan_11 1.00 cop[tikv] table:tbl_0, index:idx_3(col_0) range:[-inf,0], keep order:false explain select count(*) from wout_cluster_index.tbl_0 where col_0 <= 0 ; id estRows task access object operator info StreamAgg_16 1.00 root funcs:count(Column#9)->Column#7 └─IndexReader_17 1.00 root index:StreamAgg_9 - └─StreamAgg_9 1.00 cop[tikv] funcs:count(wout_cluster_index.tbl_0.col_0)->Column#9 + └─StreamAgg_9 1.00 cop[tikv] funcs:count(1)->Column#9 └─IndexRangeScan_11 1.00 cop[tikv] table:tbl_0, index:idx_3(col_0) range:[-inf,0], keep order:false explain select count(*) from with_cluster_index.tbl_0 where col_0 >= 803163 ; id estRows task access object operator info StreamAgg_17 1.00 root funcs:count(Column#8)->Column#6 └─IndexReader_18 1.00 root index:StreamAgg_9 - └─StreamAgg_9 1.00 cop[tikv] funcs:count(with_cluster_index.tbl_0.col_0)->Column#8 + └─StreamAgg_9 1.00 cop[tikv] funcs:count(1)->Column#8 └─IndexRangeScan_16 109.70 cop[tikv] table:tbl_0, index:idx_3(col_0) range:[803163,+inf], keep order:false explain select count(*) from wout_cluster_index.tbl_0 where col_0 >= 803163 ; id estRows task access object operator info StreamAgg_17 1.00 root funcs:count(Column#9)->Column#7 └─IndexReader_18 1.00 root index:StreamAgg_9 - └─StreamAgg_9 1.00 cop[tikv] funcs:count(wout_cluster_index.tbl_0.col_0)->Column#9 + └─StreamAgg_9 1.00 cop[tikv] funcs:count(1)->Column#9 └─IndexRangeScan_16 109.70 cop[tikv] table:tbl_0, index:idx_3(col_0) range:[803163,+inf], keep order:false set @@tidb_enable_outer_join_reorder=false; diff --git a/cmd/explaintest/r/explain_easy.result b/cmd/explaintest/r/explain_easy.result index bf0b3e4c2884b..6e367f1f1a0dc 100644 --- a/cmd/explaintest/r/explain_easy.result +++ b/cmd/explaintest/r/explain_easy.result @@ -218,14 +218,14 @@ id estRows task access object operator info Projection 1.00 root 1->Column#6 └─StreamAgg 1.00 root funcs:count(Column#14)->Column#9 └─TableReader 1.00 root data:StreamAgg - └─StreamAgg 1.00 cop[tikv] funcs:count(test.t1.c1)->Column#14 + └─StreamAgg 1.00 cop[tikv] funcs:count(1)->Column#14 └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo explain format = 'brief' select count(1) from (select max(c2), count(c3) as m from t1) k; id estRows task access object operator info StreamAgg 1.00 root funcs:count(1)->Column#6 └─StreamAgg 1.00 root funcs:count(Column#13)->Column#8 └─TableReader 1.00 root data:StreamAgg - └─StreamAgg 1.00 cop[tikv] funcs:count(test.t1.c1)->Column#13 + └─StreamAgg 1.00 cop[tikv] funcs:count(1)->Column#13 └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo explain format = 'brief' select count(1) from (select count(c2) from t1 group by c3) k; id estRows task access object operator info diff --git a/cmd/explaintest/r/index_merge.result b/cmd/explaintest/r/index_merge.result index 7d798cb972006..57835dbaed539 100644 --- a/cmd/explaintest/r/index_merge.result +++ b/cmd/explaintest/r/index_merge.result @@ -126,7 +126,7 @@ Sort_11 5098.44 root test.t1.c1 ├─StreamAgg_23(Build) 1.00 root funcs:min(Column#9)->Column#10, funcs:sum(0)->Column#11, funcs:count(1)->Column#12 │ └─StreamAgg_43 1.00 root funcs:count(Column#25)->Column#9 │ └─IndexReader_44 1.00 root index:StreamAgg_27 - │ └─StreamAgg_27 1.00 cop[tikv] funcs:count(test.t2._tidb_rowid)->Column#25 + │ └─StreamAgg_27 1.00 cop[tikv] funcs:count(1)->Column#25 │ └─IndexFullScan_41 10000.00 cop[tikv] table:t2, index:c1(c1) keep order:false, stats:pseudo └─IndexMerge_21(Probe) 2825.66 root type: union ├─IndexRangeScan_17(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo @@ -148,7 +148,7 @@ Sort_11 5098.44 root test.t1.c1 ├─StreamAgg_23(Build) 1.00 root funcs:min(Column#9)->Column#10, funcs:sum(0)->Column#11, funcs:count(1)->Column#12 │ └─StreamAgg_43 1.00 root funcs:count(Column#25)->Column#9 │ └─IndexReader_44 1.00 root index:StreamAgg_27 - │ └─StreamAgg_27 1.00 cop[tikv] funcs:count(test.t2._tidb_rowid)->Column#25 + │ └─StreamAgg_27 1.00 cop[tikv] funcs:count(1)->Column#25 │ └─IndexFullScan_41 10000.00 cop[tikv] table:t2, index:c1(c1) keep order:false, stats:pseudo └─IndexMerge_21(Probe) 2825.66 root type: union ├─IndexRangeScan_17(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo @@ -170,7 +170,7 @@ Sort_11 5542.21 root test.t1.c1 ├─StreamAgg_22(Build) 1.00 root funcs:max(Column#9)->Column#10, funcs:sum(0)->Column#11, funcs:count(1)->Column#12 │ └─StreamAgg_42 1.00 root funcs:count(Column#25)->Column#9 │ └─IndexReader_43 1.00 root index:StreamAgg_26 - │ └─StreamAgg_26 1.00 cop[tikv] funcs:count(test.t2._tidb_rowid)->Column#25 + │ └─StreamAgg_26 1.00 cop[tikv] funcs:count(1)->Column#25 │ └─IndexFullScan_40 10000.00 cop[tikv] table:t2, index:c1(c1) keep order:false, stats:pseudo └─IndexMerge_20(Probe) 5542.21 root type: union ├─IndexRangeScan_17(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo @@ -192,7 +192,7 @@ Sort_39 5542.21 root test.t1.c1 ├─StreamAgg_50(Build) 1.00 root funcs:max(Column#13)->Column#14, funcs:sum(0)->Column#15, funcs:count(1)->Column#16 │ └─StreamAgg_70 1.00 root funcs:count(Column#38)->Column#13 │ └─IndexReader_71 1.00 root index:StreamAgg_54 - │ └─StreamAgg_54 1.00 cop[tikv] funcs:count(test.t2._tidb_rowid)->Column#38 + │ └─StreamAgg_54 1.00 cop[tikv] funcs:count(1)->Column#38 │ └─IndexFullScan_68 10000.00 cop[tikv] table:t2, index:c1(c1) keep order:false, stats:pseudo └─IndexMerge_48(Probe) 5542.21 root type: union ├─IndexRangeScan_45(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo @@ -323,7 +323,7 @@ drop view if exists v2; create view v2 as select /*+ use_index_merge(t1) */ * from t1 where c1 < 10 or c2 < 10 and c3 < 10; show create view v2; View Create View character_set_client collation_connection -v2 CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`%` SQL SECURITY DEFINER VIEW `v2` (`c1`, `c2`, `c3`) AS SELECT /*+ USE_INDEX_MERGE(`t1` )*/ `test`.`t1`.`c1` AS `c1`,`test`.`t1`.`c2` AS `c2`,`test`.`t1`.`c3` AS `c3` FROM `test`.`t1` WHERE `c1`<10 OR `c2`<10 AND `c3`<10 utf8mb4 utf8mb4_bin +v2 CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`%` SQL SECURITY DEFINER VIEW `v2` (`c1`, `c2`, `c3`) AS SELECT /*+ USE_INDEX_MERGE(`t1` )*/ `test`.`t1`.`c1` AS `c1`,`test`.`t1`.`c2` AS `c2`,`test`.`t1`.`c3` AS `c3` FROM `test`.`t1` WHERE `c1`<10 OR `c2`<10 AND `c3`<10 utf8mb4 utf8mb4_general_ci select * from v2 order by 1; c1 c2 c3 1 1 1 diff --git a/planner/core/BUILD.bazel b/planner/core/BUILD.bazel index 5b5d6b150d971..dcbb25512f458 100644 --- a/planner/core/BUILD.bazel +++ b/planner/core/BUILD.bazel @@ -48,7 +48,6 @@ go_library( "rule_aggregation_skew_rewrite.go", "rule_build_key_info.go", "rule_column_pruning.go", - "rule_count_star_rewriter.go", "rule_decorrelate.go", "rule_eliminate_projection.go", "rule_generate_column_substitute.go", diff --git a/planner/core/partition_pruner_test.go b/planner/core/partition_pruner_test.go index f037d7fe887e7..166f251b53fc3 100644 --- a/planner/core/partition_pruner_test.go +++ b/planner/core/partition_pruner_test.go @@ -91,12 +91,12 @@ func TestRangeColumnPartitionPruningForIn(t *testing.T) { "└─PartitionUnion 2.00 root ", " ├─HashAgg 1.00 root funcs:count(Column#7)->Column#5", " │ └─IndexReader 1.00 root index:HashAgg", - " │ └─HashAgg 1.00 cop[tikv] funcs:count(test_range_col_in.t1.dt)->Column#7", + " │ └─HashAgg 1.00 cop[tikv] funcs:count(1)->Column#7", " │ └─Selection 20.00 cop[tikv] in(test_range_col_in.t1.dt, 2020-11-27 00:00:00.000000, 2020-11-28 00:00:00.000000)", " │ └─IndexFullScan 10000.00 cop[tikv] table:t1, partition:p20201127, index:PRIMARY(id, dt) keep order:false, stats:pseudo", " └─HashAgg 1.00 root funcs:count(Column#10)->Column#5", " └─IndexReader 1.00 root index:HashAgg", - " └─HashAgg 1.00 cop[tikv] funcs:count(test_range_col_in.t1.dt)->Column#10", + " └─HashAgg 1.00 cop[tikv] funcs:count(1)->Column#10", " └─Selection 20.00 cop[tikv] in(test_range_col_in.t1.dt, 2020-11-27 00:00:00.000000, 2020-11-28 00:00:00.000000)", " └─IndexFullScan 10000.00 cop[tikv] table:t1, partition:p20201128, index:PRIMARY(id, dt) keep order:false, stats:pseudo")) diff --git a/planner/core/testdata/enforce_mpp_suite_out.json b/planner/core/testdata/enforce_mpp_suite_out.json index 12cde0ede133e..91de0d5bd1348 100644 --- a/planner/core/testdata/enforce_mpp_suite_out.json +++ b/planner/core/testdata/enforce_mpp_suite_out.json @@ -1076,8 +1076,8 @@ " └─HashAgg_28 1.00 mpp[tiflash] funcs:sum(Column#18)->Column#21, funcs:sum(Column#19)->Column#22, funcs:sum(Column#20)->Column#23, funcs:count(distinct test.t.c)->Column#24", " └─ExchangeReceiver_30 1.00 mpp[tiflash] ", " └─ExchangeSender_29 1.00 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: test.t.c, collate: binary]", - " └─HashAgg_26 1.00 mpp[tiflash] group by:Column#28, funcs:sum(Column#25)->Column#18, funcs:count(Column#26)->Column#19, funcs:count(Column#27)->Column#20", - " └─Projection_35 10000.00 mpp[tiflash] cast(test.t.b, decimal(20,0) BINARY)->Column#25, test.t.a, test.t.b, test.t.c", + " └─HashAgg_26 1.00 mpp[tiflash] group by:Column#27, funcs:sum(Column#25)->Column#18, funcs:count(Column#26)->Column#19, funcs:count(1)->Column#20", + " └─Projection_35 10000.00 mpp[tiflash] cast(test.t.b, decimal(20,0) BINARY)->Column#25, test.t.a, test.t.c", " └─TableFullScan_13 10000.00 mpp[tiflash] table:t keep order:false, stats:pseudo" ], "Warn": null @@ -1095,8 +1095,8 @@ " └─HashAgg_32 1.00 mpp[tiflash] funcs:sum(Column#22)->Column#25, funcs:sum(Column#23)->Column#26, funcs:count(distinct test.t.c)->Column#27, funcs:sum(Column#24)->Column#28", " └─ExchangeReceiver_34 1.00 mpp[tiflash] ", " └─ExchangeSender_33 1.00 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: test.t.c, collate: binary]", - " └─HashAgg_30 1.00 mpp[tiflash] group by:Column#32, funcs:sum(Column#29)->Column#22, funcs:count(Column#30)->Column#23, funcs:count(Column#31)->Column#24", - " └─Projection_42 10000.00 mpp[tiflash] cast(plus(test.t.b, test.t.a), decimal(20,0) BINARY)->Column#29, test.t.b, test.t.a, test.t.c", + " └─HashAgg_30 1.00 mpp[tiflash] group by:Column#31, funcs:sum(Column#29)->Column#22, funcs:count(1)->Column#23, funcs:count(Column#30)->Column#24", + " └─Projection_42 10000.00 mpp[tiflash] cast(plus(test.t.b, test.t.a), decimal(20,0) BINARY)->Column#29, test.t.a, test.t.c", " └─TableFullScan_17 10000.00 mpp[tiflash] table:t keep order:false, stats:pseudo" ], "Warn": null @@ -1115,8 +1115,8 @@ " └─HashAgg_32 1.00 mpp[tiflash] funcs:sum(Column#21)->Column#24, funcs:sum(Column#22)->Column#25, funcs:sum(Column#23)->Column#26, funcs:count(distinct test.t.c)->Column#27", " └─ExchangeReceiver_34 1.00 mpp[tiflash] ", " └─ExchangeSender_33 1.00 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: test.t.c, collate: binary]", - " └─HashAgg_30 1.00 mpp[tiflash] group by:Column#31, funcs:sum(Column#28)->Column#21, funcs:count(Column#29)->Column#22, funcs:count(Column#30)->Column#23", - " └─Projection_42 10000.00 mpp[tiflash] cast(plus(test.t.b, test.t.a), decimal(20,0) BINARY)->Column#28, test.t.b, test.t.a, test.t.c", + " └─HashAgg_30 1.00 mpp[tiflash] group by:Column#30, funcs:sum(Column#28)->Column#21, funcs:count(1)->Column#22, funcs:count(Column#29)->Column#23", + " └─Projection_42 10000.00 mpp[tiflash] cast(plus(test.t.b, test.t.a), decimal(20,0) BINARY)->Column#28, test.t.a, test.t.c", " └─TableFullScan_17 10000.00 mpp[tiflash] table:t keep order:false, stats:pseudo" ], "Warn": null @@ -1135,8 +1135,8 @@ " └─HashAgg_32 1.00 mpp[tiflash] funcs:sum(Column#25)->Column#29, funcs:max(Column#26)->Column#30, funcs:count(distinct test.t.c)->Column#31, funcs:sum(Column#27)->Column#32, funcs:sum(Column#28)->Column#33", " └─ExchangeReceiver_34 1.00 mpp[tiflash] ", " └─ExchangeSender_33 1.00 mpp[tiflash] ExchangeType: HashPartition, Hash Cols: [name: test.t.c, collate: binary]", - " └─HashAgg_30 1.00 mpp[tiflash] group by:Column#38, funcs:sum(Column#34)->Column#25, funcs:max(Column#35)->Column#26, funcs:count(Column#36)->Column#27, funcs:count(Column#37)->Column#28", - " └─Projection_42 10000.00 mpp[tiflash] cast(plus(test.t.b, test.t.a), decimal(20,0) BINARY)->Column#34, test.t.b, test.t.b, test.t.a, test.t.c", + " └─HashAgg_30 1.00 mpp[tiflash] group by:Column#37, funcs:sum(Column#34)->Column#25, funcs:max(Column#35)->Column#26, funcs:count(1)->Column#27, funcs:count(Column#36)->Column#28", + " └─Projection_42 10000.00 mpp[tiflash] cast(plus(test.t.b, test.t.a), decimal(20,0) BINARY)->Column#34, test.t.b, test.t.a, test.t.c", " └─TableFullScan_17 10000.00 mpp[tiflash] table:t keep order:false, stats:pseudo" ], "Warn": null diff --git a/planner/core/testdata/integration_suite_out.json b/planner/core/testdata/integration_suite_out.json index 5527622476829..14c04c6cfb0ab 100644 --- a/planner/core/testdata/integration_suite_out.json +++ b/planner/core/testdata/integration_suite_out.json @@ -1486,7 +1486,7 @@ { "SQL": "explain format = 'brief' SELECT /*+ use_index_merge(t1)*/ COUNT(*) FROM t1 WHERE (key4=42 AND key6 IS NOT NULL) OR (key1=4 AND key3=6)", "Plan": [ - "StreamAgg 1.00 root funcs:count(test.t1.key1)->Column#10", + "StreamAgg 1.00 root funcs:count(1)->Column#10", "└─IndexMerge 0.02 root type: union", " ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t1, index:i4(key4) range:[42,42], keep order:false, stats:pseudo", " ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t1, index:i1(key1) range:[4,4], keep order:false, stats:pseudo", @@ -2487,7 +2487,7 @@ "Plan": [ "StreamAgg_20 1.00 102.69 root funcs:count(Column#9)->Column#4", "└─IndexReader_21 1.00 52.79 root index:StreamAgg_8", - " └─StreamAgg_8 1.00 760.20 cop[tikv] funcs:count(test.t3._tidb_rowid)->Column#9", + " └─StreamAgg_8 1.00 760.20 cop[tikv] funcs:count(1)->Column#9", " └─IndexFullScan_19 3.00 610.50 cop[tikv] table:t3, index:c(b) keep order:false" ] }, @@ -2496,7 +2496,7 @@ "Plan": [ "StreamAgg_26 1.00 107.45 root funcs:count(Column#7)->Column#4", "└─TableReader_27 1.00 57.55 root data:StreamAgg_10", - " └─StreamAgg_10 1.00 831.62 cop[tikv] funcs:count(test.t2._tidb_rowid)->Column#7", + " └─StreamAgg_10 1.00 831.62 cop[tikv] funcs:count(1)->Column#7", " └─TableFullScan_24 3.00 681.92 cop[tikv] table:t2 keep order:false" ] }, @@ -6519,7 +6519,7 @@ "Plan": [ "HashAgg 1.00 root funcs:count(Column#15)->Column#14", "└─TableReader 1.00 root data:HashAgg", - " └─HashAgg 1.00 cop[tikv] funcs:count(test.t._tidb_rowid)->Column#15", + " └─HashAgg 1.00 cop[tikv] funcs:count(1)->Column#15", " └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" ] }, @@ -6546,7 +6546,7 @@ "Plan": [ "StreamAgg 1.00 root funcs:count(Column#15)->Column#14", "└─TableReader 1.00 root data:StreamAgg", - " └─StreamAgg 1.00 cop[tikv] funcs:count(test.t._tidb_rowid)->Column#15", + " └─StreamAgg 1.00 cop[tikv] funcs:count(1)->Column#15", " └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" ] }, @@ -8358,8 +8358,8 @@ "TableReader 1.00 root data:ExchangeSender", "└─ExchangeSender 1.00 mpp[tiflash] ExchangeType: PassThrough", " └─Projection 1.00 mpp[tiflash] Column#5, Column#6, Column#7", - " └─HashAgg 1.00 mpp[tiflash] funcs:group_concat(Column#8, Column#9, Column#10 order by Column#11 separator \",\")->Column#5, funcs:count(Column#12)->Column#6, funcs:min(Column#13)->Column#7", - " └─Projection 10000.00 mpp[tiflash] test.ts.col_0, test.ts.col_1, cast(test.ts.id, var_string(20))->Column#10, test.ts.col_0, test.ts.id, test.ts.col_1", + " └─HashAgg 1.00 mpp[tiflash] funcs:group_concat(Column#8, Column#9, Column#10 order by Column#11 separator \",\")->Column#5, funcs:count(1)->Column#6, funcs:min(Column#12)->Column#7", + " └─Projection 10000.00 mpp[tiflash] test.ts.col_0, test.ts.col_1, cast(test.ts.id, var_string(20))->Column#10, test.ts.col_0, test.ts.col_1", " └─ExchangeReceiver 10000.00 mpp[tiflash] ", " └─ExchangeSender 10000.00 mpp[tiflash] ExchangeType: PassThrough", " └─TableFullScan 10000.00 mpp[tiflash] table:ts keep order:false, stats:pseudo" @@ -8378,7 +8378,7 @@ " └─Projection 1.00 mpp[tiflash] test.ts.col_0, test.ts.col_1, cast(test.ts.id, var_string(20))->Column#14, test.ts.col_0, Column#10, Column#11", " └─ExchangeReceiver 1.00 mpp[tiflash] ", " └─ExchangeSender 1.00 mpp[tiflash] ExchangeType: PassThrough", - " └─HashAgg 1.00 mpp[tiflash] group by:test.ts.col_0, test.ts.col_1, test.ts.id, funcs:count(test.ts.id)->Column#10, funcs:max(test.ts.col_0)->Column#11", + " └─HashAgg 1.00 mpp[tiflash] group by:test.ts.col_0, test.ts.col_1, test.ts.id, funcs:count(1)->Column#10, funcs:max(test.ts.col_0)->Column#11", " └─TableFullScan 10000.00 mpp[tiflash] table:ts keep order:false, stats:pseudo" ], "Warning": [ From d4b6e88bfc79d8b2088b6520ebf7a2b68692fcee Mon Sep 17 00:00:00 2001 From: elsa0520 Date: Thu, 1 Dec 2022 21:13:49 +0800 Subject: [PATCH 4/8] fix ut --- cmd/explaintest/r/index_merge.result | 2 +- planner/core/optimizer.go | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/cmd/explaintest/r/index_merge.result b/cmd/explaintest/r/index_merge.result index 57835dbaed539..0233dbdb55f52 100644 --- a/cmd/explaintest/r/index_merge.result +++ b/cmd/explaintest/r/index_merge.result @@ -323,7 +323,7 @@ drop view if exists v2; create view v2 as select /*+ use_index_merge(t1) */ * from t1 where c1 < 10 or c2 < 10 and c3 < 10; show create view v2; View Create View character_set_client collation_connection -v2 CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`%` SQL SECURITY DEFINER VIEW `v2` (`c1`, `c2`, `c3`) AS SELECT /*+ USE_INDEX_MERGE(`t1` )*/ `test`.`t1`.`c1` AS `c1`,`test`.`t1`.`c2` AS `c2`,`test`.`t1`.`c3` AS `c3` FROM `test`.`t1` WHERE `c1`<10 OR `c2`<10 AND `c3`<10 utf8mb4 utf8mb4_general_ci +v2 CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`%` SQL SECURITY DEFINER VIEW `v2` (`c1`, `c2`, `c3`) AS SELECT /*+ USE_INDEX_MERGE(`t1` )*/ `test`.`t1`.`c1` AS `c1`,`test`.`t1`.`c2` AS `c2`,`test`.`t1`.`c3` AS `c3` FROM `test`.`t1` WHERE `c1`<10 OR `c2`<10 AND `c3`<10 utf8mb4 utf8mb4_bin select * from v2 order by 1; c1 c2 c3 1 1 1 diff --git a/planner/core/optimizer.go b/planner/core/optimizer.go index 13ea4f72cd2e2..2fdf619842f8b 100644 --- a/planner/core/optimizer.go +++ b/planner/core/optimizer.go @@ -16,7 +16,6 @@ package core import ( "context" - "fmt" "math" "github.com/pingcap/errors" @@ -573,14 +572,10 @@ func countStarRewrite(plan PhysicalPlan) { countStarRewrite(child) } } - } func countStarRewriteInternal(plan PhysicalPlan) { // match pattern any agg(count(constant)) -> tablefullscan(tiflash) - if !plan.SCtx().GetSessionVars().InRestrictedSQL { - fmt.Println("") - } var physicalAgg *basePhysicalAgg switch x := plan.(type) { case *PhysicalHashAgg: From f3388fa3c5026c87561a67d35ff6b2cfaa99ea8b Mon Sep 17 00:00:00 2001 From: elsa0520 Date: Thu, 1 Dec 2022 21:15:39 +0800 Subject: [PATCH 5/8] fix compile error --- planner/core/optimizer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/planner/core/optimizer.go b/planner/core/optimizer.go index 2fdf619842f8b..8ad20be1143fa 100644 --- a/planner/core/optimizer.go +++ b/planner/core/optimizer.go @@ -16,6 +16,7 @@ package core import ( "context" + "fmt" "math" "github.com/pingcap/errors" From 76b60983a9109c4277cd5019aef4d64c9afba766 Mon Sep 17 00:00:00 2001 From: elsa0520 Date: Thu, 1 Dec 2022 22:06:14 +0800 Subject: [PATCH 6/8] change comment --- planner/core/optimizer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/planner/core/optimizer.go b/planner/core/optimizer.go index 8ad20be1143fa..1ac83cec371f0 100644 --- a/planner/core/optimizer.go +++ b/planner/core/optimizer.go @@ -605,7 +605,7 @@ func countStarRewriteInternal(plan PhysicalPlan) { rewriteTableScanAndAggArgs(physicalTableScan, physicalAgg.AggFuncs) } -// Pick the narrowest and not null column from table +// rewriteTableScanAndAggArgs Pick the narrowest and not null column from table // If there is no not null column in Data Source, the row_id or pk column will be retained func rewriteTableScanAndAggArgs(physicalTableScan *PhysicalTableScan, aggFuncs []*aggregation.AggFuncDesc) { var resultColumnInfo *model.ColumnInfo From bd8bfbf349f535643c035c239ba138ff78c950ed Mon Sep 17 00:00:00 2001 From: elsa0520 Date: Thu, 1 Dec 2022 22:33:44 +0800 Subject: [PATCH 7/8] clone column for agg args --- planner/core/optimizer.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/planner/core/optimizer.go b/planner/core/optimizer.go index 1ac83cec371f0..a37b696c0397b 100644 --- a/planner/core/optimizer.go +++ b/planner/core/optimizer.go @@ -634,6 +634,7 @@ func rewriteTableScanAndAggArgs(physicalTableScan *PhysicalTableScan, aggFuncs [ physicalTableScan.Columns[0] = resultColumnInfo physicalTableScan.schema.Columns[0] = resultColumn // agg args count(1) -> count(not null column) + args := resultColumn.Clone() for _, aggFunc := range aggFuncs { constExpr, ok := aggFunc.Args[0].(*expression.Constant) if !ok { @@ -643,7 +644,7 @@ func rewriteTableScanAndAggArgs(physicalTableScan *PhysicalTableScan, aggFuncs [ if constExpr.Value.IsNull() { continue } - aggFunc.Args[0] = resultColumn + aggFunc.Args[0] = args } } From 127867110bbc2b34fc2919258a36a6fdc0721278 Mon Sep 17 00:00:00 2001 From: elsa0520 Date: Thu, 1 Dec 2022 22:41:20 +0800 Subject: [PATCH 8/8] fix --- planner/core/optimizer.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/planner/core/optimizer.go b/planner/core/optimizer.go index a37b696c0397b..f28b721868cd4 100644 --- a/planner/core/optimizer.go +++ b/planner/core/optimizer.go @@ -633,8 +633,8 @@ func rewriteTableScanAndAggArgs(physicalTableScan *PhysicalTableScan, aggFuncs [ // table scan (row_id) -> (not null column) physicalTableScan.Columns[0] = resultColumnInfo physicalTableScan.schema.Columns[0] = resultColumn - // agg args count(1) -> count(not null column) - args := resultColumn.Clone() + // agg arg count(1) -> count(not null column) + arg := resultColumn.Clone() for _, aggFunc := range aggFuncs { constExpr, ok := aggFunc.Args[0].(*expression.Constant) if !ok { @@ -644,7 +644,7 @@ func rewriteTableScanAndAggArgs(physicalTableScan *PhysicalTableScan, aggFuncs [ if constExpr.Value.IsNull() { continue } - aggFunc.Args[0] = args + aggFunc.Args[0] = arg } }