From 4fe2484b4b038eb81d20770ccac755ab0b046c78 Mon Sep 17 00:00:00 2001 From: yisaer Date: Mon, 27 Dec 2021 17:59:30 +0800 Subject: [PATCH 1/7] trace column pruner Signed-off-by: yisaer --- planner/cascades/optimize.go | 2 +- planner/core/optimizer.go | 6 +- planner/core/plan.go | 6 +- planner/core/rule_column_pruning.go | 158 +++++++++++++++++++------ planner/core/rule_max_min_eliminate.go | 2 +- 5 files changed, 131 insertions(+), 43 deletions(-) diff --git a/planner/cascades/optimize.go b/planner/cascades/optimize.go index 0d1a0969d7568..c1d73f21f8c89 100644 --- a/planner/cascades/optimize.go +++ b/planner/cascades/optimize.go @@ -116,7 +116,7 @@ func (opt *Optimizer) FindBestPlan(sctx sessionctx.Context, logical plannercore. } func (opt *Optimizer) onPhasePreprocessing(sctx sessionctx.Context, plan plannercore.LogicalPlan) (plannercore.LogicalPlan, error) { - err := plan.PruneColumns(plan.Schema().Columns) + err := plan.PruneColumns(plan.Schema().Columns, nil) if err != nil { return nil, err } diff --git a/planner/core/optimizer.go b/planner/core/optimizer.go index 36ef3b376bf55..4b12a1c933996 100644 --- a/planner/core/optimizer.go +++ b/planner/core/optimizer.go @@ -104,21 +104,21 @@ func (op *logicalOptimizeOp) withEnableOptimizeTracer(tracer *tracing.LogicalOpt } func (op *logicalOptimizeOp) appendBeforeRuleOptimize(index int, name string, before LogicalPlan) { - if op.tracer == nil { + if op == nil || op.tracer == nil { return } op.tracer.AppendRuleTracerBeforeRuleOptimize(index, name, before.buildLogicalPlanTrace()) } func (op *logicalOptimizeOp) appendStepToCurrent(id int, tp string, reason, action func() string) { - if op.tracer == nil { + if op == nil || op.tracer == nil { return } op.tracer.AppendRuleTracerStepToCurrent(id, tp, reason(), action()) } func (op *logicalOptimizeOp) recordFinalLogicalPlan(final LogicalPlan) { - if op.tracer == nil { + if op == nil || op.tracer == nil { return } op.tracer.RecordFinalLogicalPlan(final.buildLogicalPlanTrace()) diff --git a/planner/core/plan.go b/planner/core/plan.go index fed517996af1a..46b20ac6ae43a 100644 --- a/planner/core/plan.go +++ b/planner/core/plan.go @@ -240,7 +240,7 @@ type LogicalPlan interface { PredicatePushDown([]expression.Expression, *logicalOptimizeOp) ([]expression.Expression, LogicalPlan) // PruneColumns prunes the unused columns. - PruneColumns([]*expression.Column) error + PruneColumns([]*expression.Column, *logicalOptimizeOp) error // findBestTask converts the logical plan to the physical plan. It's a new interface. // It is called recursively from the parent to the children to create the result physical plan. @@ -593,11 +593,11 @@ func (p *baseLogicalPlan) ExtractCorrelatedCols() []*expression.CorrelatedColumn } // PruneColumns implements LogicalPlan interface. -func (p *baseLogicalPlan) PruneColumns(parentUsedCols []*expression.Column) error { +func (p *baseLogicalPlan) PruneColumns(parentUsedCols []*expression.Column, opt *logicalOptimizeOp) error { if len(p.children) == 0 { return nil } - return p.children[0].PruneColumns(parentUsedCols) + return p.children[0].PruneColumns(parentUsedCols, opt) } // basePlan implements base Plan interface. diff --git a/planner/core/rule_column_pruning.go b/planner/core/rule_column_pruning.go index 1d144c72807f9..904e2e245a762 100644 --- a/planner/core/rule_column_pruning.go +++ b/planner/core/rule_column_pruning.go @@ -15,7 +15,9 @@ package core import ( + "bytes" "context" + "fmt" "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/expression/aggregation" @@ -30,7 +32,7 @@ type columnPruner struct { } func (s *columnPruner) optimize(ctx context.Context, lp LogicalPlan, opt *logicalOptimizeOp) (LogicalPlan, error) { - err := lp.PruneColumns(lp.Schema().Columns) + err := lp.PruneColumns(lp.Schema().Columns, opt) return lp, err } @@ -61,34 +63,105 @@ func exprHasSetVarOrSleep(expr expression.Expression) bool { return false } +func appendColumnPruneTraceStep(p LogicalPlan, prunedColumns []*expression.Column, opt *logicalOptimizeOp) { + if len(prunedColumns) < 1 { + return + } + s := make([]string, 0) + for _, item := range prunedColumns { + s = append(s, item.String()) + } + appendItemPruneTraceStep(p, "columns", s, opt) +} + +func appendFunctionPruneTraceStep(p LogicalPlan, prunedFunctions []*aggregation.AggFuncDesc, opt *logicalOptimizeOp) { + if len(prunedFunctions) < 1 { + return + } + s := make([]string, 0) + for _, item := range prunedFunctions { + s = append(s, item.String()) + } + appendItemPruneTraceStep(p, "aggregation functions", s, opt) +} + +func appendByItemsPruneTraceStep(p LogicalPlan, prunedByItems []*util.ByItems, opt *logicalOptimizeOp) { + if len(prunedByItems) < 1 { + return + } + s := make([]string, 0) + for _, item := range prunedByItems { + s = append(s, item.String()) + } + appendItemPruneTraceStep(p, "byItems", s, opt) +} + +func appendItemPruneTraceStep(p LogicalPlan, itemType string, prunedObjects []string, opt *logicalOptimizeOp) { + if len(prunedObjects) < 1 { + return + } + action := func() string { + buffer := bytes.NewBufferString(fmt.Sprintf("%v_%v's %v[", p.TP(), p.ID(), itemType)) + for i, item := range prunedObjects { + if i > 0 { + buffer.WriteString(",") + } + buffer.WriteString(item) + } + buffer.WriteString("] have been pruned") + return buffer.String() + } + reason := func() string { + return "" + } + opt.appendStepToCurrent(p.ID(), p.TP(), reason, action) +} + +func appendGroupByItemsPruneTraceStep(p LogicalPlan, prunedGroupByItems []expression.Expression, opt *logicalOptimizeOp) { + if len(prunedGroupByItems) < 1 { + return + } + s := make([]string, 0) + for _, item := range prunedGroupByItems { + s = append(s, item.String()) + } + appendItemPruneTraceStep(p, "groupByItems", s, opt) +} + // PruneColumns implements LogicalPlan interface. // If any expression has SetVar function or Sleep function, we do not prune it. -func (p *LogicalProjection) PruneColumns(parentUsedCols []*expression.Column) error { +func (p *LogicalProjection) PruneColumns(parentUsedCols []*expression.Column, opt *logicalOptimizeOp) error { child := p.children[0] used := expression.GetUsedList(parentUsedCols, p.schema) + prunedColumns := make([]*expression.Column, 0) for i := len(used) - 1; i >= 0; i-- { if !used[i] && !exprHasSetVarOrSleep(p.Exprs[i]) { + prunedColumns = append(prunedColumns, p.schema.Columns[i]) p.schema.Columns = append(p.schema.Columns[:i], p.schema.Columns[i+1:]...) p.Exprs = append(p.Exprs[:i], p.Exprs[i+1:]...) } } + appendColumnPruneTraceStep(p, prunedColumns, opt) selfUsedCols := make([]*expression.Column, 0, len(p.Exprs)) selfUsedCols = expression.ExtractColumnsFromExpressions(selfUsedCols, p.Exprs, nil) - return child.PruneColumns(selfUsedCols) + return child.PruneColumns(selfUsedCols, opt) } // PruneColumns implements LogicalPlan interface. -func (p *LogicalSelection) PruneColumns(parentUsedCols []*expression.Column) error { +func (p *LogicalSelection) PruneColumns(parentUsedCols []*expression.Column, opt *logicalOptimizeOp) error { child := p.children[0] parentUsedCols = expression.ExtractColumnsFromExpressions(parentUsedCols, p.Conditions, nil) - return child.PruneColumns(parentUsedCols) + return child.PruneColumns(parentUsedCols, opt) } // PruneColumns implements LogicalPlan interface. -func (la *LogicalAggregation) PruneColumns(parentUsedCols []*expression.Column) error { +func (la *LogicalAggregation) PruneColumns(parentUsedCols []*expression.Column, opt *logicalOptimizeOp) error { child := la.children[0] used := expression.GetUsedList(parentUsedCols, la.Schema()) + prunedColumns := make([]*expression.Column, 0) + prunedFunctions := make([]*aggregation.AggFuncDesc, 0) + prunedGroupByItems := make([]expression.Expression, 0) allFirstRow := true allRemainFirstRow := true @@ -97,18 +170,22 @@ func (la *LogicalAggregation) PruneColumns(parentUsedCols []*expression.Column) allFirstRow = false } if !used[i] && !ExprsHasSideEffects(la.AggFuncs[i].Args) { + prunedColumns = append(prunedColumns, la.schema.Columns[i]) + prunedFunctions = append(prunedFunctions, la.AggFuncs[i]) la.schema.Columns = append(la.schema.Columns[:i], la.schema.Columns[i+1:]...) la.AggFuncs = append(la.AggFuncs[:i], la.AggFuncs[i+1:]...) } else if la.AggFuncs[i].Name != ast.AggFuncFirstRow { allRemainFirstRow = false } } + appendColumnPruneTraceStep(la, prunedColumns, opt) + appendFunctionPruneTraceStep(la, prunedFunctions, opt) var selfUsedCols []*expression.Column for _, aggrFunc := range la.AggFuncs { selfUsedCols = expression.ExtractColumnsFromExpressions(selfUsedCols, aggrFunc.Args, nil) var cols []*expression.Column - aggrFunc.OrderByItems, cols = pruneByItems(aggrFunc.OrderByItems) + aggrFunc.OrderByItems, cols = pruneByItems(la, aggrFunc.OrderByItems, opt) selfUsedCols = append(selfUsedCols, cols...) } if len(la.AggFuncs) == 0 || (!allFirstRow && allRemainFirstRow) { @@ -138,6 +215,7 @@ func (la *LogicalAggregation) PruneColumns(parentUsedCols []*expression.Column) for i := len(la.GroupByItems) - 1; i >= 0; i-- { cols := expression.ExtractColumns(la.GroupByItems[i]) if len(cols) == 0 && !exprHasSetVarOrSleep(la.GroupByItems[i]) { + prunedGroupByItems = append(prunedGroupByItems, la.GroupByItems[i]) la.GroupByItems = append(la.GroupByItems[:i], la.GroupByItems[i+1:]...) } else { selfUsedCols = append(selfUsedCols, cols...) @@ -149,7 +227,8 @@ func (la *LogicalAggregation) PruneColumns(parentUsedCols []*expression.Column) la.GroupByItems = []expression.Expression{expression.NewOne()} } } - err := child.PruneColumns(selfUsedCols) + appendGroupByItemsPruneTraceStep(la, prunedGroupByItems, opt) + err := child.PruneColumns(selfUsedCols, opt) if err != nil { return err } @@ -166,10 +245,13 @@ func (la *LogicalAggregation) PruneColumns(parentUsedCols []*expression.Column) return nil } -func pruneByItems(old []*util.ByItems) (new []*util.ByItems, parentUsedCols []*expression.Column) { +func pruneByItems(p LogicalPlan, old []*util.ByItems, opt *logicalOptimizeOp) (new []*util.ByItems, + parentUsedCols []*expression.Column) { + prunedByItems := make([]*util.ByItems, 0) new = make([]*util.ByItems, 0, len(old)) seen := make(map[string]struct{}, len(old)) for _, byItem := range old { + pruned := true hash := string(byItem.Expr.HashCode(nil)) _, hashMatch := seen[hash] seen[hash] = struct{}{} @@ -178,42 +260,48 @@ func pruneByItems(old []*util.ByItems) (new []*util.ByItems, parentUsedCols []*e // do nothing, should be filtered } else if len(cols) == 0 { if !expression.IsRuntimeConstExpr(byItem.Expr) { + pruned = false new = append(new, byItem) } } else if byItem.Expr.GetType().Tp == mysql.TypeNull { // do nothing, should be filtered } else { + pruned = false parentUsedCols = append(parentUsedCols, cols...) new = append(new, byItem) } + if pruned { + prunedByItems = append(prunedByItems, byItem) + } } + appendByItemsPruneTraceStep(p, prunedByItems, opt) return } // PruneColumns implements LogicalPlan interface. // If any expression can view as a constant in execution stage, such as correlated column, constant, // we do prune them. Note that we can't prune the expressions contain non-deterministic functions, such as rand(). -func (ls *LogicalSort) PruneColumns(parentUsedCols []*expression.Column) error { +func (ls *LogicalSort) PruneColumns(parentUsedCols []*expression.Column, opt *logicalOptimizeOp) error { child := ls.children[0] var cols []*expression.Column - ls.ByItems, cols = pruneByItems(ls.ByItems) + ls.ByItems, cols = pruneByItems(ls, ls.ByItems, opt) parentUsedCols = append(parentUsedCols, cols...) - return child.PruneColumns(parentUsedCols) + return child.PruneColumns(parentUsedCols, opt) } // PruneColumns implements LogicalPlan interface. // If any expression can view as a constant in execution stage, such as correlated column, constant, // we do prune them. Note that we can't prune the expressions contain non-deterministic functions, such as rand(). -func (lt *LogicalTopN) PruneColumns(parentUsedCols []*expression.Column) error { +func (lt *LogicalTopN) PruneColumns(parentUsedCols []*expression.Column, opt *logicalOptimizeOp) error { child := lt.children[0] var cols []*expression.Column - lt.ByItems, cols = pruneByItems(lt.ByItems) + lt.ByItems, cols = pruneByItems(lt, lt.ByItems, opt) parentUsedCols = append(parentUsedCols, cols...) - return child.PruneColumns(parentUsedCols) + return child.PruneColumns(parentUsedCols, opt) } // PruneColumns implements LogicalPlan interface. -func (p *LogicalUnionAll) PruneColumns(parentUsedCols []*expression.Column) error { +func (p *LogicalUnionAll) PruneColumns(parentUsedCols []*expression.Column, opt *logicalOptimizeOp) error { used := expression.GetUsedList(parentUsedCols, p.schema) hasBeenUsed := false for i := range used { @@ -227,7 +315,7 @@ func (p *LogicalUnionAll) PruneColumns(parentUsedCols []*expression.Column) erro copy(parentUsedCols, p.schema.Columns) } for _, child := range p.Children() { - err := child.PruneColumns(parentUsedCols) + err := child.PruneColumns(parentUsedCols, opt) if err != nil { return err } @@ -263,17 +351,17 @@ func (p *LogicalUnionAll) PruneColumns(parentUsedCols []*expression.Column) erro } // PruneColumns implements LogicalPlan interface. -func (p *LogicalUnionScan) PruneColumns(parentUsedCols []*expression.Column) error { +func (p *LogicalUnionScan) PruneColumns(parentUsedCols []*expression.Column, opt *logicalOptimizeOp) error { for i := 0; i < p.handleCols.NumCols(); i++ { parentUsedCols = append(parentUsedCols, p.handleCols.GetCol(i)) } condCols := expression.ExtractColumnsFromExpressions(nil, p.conditions, nil) parentUsedCols = append(parentUsedCols, condCols...) - return p.children[0].PruneColumns(parentUsedCols) + return p.children[0].PruneColumns(parentUsedCols, opt) } // PruneColumns implements LogicalPlan interface. -func (ds *DataSource) PruneColumns(parentUsedCols []*expression.Column) error { +func (ds *DataSource) PruneColumns(parentUsedCols []*expression.Column, opt *logicalOptimizeOp) error { used := expression.GetUsedList(parentUsedCols, ds.schema) exprCols := expression.ExtractColumnsFromExpressions(nil, ds.allConds, nil) @@ -315,7 +403,7 @@ func (ds *DataSource) PruneColumns(parentUsedCols []*expression.Column) error { } // PruneColumns implements LogicalPlan interface. -func (p *LogicalMemTable) PruneColumns(parentUsedCols []*expression.Column) error { +func (p *LogicalMemTable) PruneColumns(parentUsedCols []*expression.Column, opt *logicalOptimizeOp) error { switch p.TableInfo.Name.O { case infoschema.TableStatementsSummary, infoschema.TableStatementsSummaryHistory, @@ -343,7 +431,7 @@ func (p *LogicalMemTable) PruneColumns(parentUsedCols []*expression.Column) erro } // PruneColumns implements LogicalPlan interface. -func (p *LogicalTableDual) PruneColumns(parentUsedCols []*expression.Column) error { +func (p *LogicalTableDual) PruneColumns(parentUsedCols []*expression.Column, opt *logicalOptimizeOp) error { used := expression.GetUsedList(parentUsedCols, p.Schema()) for i := len(used) - 1; i >= 0; i-- { @@ -384,16 +472,16 @@ func (p *LogicalJoin) mergeSchema() { } // PruneColumns implements LogicalPlan interface. -func (p *LogicalJoin) PruneColumns(parentUsedCols []*expression.Column) error { +func (p *LogicalJoin) PruneColumns(parentUsedCols []*expression.Column, opt *logicalOptimizeOp) error { leftCols, rightCols := p.extractUsedCols(parentUsedCols) - err := p.children[0].PruneColumns(leftCols) + err := p.children[0].PruneColumns(leftCols, opt) if err != nil { return err } addConstOneForEmptyProjection(p.children[0]) - err = p.children[1].PruneColumns(rightCols) + err = p.children[1].PruneColumns(rightCols, opt) if err != nil { return err } @@ -409,10 +497,10 @@ func (p *LogicalJoin) PruneColumns(parentUsedCols []*expression.Column) error { } // PruneColumns implements LogicalPlan interface. -func (la *LogicalApply) PruneColumns(parentUsedCols []*expression.Column) error { +func (la *LogicalApply) PruneColumns(parentUsedCols []*expression.Column, opt *logicalOptimizeOp) error { leftCols, rightCols := la.extractUsedCols(parentUsedCols) - err := la.children[1].PruneColumns(rightCols) + err := la.children[1].PruneColumns(rightCols, opt) if err != nil { return err } @@ -423,7 +511,7 @@ func (la *LogicalApply) PruneColumns(parentUsedCols []*expression.Column) error leftCols = append(leftCols, &col.Column) } - err = la.children[0].PruneColumns(leftCols) + err = la.children[0].PruneColumns(leftCols, opt) if err != nil { return err } @@ -434,9 +522,9 @@ func (la *LogicalApply) PruneColumns(parentUsedCols []*expression.Column) error } // PruneColumns implements LogicalPlan interface. -func (p *LogicalLock) PruneColumns(parentUsedCols []*expression.Column) error { +func (p *LogicalLock) PruneColumns(parentUsedCols []*expression.Column, opt *logicalOptimizeOp) error { if !IsSelectForUpdateLockType(p.Lock.LockType) { - return p.baseLogicalPlan.PruneColumns(parentUsedCols) + return p.baseLogicalPlan.PruneColumns(parentUsedCols, opt) } if len(p.partitionedTable) > 0 { @@ -451,11 +539,11 @@ func (p *LogicalLock) PruneColumns(parentUsedCols []*expression.Column) error { } } } - return p.children[0].PruneColumns(parentUsedCols) + return p.children[0].PruneColumns(parentUsedCols, opt) } // PruneColumns implements LogicalPlan interface. -func (p *LogicalWindow) PruneColumns(parentUsedCols []*expression.Column) error { +func (p *LogicalWindow) PruneColumns(parentUsedCols []*expression.Column, opt *logicalOptimizeOp) error { windowColumns := p.GetWindowResultColumns() cnt := 0 for _, col := range parentUsedCols { @@ -473,7 +561,7 @@ func (p *LogicalWindow) PruneColumns(parentUsedCols []*expression.Column) error } parentUsedCols = parentUsedCols[:cnt] parentUsedCols = p.extractUsedCols(parentUsedCols) - err := p.children[0].PruneColumns(parentUsedCols) + err := p.children[0].PruneColumns(parentUsedCols, opt) if err != nil { return err } @@ -499,14 +587,14 @@ func (p *LogicalWindow) extractUsedCols(parentUsedCols []*expression.Column) []* } // PruneColumns implements LogicalPlan interface. -func (p *LogicalLimit) PruneColumns(parentUsedCols []*expression.Column) error { +func (p *LogicalLimit) PruneColumns(parentUsedCols []*expression.Column, opt *logicalOptimizeOp) error { if len(parentUsedCols) == 0 { // happens when LIMIT appears in UPDATE. return nil } savedUsedCols := make([]*expression.Column, len(parentUsedCols)) copy(savedUsedCols, parentUsedCols) - if err := p.children[0].PruneColumns(parentUsedCols); err != nil { + if err := p.children[0].PruneColumns(parentUsedCols, opt); err != nil { return err } p.schema = nil diff --git a/planner/core/rule_max_min_eliminate.go b/planner/core/rule_max_min_eliminate.go index 20d6fdba2857f..a2af6c0c21a4d 100644 --- a/planner/core/rule_max_min_eliminate.go +++ b/planner/core/rule_max_min_eliminate.go @@ -154,7 +154,7 @@ func (a *maxMinEliminator) splitAggFuncAndCheckIndices(agg *LogicalAggregation, newAgg := LogicalAggregation{AggFuncs: []*aggregation.AggFuncDesc{f}}.Init(agg.ctx, agg.blockOffset) newAgg.SetChildren(a.cloneSubPlans(agg.children[0])) newAgg.schema = expression.NewSchema(agg.schema.Columns[i]) - if err := newAgg.PruneColumns([]*expression.Column{newAgg.schema.Columns[0]}); err != nil { + if err := newAgg.PruneColumns([]*expression.Column{newAgg.schema.Columns[0]}, opt); err != nil { return nil, false } aggs = append(aggs, newAgg) From 97e307285ecddb9ef618ddd94cb9f2bbdc65542a Mon Sep 17 00:00:00 2001 From: yisaer Date: Tue, 28 Dec 2021 13:45:49 +0800 Subject: [PATCH 2/7] add testcase Signed-off-by: yisaer --- planner/core/logical_plan_trace_test.go | 14 +++ planner/core/rule_column_pruning.go | 147 +++++++++++++----------- planner/core/util.go | 5 +- 3 files changed, 97 insertions(+), 69 deletions(-) diff --git a/planner/core/logical_plan_trace_test.go b/planner/core/logical_plan_trace_test.go index a315e9a23ea3f..4f2ddc59b92cb 100644 --- a/planner/core/logical_plan_trace_test.go +++ b/planner/core/logical_plan_trace_test.go @@ -86,6 +86,20 @@ func (s *testPlanSuite) TestSingleRuleTraceStep(c *C) { assertRuleName string assertRuleSteps []assertTraceStep }{ + { + + }, + { + sql: "select a from t where b > 5;", + flags: []uint64{flagPrunColumns}, + assertRuleName: "column_prune", + assertRuleSteps: []assertTraceStep{ + { + assertReason: "", + assertAction: "DataSource_1's columns[test.t.i_date,test.t.h,test.t.g,test.t.f,test.t.e_str,test.t.d_str,test.t.c_str,test.t.e,test.t.d,test.t.c] have been pruned", + }, + }, + }, { sql: "select * from t as t1 join t as t2 on t1.a = t2.a where t1.a < 1;", flags: []uint64{flagPredicatePushDown, flagBuildKeyInfo, flagPrunColumns}, diff --git a/planner/core/rule_column_pruning.go b/planner/core/rule_column_pruning.go index 904e2e245a762..eb9a2e3be034d 100644 --- a/planner/core/rule_column_pruning.go +++ b/planner/core/rule_column_pruning.go @@ -63,71 +63,6 @@ func exprHasSetVarOrSleep(expr expression.Expression) bool { return false } -func appendColumnPruneTraceStep(p LogicalPlan, prunedColumns []*expression.Column, opt *logicalOptimizeOp) { - if len(prunedColumns) < 1 { - return - } - s := make([]string, 0) - for _, item := range prunedColumns { - s = append(s, item.String()) - } - appendItemPruneTraceStep(p, "columns", s, opt) -} - -func appendFunctionPruneTraceStep(p LogicalPlan, prunedFunctions []*aggregation.AggFuncDesc, opt *logicalOptimizeOp) { - if len(prunedFunctions) < 1 { - return - } - s := make([]string, 0) - for _, item := range prunedFunctions { - s = append(s, item.String()) - } - appendItemPruneTraceStep(p, "aggregation functions", s, opt) -} - -func appendByItemsPruneTraceStep(p LogicalPlan, prunedByItems []*util.ByItems, opt *logicalOptimizeOp) { - if len(prunedByItems) < 1 { - return - } - s := make([]string, 0) - for _, item := range prunedByItems { - s = append(s, item.String()) - } - appendItemPruneTraceStep(p, "byItems", s, opt) -} - -func appendItemPruneTraceStep(p LogicalPlan, itemType string, prunedObjects []string, opt *logicalOptimizeOp) { - if len(prunedObjects) < 1 { - return - } - action := func() string { - buffer := bytes.NewBufferString(fmt.Sprintf("%v_%v's %v[", p.TP(), p.ID(), itemType)) - for i, item := range prunedObjects { - if i > 0 { - buffer.WriteString(",") - } - buffer.WriteString(item) - } - buffer.WriteString("] have been pruned") - return buffer.String() - } - reason := func() string { - return "" - } - opt.appendStepToCurrent(p.ID(), p.TP(), reason, action) -} - -func appendGroupByItemsPruneTraceStep(p LogicalPlan, prunedGroupByItems []expression.Expression, opt *logicalOptimizeOp) { - if len(prunedGroupByItems) < 1 { - return - } - s := make([]string, 0) - for _, item := range prunedGroupByItems { - s = append(s, item.String()) - } - appendItemPruneTraceStep(p, "groupByItems", s, opt) -} - // PruneColumns implements LogicalPlan interface. // If any expression has SetVar function or Sleep function, we do not prune it. func (p *LogicalProjection) PruneColumns(parentUsedCols []*expression.Column, opt *logicalOptimizeOp) error { @@ -321,14 +256,17 @@ func (p *LogicalUnionAll) PruneColumns(parentUsedCols []*expression.Column, opt } } + prunedColumns := make([]*expression.Column, 0) if hasBeenUsed { // keep the schema of LogicalUnionAll same as its children's used := expression.GetUsedList(p.children[0].Schema().Columns, p.schema) for i := len(used) - 1; i >= 0; i-- { if !used[i] { + prunedColumns = append(prunedColumns, p.schema.Columns[i]) p.schema.Columns = append(p.schema.Columns[:i], p.schema.Columns[i+1:]...) } } + appendColumnPruneTraceStep(p, prunedColumns, opt) // It's possible that the child operator adds extra columns to the schema. // Currently, (*LogicalAggregation).PruneColumns() might do this. // But we don't need such columns, so we add an extra Projection to prune this column when this happened. @@ -366,15 +304,18 @@ func (ds *DataSource) PruneColumns(parentUsedCols []*expression.Column, opt *log exprCols := expression.ExtractColumnsFromExpressions(nil, ds.allConds, nil) exprUsed := expression.GetUsedList(exprCols, ds.schema) + prunedColumns := make([]*expression.Column, 0) originSchemaColumns := ds.schema.Columns originColumns := ds.Columns for i := len(used) - 1; i >= 0; i-- { if !used[i] && !exprUsed[i] { + prunedColumns = append(prunedColumns, ds.schema.Columns[i]) ds.schema.Columns = append(ds.schema.Columns[:i], ds.schema.Columns[i+1:]...) ds.Columns = append(ds.Columns[:i], ds.Columns[i+1:]...) } } + appendColumnPruneTraceStep(ds, prunedColumns, opt) // For SQL like `select 1 from t`, tikv's response will be empty if no column is in schema. // So we'll force to push one if schema doesn't have any column. if ds.schema.Len() == 0 { @@ -419,26 +360,31 @@ func (p *LogicalMemTable) PruneColumns(parentUsedCols []*expression.Column, opt default: return nil } + prunedColumns := make([]*expression.Column, 0) used := expression.GetUsedList(parentUsedCols, p.schema) for i := len(used) - 1; i >= 0; i-- { if !used[i] && p.schema.Len() > 1 { + prunedColumns = append(prunedColumns, p.schema.Columns[i]) p.schema.Columns = append(p.schema.Columns[:i], p.schema.Columns[i+1:]...) p.names = append(p.names[:i], p.names[i+1:]...) p.Columns = append(p.Columns[:i], p.Columns[i+1:]...) } } + appendColumnPruneTraceStep(p, prunedColumns, opt) return nil } // PruneColumns implements LogicalPlan interface. func (p *LogicalTableDual) PruneColumns(parentUsedCols []*expression.Column, opt *logicalOptimizeOp) error { used := expression.GetUsedList(parentUsedCols, p.Schema()) - + prunedColumns := make([]*expression.Column, 0) for i := len(used) - 1; i >= 0; i-- { if !used[i] { + prunedColumns = append(prunedColumns, p.schema.Columns[i]) p.schema.Columns = append(p.schema.Columns[:i], p.schema.Columns[i+1:]...) } } + appendColumnPruneTraceStep(p, prunedColumns, opt) return nil } @@ -492,7 +438,7 @@ func (p *LogicalJoin) PruneColumns(parentUsedCols []*expression.Column, opt *log joinCol := p.schema.Columns[len(p.schema.Columns)-1] parentUsedCols = append(parentUsedCols, joinCol) } - p.inlineProjection(parentUsedCols) + p.inlineProjection(parentUsedCols, opt) return nil } @@ -598,7 +544,7 @@ func (p *LogicalLimit) PruneColumns(parentUsedCols []*expression.Column, opt *lo return err } p.schema = nil - p.inlineProjection(savedUsedCols) + p.inlineProjection(savedUsedCols, opt) return nil } @@ -627,3 +573,68 @@ func addConstOneForEmptyProjection(p LogicalPlan) { RetType: constOne.GetType(), }) } + +func appendColumnPruneTraceStep(p LogicalPlan, prunedColumns []*expression.Column, opt *logicalOptimizeOp) { + if len(prunedColumns) < 1 { + return + } + s := make([]string, 0) + for _, item := range prunedColumns { + s = append(s, item.String()) + } + appendItemPruneTraceStep(p, "columns", s, opt) +} + +func appendFunctionPruneTraceStep(p LogicalPlan, prunedFunctions []*aggregation.AggFuncDesc, opt *logicalOptimizeOp) { + if len(prunedFunctions) < 1 { + return + } + s := make([]string, 0) + for _, item := range prunedFunctions { + s = append(s, item.String()) + } + appendItemPruneTraceStep(p, "aggregation functions", s, opt) +} + +func appendByItemsPruneTraceStep(p LogicalPlan, prunedByItems []*util.ByItems, opt *logicalOptimizeOp) { + if len(prunedByItems) < 1 { + return + } + s := make([]string, 0) + for _, item := range prunedByItems { + s = append(s, item.String()) + } + appendItemPruneTraceStep(p, "byItems", s, opt) +} + +func appendItemPruneTraceStep(p LogicalPlan, itemType string, prunedObjects []string, opt *logicalOptimizeOp) { + if len(prunedObjects) < 1 { + return + } + action := func() string { + buffer := bytes.NewBufferString(fmt.Sprintf("%v_%v's %v[", p.TP(), p.ID(), itemType)) + for i, item := range prunedObjects { + if i > 0 { + buffer.WriteString(",") + } + buffer.WriteString(item) + } + buffer.WriteString("] have been pruned") + return buffer.String() + } + reason := func() string { + return "" + } + opt.appendStepToCurrent(p.ID(), p.TP(), reason, action) +} + +func appendGroupByItemsPruneTraceStep(p LogicalPlan, prunedGroupByItems []expression.Expression, opt *logicalOptimizeOp) { + if len(prunedGroupByItems) < 1 { + return + } + s := make([]string, 0) + for _, item := range prunedGroupByItems { + s = append(s, item.String()) + } + appendItemPruneTraceStep(p, "groupByItems", s, opt) +} diff --git a/planner/core/util.go b/planner/core/util.go index 480c4bdfabd0c..9d0fe9d0f3bfc 100644 --- a/planner/core/util.go +++ b/planner/core/util.go @@ -128,13 +128,16 @@ func (s *logicalSchemaProducer) setSchemaAndNames(schema *expression.Schema, nam } // inlineProjection prunes unneeded columns inline a executor. -func (s *logicalSchemaProducer) inlineProjection(parentUsedCols []*expression.Column) { +func (s *logicalSchemaProducer) inlineProjection(parentUsedCols []*expression.Column, opt *logicalOptimizeOp) { + prunedColumns := make([]*expression.Column, 0) used := expression.GetUsedList(parentUsedCols, s.Schema()) for i := len(used) - 1; i >= 0; i-- { if !used[i] { + prunedColumns = append(prunedColumns, s.Schema().Columns[i]) s.schema.Columns = append(s.Schema().Columns[:i], s.Schema().Columns[i+1:]...) } } + appendColumnPruneTraceStep(s.self, prunedColumns, opt) } // physicalSchemaProducer stores the schema for the physical plans who can produce schema directly. From c5791dae982fd417d16bee58d9485511b3288470 Mon Sep 17 00:00:00 2001 From: yisaer Date: Tue, 28 Dec 2021 15:01:07 +0800 Subject: [PATCH 3/7] add test Signed-off-by: yisaer --- planner/core/logical_plan_trace_test.go | 30 ++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/planner/core/logical_plan_trace_test.go b/planner/core/logical_plan_trace_test.go index 4f2ddc59b92cb..98d732f2e4d0c 100644 --- a/planner/core/logical_plan_trace_test.go +++ b/planner/core/logical_plan_trace_test.go @@ -87,7 +87,35 @@ func (s *testPlanSuite) TestSingleRuleTraceStep(c *C) { assertRuleSteps []assertTraceStep }{ { - + sql: "select count(1) from t join (select count(1) from t where false) as tmp;", + flags: []uint64{flagPrunColumns}, + assertRuleName: "column_prune", + assertRuleSteps: []assertTraceStep{ + { + assertAction: "Aggregation_8's columns[Column#25,test.t.i_date,test.t.h,test.t.g,test.t.f,test.t.e_str,test.t.d_str,test.t.c_str,test.t.e,test.t.d,test.t.c,test.t.b,test.t.a] have been pruned", + }, + { + assertAction: "Aggregation_8's aggregation functions[firstrow(Column#25),firstrow(test.t.i_date),firstrow(test.t.h),firstrow(test.t.g),firstrow(test.t.f),firstrow(test.t.e_str),firstrow(test.t.d_str),firstrow(test.t.c_str),firstrow(test.t.e),firstrow(test.t.d),firstrow(test.t.c),firstrow(test.t.b),firstrow(test.t.a)] have been pruned", + }, + { + assertAction: "DataSource_1's columns[test.t.i_date,test.t.h,test.t.g,test.t.f,test.t.e_str,test.t.d_str,test.t.c_str,test.t.e,test.t.d,test.t.c,test.t.b,test.t.a] have been pruned", + }, + { + assertAction: "Projection_6's columns[Column#25] have been pruned", + }, + { + assertAction: "Aggregation_5's columns[test.t.i_date,test.t.h,test.t.g,test.t.f,test.t.e_str,test.t.d_str,test.t.c_str,test.t.e,test.t.d,test.t.c,test.t.b,test.t.a,Column#25] have been pruned", + }, + { + assertAction: "Aggregation_5's aggregation functions[firstrow(test.t.i_date),firstrow(test.t.h),firstrow(test.t.g),firstrow(test.t.f),firstrow(test.t.e_str),firstrow(test.t.d_str),firstrow(test.t.c_str),firstrow(test.t.e),firstrow(test.t.d),firstrow(test.t.c),firstrow(test.t.b),firstrow(test.t.a),count(1)] have been pruned", + }, + { + assertAction: "TableDual_4's columns[test.t.i_date,test.t.h,test.t.g,test.t.f,test.t.e_str,test.t.d_str,test.t.c_str,test.t.e,test.t.d,test.t.c,test.t.b,test.t.a] have been pruned", + }, + { + assertAction: "Join_7's columns[Column#28,test.t.a] have been pruned", + }, + }, }, { sql: "select a from t where b > 5;", From aad83ef5bc32f45fab4c8470827c02f4bea1156a Mon Sep 17 00:00:00 2001 From: yisaer Date: Wed, 29 Dec 2021 15:33:32 +0800 Subject: [PATCH 4/7] address the comment Signed-off-by: yisaer --- planner/core/rule_column_pruning.go | 18 +++++++++--------- planner/core/util.go | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/planner/core/rule_column_pruning.go b/planner/core/rule_column_pruning.go index eb9a2e3be034d..5ef0408f62948 100644 --- a/planner/core/rule_column_pruning.go +++ b/planner/core/rule_column_pruning.go @@ -68,7 +68,7 @@ func exprHasSetVarOrSleep(expr expression.Expression) bool { func (p *LogicalProjection) PruneColumns(parentUsedCols []*expression.Column, opt *logicalOptimizeOp) error { child := p.children[0] used := expression.GetUsedList(parentUsedCols, p.schema) - prunedColumns := make([]*expression.Column, 0) + prunedColumns := make([]*expression.Column, 0, 0) for i := len(used) - 1; i >= 0; i-- { if !used[i] && !exprHasSetVarOrSleep(p.Exprs[i]) { @@ -94,9 +94,9 @@ func (p *LogicalSelection) PruneColumns(parentUsedCols []*expression.Column, opt func (la *LogicalAggregation) PruneColumns(parentUsedCols []*expression.Column, opt *logicalOptimizeOp) error { child := la.children[0] used := expression.GetUsedList(parentUsedCols, la.Schema()) - prunedColumns := make([]*expression.Column, 0) - prunedFunctions := make([]*aggregation.AggFuncDesc, 0) - prunedGroupByItems := make([]expression.Expression, 0) + prunedColumns := make([]*expression.Column, 0, 0) + prunedFunctions := make([]*aggregation.AggFuncDesc, 0, 0) + prunedGroupByItems := make([]expression.Expression, 0, 0) allFirstRow := true allRemainFirstRow := true @@ -182,7 +182,7 @@ func (la *LogicalAggregation) PruneColumns(parentUsedCols []*expression.Column, func pruneByItems(p LogicalPlan, old []*util.ByItems, opt *logicalOptimizeOp) (new []*util.ByItems, parentUsedCols []*expression.Column) { - prunedByItems := make([]*util.ByItems, 0) + prunedByItems := make([]*util.ByItems, 0, 0) new = make([]*util.ByItems, 0, len(old)) seen := make(map[string]struct{}, len(old)) for _, byItem := range old { @@ -256,7 +256,7 @@ func (p *LogicalUnionAll) PruneColumns(parentUsedCols []*expression.Column, opt } } - prunedColumns := make([]*expression.Column, 0) + prunedColumns := make([]*expression.Column, 0, 0) if hasBeenUsed { // keep the schema of LogicalUnionAll same as its children's used := expression.GetUsedList(p.children[0].Schema().Columns, p.schema) @@ -304,7 +304,7 @@ func (ds *DataSource) PruneColumns(parentUsedCols []*expression.Column, opt *log exprCols := expression.ExtractColumnsFromExpressions(nil, ds.allConds, nil) exprUsed := expression.GetUsedList(exprCols, ds.schema) - prunedColumns := make([]*expression.Column, 0) + prunedColumns := make([]*expression.Column, 0, 0) originSchemaColumns := ds.schema.Columns originColumns := ds.Columns @@ -360,7 +360,7 @@ func (p *LogicalMemTable) PruneColumns(parentUsedCols []*expression.Column, opt default: return nil } - prunedColumns := make([]*expression.Column, 0) + prunedColumns := make([]*expression.Column, 0, 0) used := expression.GetUsedList(parentUsedCols, p.schema) for i := len(used) - 1; i >= 0; i-- { if !used[i] && p.schema.Len() > 1 { @@ -377,7 +377,7 @@ func (p *LogicalMemTable) PruneColumns(parentUsedCols []*expression.Column, opt // PruneColumns implements LogicalPlan interface. func (p *LogicalTableDual) PruneColumns(parentUsedCols []*expression.Column, opt *logicalOptimizeOp) error { used := expression.GetUsedList(parentUsedCols, p.Schema()) - prunedColumns := make([]*expression.Column, 0) + prunedColumns := make([]*expression.Column, 0, 0) for i := len(used) - 1; i >= 0; i-- { if !used[i] { prunedColumns = append(prunedColumns, p.schema.Columns[i]) diff --git a/planner/core/util.go b/planner/core/util.go index 9d0fe9d0f3bfc..7c3d9921c9bed 100644 --- a/planner/core/util.go +++ b/planner/core/util.go @@ -129,7 +129,7 @@ func (s *logicalSchemaProducer) setSchemaAndNames(schema *expression.Schema, nam // inlineProjection prunes unneeded columns inline a executor. func (s *logicalSchemaProducer) inlineProjection(parentUsedCols []*expression.Column, opt *logicalOptimizeOp) { - prunedColumns := make([]*expression.Column, 0) + prunedColumns := make([]*expression.Column, 0, 0) used := expression.GetUsedList(parentUsedCols, s.Schema()) for i := len(used) - 1; i >= 0; i-- { if !used[i] { From 508824fd7f4cfaa872a9e9f27c1b7be6dbd9b410 Mon Sep 17 00:00:00 2001 From: yisaer Date: Thu, 30 Dec 2021 13:33:08 +0800 Subject: [PATCH 5/7] fix lint Signed-off-by: yisaer --- planner/core/rule_column_pruning.go | 18 +++++++++--------- planner/core/util.go | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/planner/core/rule_column_pruning.go b/planner/core/rule_column_pruning.go index 5ef0408f62948..c4f84d95078f3 100644 --- a/planner/core/rule_column_pruning.go +++ b/planner/core/rule_column_pruning.go @@ -68,7 +68,7 @@ func exprHasSetVarOrSleep(expr expression.Expression) bool { func (p *LogicalProjection) PruneColumns(parentUsedCols []*expression.Column, opt *logicalOptimizeOp) error { child := p.children[0] used := expression.GetUsedList(parentUsedCols, p.schema) - prunedColumns := make([]*expression.Column, 0, 0) + prunedColumns :=make([]*expression.Column, 0) for i := len(used) - 1; i >= 0; i-- { if !used[i] && !exprHasSetVarOrSleep(p.Exprs[i]) { @@ -94,9 +94,9 @@ func (p *LogicalSelection) PruneColumns(parentUsedCols []*expression.Column, opt func (la *LogicalAggregation) PruneColumns(parentUsedCols []*expression.Column, opt *logicalOptimizeOp) error { child := la.children[0] used := expression.GetUsedList(parentUsedCols, la.Schema()) - prunedColumns := make([]*expression.Column, 0, 0) - prunedFunctions := make([]*aggregation.AggFuncDesc, 0, 0) - prunedGroupByItems := make([]expression.Expression, 0, 0) + prunedColumns :=make([]*expression.Column, 0) + prunedFunctions := make([]*aggregation.AggFuncDesc, 0) + prunedGroupByItems := make([]expression.Expression, 0) allFirstRow := true allRemainFirstRow := true @@ -182,7 +182,7 @@ func (la *LogicalAggregation) PruneColumns(parentUsedCols []*expression.Column, func pruneByItems(p LogicalPlan, old []*util.ByItems, opt *logicalOptimizeOp) (new []*util.ByItems, parentUsedCols []*expression.Column) { - prunedByItems := make([]*util.ByItems, 0, 0) + prunedByItems := make([]*util.ByItems, 0) new = make([]*util.ByItems, 0, len(old)) seen := make(map[string]struct{}, len(old)) for _, byItem := range old { @@ -256,7 +256,7 @@ func (p *LogicalUnionAll) PruneColumns(parentUsedCols []*expression.Column, opt } } - prunedColumns := make([]*expression.Column, 0, 0) + prunedColumns :=make([]*expression.Column, 0) if hasBeenUsed { // keep the schema of LogicalUnionAll same as its children's used := expression.GetUsedList(p.children[0].Schema().Columns, p.schema) @@ -304,7 +304,7 @@ func (ds *DataSource) PruneColumns(parentUsedCols []*expression.Column, opt *log exprCols := expression.ExtractColumnsFromExpressions(nil, ds.allConds, nil) exprUsed := expression.GetUsedList(exprCols, ds.schema) - prunedColumns := make([]*expression.Column, 0, 0) + prunedColumns :=make([]*expression.Column, 0) originSchemaColumns := ds.schema.Columns originColumns := ds.Columns @@ -360,7 +360,7 @@ func (p *LogicalMemTable) PruneColumns(parentUsedCols []*expression.Column, opt default: return nil } - prunedColumns := make([]*expression.Column, 0, 0) + prunedColumns :=make([]*expression.Column, 0) used := expression.GetUsedList(parentUsedCols, p.schema) for i := len(used) - 1; i >= 0; i-- { if !used[i] && p.schema.Len() > 1 { @@ -377,7 +377,7 @@ func (p *LogicalMemTable) PruneColumns(parentUsedCols []*expression.Column, opt // PruneColumns implements LogicalPlan interface. func (p *LogicalTableDual) PruneColumns(parentUsedCols []*expression.Column, opt *logicalOptimizeOp) error { used := expression.GetUsedList(parentUsedCols, p.Schema()) - prunedColumns := make([]*expression.Column, 0, 0) + prunedColumns :=make([]*expression.Column, 0) for i := len(used) - 1; i >= 0; i-- { if !used[i] { prunedColumns = append(prunedColumns, p.schema.Columns[i]) diff --git a/planner/core/util.go b/planner/core/util.go index 7c3d9921c9bed..c45ba5445a1af 100644 --- a/planner/core/util.go +++ b/planner/core/util.go @@ -129,7 +129,7 @@ func (s *logicalSchemaProducer) setSchemaAndNames(schema *expression.Schema, nam // inlineProjection prunes unneeded columns inline a executor. func (s *logicalSchemaProducer) inlineProjection(parentUsedCols []*expression.Column, opt *logicalOptimizeOp) { - prunedColumns := make([]*expression.Column, 0, 0) + prunedColumns :=make([]*expression.Column, 0) used := expression.GetUsedList(parentUsedCols, s.Schema()) for i := len(used) - 1; i >= 0; i-- { if !used[i] { From 49388138c1e24b5e11ec994d0f07855a5e2fc2a2 Mon Sep 17 00:00:00 2001 From: yisaer Date: Thu, 30 Dec 2021 13:43:36 +0800 Subject: [PATCH 6/7] fix fmt Signed-off-by: yisaer --- planner/core/rule_column_pruning.go | 12 ++++++------ planner/core/util.go | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/planner/core/rule_column_pruning.go b/planner/core/rule_column_pruning.go index c4f84d95078f3..eb9a2e3be034d 100644 --- a/planner/core/rule_column_pruning.go +++ b/planner/core/rule_column_pruning.go @@ -68,7 +68,7 @@ func exprHasSetVarOrSleep(expr expression.Expression) bool { func (p *LogicalProjection) PruneColumns(parentUsedCols []*expression.Column, opt *logicalOptimizeOp) error { child := p.children[0] used := expression.GetUsedList(parentUsedCols, p.schema) - prunedColumns :=make([]*expression.Column, 0) + prunedColumns := make([]*expression.Column, 0) for i := len(used) - 1; i >= 0; i-- { if !used[i] && !exprHasSetVarOrSleep(p.Exprs[i]) { @@ -94,7 +94,7 @@ func (p *LogicalSelection) PruneColumns(parentUsedCols []*expression.Column, opt func (la *LogicalAggregation) PruneColumns(parentUsedCols []*expression.Column, opt *logicalOptimizeOp) error { child := la.children[0] used := expression.GetUsedList(parentUsedCols, la.Schema()) - prunedColumns :=make([]*expression.Column, 0) + prunedColumns := make([]*expression.Column, 0) prunedFunctions := make([]*aggregation.AggFuncDesc, 0) prunedGroupByItems := make([]expression.Expression, 0) @@ -256,7 +256,7 @@ func (p *LogicalUnionAll) PruneColumns(parentUsedCols []*expression.Column, opt } } - prunedColumns :=make([]*expression.Column, 0) + prunedColumns := make([]*expression.Column, 0) if hasBeenUsed { // keep the schema of LogicalUnionAll same as its children's used := expression.GetUsedList(p.children[0].Schema().Columns, p.schema) @@ -304,7 +304,7 @@ func (ds *DataSource) PruneColumns(parentUsedCols []*expression.Column, opt *log exprCols := expression.ExtractColumnsFromExpressions(nil, ds.allConds, nil) exprUsed := expression.GetUsedList(exprCols, ds.schema) - prunedColumns :=make([]*expression.Column, 0) + prunedColumns := make([]*expression.Column, 0) originSchemaColumns := ds.schema.Columns originColumns := ds.Columns @@ -360,7 +360,7 @@ func (p *LogicalMemTable) PruneColumns(parentUsedCols []*expression.Column, opt default: return nil } - prunedColumns :=make([]*expression.Column, 0) + prunedColumns := make([]*expression.Column, 0) used := expression.GetUsedList(parentUsedCols, p.schema) for i := len(used) - 1; i >= 0; i-- { if !used[i] && p.schema.Len() > 1 { @@ -377,7 +377,7 @@ func (p *LogicalMemTable) PruneColumns(parentUsedCols []*expression.Column, opt // PruneColumns implements LogicalPlan interface. func (p *LogicalTableDual) PruneColumns(parentUsedCols []*expression.Column, opt *logicalOptimizeOp) error { used := expression.GetUsedList(parentUsedCols, p.Schema()) - prunedColumns :=make([]*expression.Column, 0) + prunedColumns := make([]*expression.Column, 0) for i := len(used) - 1; i >= 0; i-- { if !used[i] { prunedColumns = append(prunedColumns, p.schema.Columns[i]) diff --git a/planner/core/util.go b/planner/core/util.go index c45ba5445a1af..9d0fe9d0f3bfc 100644 --- a/planner/core/util.go +++ b/planner/core/util.go @@ -129,7 +129,7 @@ func (s *logicalSchemaProducer) setSchemaAndNames(schema *expression.Schema, nam // inlineProjection prunes unneeded columns inline a executor. func (s *logicalSchemaProducer) inlineProjection(parentUsedCols []*expression.Column, opt *logicalOptimizeOp) { - prunedColumns :=make([]*expression.Column, 0) + prunedColumns := make([]*expression.Column, 0) used := expression.GetUsedList(parentUsedCols, s.Schema()) for i := len(used) - 1; i >= 0; i-- { if !used[i] { From 7ddb75a3c2b42ae864de2c0d3e966bdee9f3764e Mon Sep 17 00:00:00 2001 From: yisaer Date: Fri, 31 Dec 2021 15:47:18 +0800 Subject: [PATCH 7/7] fix test Signed-off-by: yisaer --- planner/core/rule_column_pruning.go | 38 ++++++++++++++--------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/planner/core/rule_column_pruning.go b/planner/core/rule_column_pruning.go index eb9a2e3be034d..17dcb87965e77 100644 --- a/planner/core/rule_column_pruning.go +++ b/planner/core/rule_column_pruning.go @@ -578,9 +578,9 @@ func appendColumnPruneTraceStep(p LogicalPlan, prunedColumns []*expression.Colum if len(prunedColumns) < 1 { return } - s := make([]string, 0) + s := make([]fmt.Stringer, 0, len(prunedColumns)) for _, item := range prunedColumns { - s = append(s, item.String()) + s = append(s, item) } appendItemPruneTraceStep(p, "columns", s, opt) } @@ -589,9 +589,9 @@ func appendFunctionPruneTraceStep(p LogicalPlan, prunedFunctions []*aggregation. if len(prunedFunctions) < 1 { return } - s := make([]string, 0) + s := make([]fmt.Stringer, 0, len(prunedFunctions)) for _, item := range prunedFunctions { - s = append(s, item.String()) + s = append(s, item) } appendItemPruneTraceStep(p, "aggregation functions", s, opt) } @@ -600,14 +600,25 @@ func appendByItemsPruneTraceStep(p LogicalPlan, prunedByItems []*util.ByItems, o if len(prunedByItems) < 1 { return } - s := make([]string, 0) + s := make([]fmt.Stringer, 0, len(prunedByItems)) for _, item := range prunedByItems { - s = append(s, item.String()) + s = append(s, item) } appendItemPruneTraceStep(p, "byItems", s, opt) } -func appendItemPruneTraceStep(p LogicalPlan, itemType string, prunedObjects []string, opt *logicalOptimizeOp) { +func appendGroupByItemsPruneTraceStep(p LogicalPlan, prunedGroupByItems []expression.Expression, opt *logicalOptimizeOp) { + if len(prunedGroupByItems) < 1 { + return + } + s := make([]fmt.Stringer, 0, len(prunedGroupByItems)) + for _, item := range prunedGroupByItems { + s = append(s, item) + } + appendItemPruneTraceStep(p, "groupByItems", s, opt) +} + +func appendItemPruneTraceStep(p LogicalPlan, itemType string, prunedObjects []fmt.Stringer, opt *logicalOptimizeOp) { if len(prunedObjects) < 1 { return } @@ -617,7 +628,7 @@ func appendItemPruneTraceStep(p LogicalPlan, itemType string, prunedObjects []st if i > 0 { buffer.WriteString(",") } - buffer.WriteString(item) + buffer.WriteString(item.String()) } buffer.WriteString("] have been pruned") return buffer.String() @@ -627,14 +638,3 @@ func appendItemPruneTraceStep(p LogicalPlan, itemType string, prunedObjects []st } opt.appendStepToCurrent(p.ID(), p.TP(), reason, action) } - -func appendGroupByItemsPruneTraceStep(p LogicalPlan, prunedGroupByItems []expression.Expression, opt *logicalOptimizeOp) { - if len(prunedGroupByItems) < 1 { - return - } - s := make([]string, 0) - for _, item := range prunedGroupByItems { - s = append(s, item.String()) - } - appendItemPruneTraceStep(p, "groupByItems", s, opt) -}