From 927f3c61ee9c36e60b9c24250ae7634ebf5b73e6 Mon Sep 17 00:00:00 2001 From: Arenatlx <314806019@qq.com> Date: Thu, 28 Mar 2024 20:09:48 +0800 Subject: [PATCH] planner: move logical optimizing trace logic out of core pkg (#52161) ref pingcap/tidb#51664 --- pkg/planner/core/BUILD.bazel | 2 +- pkg/planner/core/optimizer.go | 47 ++----------- pkg/planner/core/plan.go | 14 ++-- .../core/rule_aggregation_elimination.go | 15 ++-- .../core/rule_aggregation_push_down.go | 20 +++--- .../core/rule_aggregation_skew_rewrite.go | 9 +-- pkg/planner/core/rule_build_key_info.go | 3 +- ...an_stats.go => rule_collect_plan_stats.go} | 5 +- pkg/planner/core/rule_column_pruning.go | 56 +++++++-------- pkg/planner/core/rule_constant_propagation.go | 11 +-- pkg/planner/core/rule_decorrelate.go | 43 ++++++------ .../core/rule_derive_topn_from_window.go | 10 +-- pkg/planner/core/rule_eliminate_projection.go | 13 ++-- .../core/rule_generate_column_substitute.go | 15 ++-- pkg/planner/core/rule_join_elimination.go | 15 ++-- pkg/planner/core/rule_join_reorder.go | 13 ++-- pkg/planner/core/rule_max_min_eliminate.go | 18 ++--- pkg/planner/core/rule_partition_processor.go | 25 +++---- pkg/planner/core/rule_predicate_push_down.go | 52 +++++++------- .../core/rule_predicate_simplification.go | 7 +- pkg/planner/core/rule_push_down_sequence.go | 8 ++- .../core/rule_resolve_grouping_expand.go | 6 +- pkg/planner/core/rule_result_reorder.go | 2 +- pkg/planner/core/rule_semi_join_rewrite.go | 3 +- pkg/planner/core/rule_topn_push_down.go | 44 ++++++------ pkg/planner/core/util.go | 3 +- pkg/planner/util/BUILD.bazel | 2 + pkg/planner/util/optTracer.go | 68 +++++++++++++++++++ 28 files changed, 292 insertions(+), 237 deletions(-) rename pkg/planner/core/{plan_stats.go => rule_collect_plan_stats.go} (98%) create mode 100644 pkg/planner/util/optTracer.go diff --git a/pkg/planner/core/BUILD.bazel b/pkg/planner/core/BUILD.bazel index 82141c369d1e2..b8dae9df6d796 100644 --- a/pkg/planner/core/BUILD.bazel +++ b/pkg/planner/core/BUILD.bazel @@ -38,7 +38,6 @@ go_library( "plan_cost_detail.go", "plan_cost_ver1.go", "plan_cost_ver2.go", - "plan_stats.go", "plan_to_pb.go", "planbuilder.go", "point_get_plan.go", @@ -50,6 +49,7 @@ go_library( "rule_aggregation_push_down.go", "rule_aggregation_skew_rewrite.go", "rule_build_key_info.go", + "rule_collect_plan_stats.go", "rule_column_pruning.go", "rule_constant_propagation.go", "rule_decorrelate.go", diff --git a/pkg/planner/core/optimizer.go b/pkg/planner/core/optimizer.go index d65cb11e265bf..f776b34c110da 100644 --- a/pkg/planner/core/optimizer.go +++ b/pkg/planner/core/optimizer.go @@ -128,41 +128,6 @@ var optRuleList = []logicalOptRule{ */ var optInteractionRuleList = map[logicalOptRule]logicalOptRule{} -type logicalOptimizeOp struct { - // tracer is goring to track optimize steps during rule optimizing - tracer *tracing.LogicalOptimizeTracer -} - -func defaultLogicalOptimizeOption() *logicalOptimizeOp { - return &logicalOptimizeOp{} -} - -func (op *logicalOptimizeOp) withEnableOptimizeTracer(tracer *tracing.LogicalOptimizeTracer) *logicalOptimizeOp { - op.tracer = tracer - return op -} - -func (op *logicalOptimizeOp) appendBeforeRuleOptimize(index int, name string, before LogicalPlan) { - if op == nil || op.tracer == nil { - return - } - op.tracer.AppendRuleTracerBeforeRuleOptimize(index, name, before.BuildPlanTrace()) -} - -func (op *logicalOptimizeOp) appendStepToCurrent(id int, tp string, reason, action func() string) { - if op == nil || op.tracer == nil { - return - } - op.tracer.AppendRuleTracerStepToCurrent(id, tp, reason(), action()) -} - -func (op *logicalOptimizeOp) recordFinalLogicalPlan(final LogicalPlan) { - if op == nil || op.tracer == nil { - return - } - op.tracer.RecordFinalLogicalPlan(final.BuildPlanTrace()) -} - // logicalOptRule means a logical optimizing rule, which contains decorrelate, ppd, column pruning, etc. type logicalOptRule interface { /* Return Parameters: @@ -172,7 +137,7 @@ type logicalOptRule interface { The default value is false. It means that no interaction rule will be triggered. 3. error: If there is error during the rule optimizer, it will be thrown */ - optimize(context.Context, LogicalPlan, *logicalOptimizeOp) (LogicalPlan, bool, error) + optimize(context.Context, LogicalPlan, *plannerutil.LogicalOptimizeOp) (LogicalPlan, bool, error) name() string } @@ -1157,14 +1122,14 @@ func logicalOptimize(ctx context.Context, flag uint64, logic LogicalPlan) (Logic debugtrace.EnterContextCommon(logic.SCtx()) defer debugtrace.LeaveContextCommon(logic.SCtx()) } - opt := defaultLogicalOptimizeOption() + opt := plannerutil.DefaultLogicalOptimizeOption() vars := logic.SCtx().GetSessionVars() if vars.StmtCtx.EnableOptimizeTrace { vars.StmtCtx.OptimizeTracer = &tracing.OptimizeTracer{} tracer := &tracing.LogicalOptimizeTracer{ Steps: make([]*tracing.LogicalRuleOptimizeTracer, 0), } - opt = opt.withEnableOptimizeTracer(tracer) + opt = opt.WithEnableOptimizeTracer(tracer) defer func() { vars.StmtCtx.OptimizeTracer.Logical = tracer }() @@ -1178,7 +1143,7 @@ func logicalOptimize(ctx context.Context, flag uint64, logic LogicalPlan) (Logic if flag&(1<= 0; i-- { @@ -496,7 +496,7 @@ func (p *LogicalJoin) mergeSchema() { } // PruneColumns implements LogicalPlan interface. -func (p *LogicalJoin) PruneColumns(parentUsedCols []*expression.Column, opt *logicalOptimizeOp) (LogicalPlan, error) { +func (p *LogicalJoin) PruneColumns(parentUsedCols []*expression.Column, opt *util.LogicalOptimizeOp) (LogicalPlan, error) { leftCols, rightCols := p.extractUsedCols(parentUsedCols) var err error @@ -522,7 +522,7 @@ func (p *LogicalJoin) PruneColumns(parentUsedCols []*expression.Column, opt *log } // PruneColumns implements LogicalPlan interface. -func (la *LogicalApply) PruneColumns(parentUsedCols []*expression.Column, opt *logicalOptimizeOp) (LogicalPlan, error) { +func (la *LogicalApply) PruneColumns(parentUsedCols []*expression.Column, opt *util.LogicalOptimizeOp) (LogicalPlan, error) { leftCols, rightCols := la.extractUsedCols(parentUsedCols) allowEliminateApply := fixcontrol.GetBoolWithDefault(la.SCtx().GetSessionVars().GetOptimizerFixControlMap(), fixcontrol.Fix45822, true) var err error @@ -556,7 +556,7 @@ func (la *LogicalApply) PruneColumns(parentUsedCols []*expression.Column, opt *l } // PruneColumns implements LogicalPlan interface. -func (p *LogicalLock) PruneColumns(parentUsedCols []*expression.Column, opt *logicalOptimizeOp) (LogicalPlan, error) { +func (p *LogicalLock) PruneColumns(parentUsedCols []*expression.Column, opt *util.LogicalOptimizeOp) (LogicalPlan, error) { var err error if !IsSelectForUpdateLockType(p.Lock.LockType) { // when use .baseLogicalPlan to call the PruneColumns, it means current plan itself has @@ -589,7 +589,7 @@ func (p *LogicalLock) PruneColumns(parentUsedCols []*expression.Column, opt *log } // PruneColumns implements LogicalPlan interface. -func (p *LogicalWindow) PruneColumns(parentUsedCols []*expression.Column, opt *logicalOptimizeOp) (LogicalPlan, error) { +func (p *LogicalWindow) PruneColumns(parentUsedCols []*expression.Column, opt *util.LogicalOptimizeOp) (LogicalPlan, error) { windowColumns := p.GetWindowResultColumns() cnt := 0 for _, col := range parentUsedCols { @@ -634,7 +634,7 @@ func (p *LogicalWindow) extractUsedCols(parentUsedCols []*expression.Column) []* } // PruneColumns implements LogicalPlan interface. -func (p *LogicalLimit) PruneColumns(parentUsedCols []*expression.Column, opt *logicalOptimizeOp) (LogicalPlan, error) { +func (p *LogicalLimit) PruneColumns(parentUsedCols []*expression.Column, opt *util.LogicalOptimizeOp) (LogicalPlan, error) { if len(parentUsedCols) == 0 { // happens when LIMIT appears in UPDATE. return p, nil } @@ -676,7 +676,7 @@ func addConstOneForEmptyProjection(p LogicalPlan) { }) } -func appendColumnPruneTraceStep(p LogicalPlan, prunedColumns []*expression.Column, opt *logicalOptimizeOp) { +func appendColumnPruneTraceStep(p LogicalPlan, prunedColumns []*expression.Column, opt *util.LogicalOptimizeOp) { if len(prunedColumns) < 1 { return } @@ -687,7 +687,7 @@ func appendColumnPruneTraceStep(p LogicalPlan, prunedColumns []*expression.Colum appendItemPruneTraceStep(p, "columns", s, opt) } -func appendFunctionPruneTraceStep(p LogicalPlan, prunedFunctions []*aggregation.AggFuncDesc, opt *logicalOptimizeOp) { +func appendFunctionPruneTraceStep(p LogicalPlan, prunedFunctions []*aggregation.AggFuncDesc, opt *util.LogicalOptimizeOp) { if len(prunedFunctions) < 1 { return } @@ -698,7 +698,7 @@ func appendFunctionPruneTraceStep(p LogicalPlan, prunedFunctions []*aggregation. appendItemPruneTraceStep(p, "aggregation functions", s, opt) } -func appendByItemsPruneTraceStep(p LogicalPlan, prunedByItems []*util.ByItems, opt *logicalOptimizeOp) { +func appendByItemsPruneTraceStep(p LogicalPlan, prunedByItems []*util.ByItems, opt *util.LogicalOptimizeOp) { if len(prunedByItems) < 1 { return } @@ -709,7 +709,7 @@ func appendByItemsPruneTraceStep(p LogicalPlan, prunedByItems []*util.ByItems, o appendItemPruneTraceStep(p, "byItems", s, opt) } -func appendGroupByItemsPruneTraceStep(p LogicalPlan, prunedGroupByItems []expression.Expression, opt *logicalOptimizeOp) { +func appendGroupByItemsPruneTraceStep(p LogicalPlan, prunedGroupByItems []expression.Expression, opt *util.LogicalOptimizeOp) { if len(prunedGroupByItems) < 1 { return } @@ -720,7 +720,7 @@ func appendGroupByItemsPruneTraceStep(p LogicalPlan, prunedGroupByItems []expres appendItemPruneTraceStep(p, "groupByItems", s, opt) } -func appendItemPruneTraceStep(p LogicalPlan, itemType string, prunedObjects []fmt.Stringer, opt *logicalOptimizeOp) { +func appendItemPruneTraceStep(p LogicalPlan, itemType string, prunedObjects []fmt.Stringer, opt *util.LogicalOptimizeOp) { if len(prunedObjects) < 1 { return } @@ -738,7 +738,7 @@ func appendItemPruneTraceStep(p LogicalPlan, itemType string, prunedObjects []fm reason := func() string { return "" } - opt.appendStepToCurrent(p.ID(), p.TP(), reason, action) + opt.AppendStepToCurrent(p.ID(), p.TP(), reason, action) } func preferKeyColumnFromTable(dataSource *DataSource, originColumns []*expression.Column, @@ -768,12 +768,12 @@ func preferKeyColumnFromTable(dataSource *DataSource, originColumns []*expressio // PruneColumns implements the interface of LogicalPlan. // LogicalCTE just do a empty function call. It's logical optimize is indivisual phase. -func (p *LogicalCTE) PruneColumns(_ []*expression.Column, _ *logicalOptimizeOp) (LogicalPlan, error) { +func (p *LogicalCTE) PruneColumns(_ []*expression.Column, _ *util.LogicalOptimizeOp) (LogicalPlan, error) { return p, nil } // PruneColumns implements the interface of LogicalPlan. -func (p *LogicalSequence) PruneColumns(parentUsedCols []*expression.Column, opt *logicalOptimizeOp) (LogicalPlan, error) { +func (p *LogicalSequence) PruneColumns(parentUsedCols []*expression.Column, opt *util.LogicalOptimizeOp) (LogicalPlan, error) { var err error p.children[len(p.children)-1], err = p.children[len(p.children)-1].PruneColumns(parentUsedCols, opt) if err != nil { @@ -782,7 +782,7 @@ func (p *LogicalSequence) PruneColumns(parentUsedCols []*expression.Column, opt return p, nil } -func applyEliminateTraceStep(lp LogicalPlan, opt *logicalOptimizeOp) { +func applyEliminateTraceStep(lp LogicalPlan, opt *util.LogicalOptimizeOp) { action := func() string { buffer := bytes.NewBufferString( fmt.Sprintf("%v_%v is eliminated.", lp.TP(), lp.ID())) @@ -791,5 +791,5 @@ func applyEliminateTraceStep(lp LogicalPlan, opt *logicalOptimizeOp) { reason := func() string { return fmt.Sprintf("%v_%v can be eliminated because it hasn't been used by it's parent.", lp.TP(), lp.ID()) } - opt.appendStepToCurrent(lp.ID(), lp.TP(), reason, action) + opt.AppendStepToCurrent(lp.ID(), lp.TP(), reason, action) } diff --git a/pkg/planner/core/rule_constant_propagation.go b/pkg/planner/core/rule_constant_propagation.go index 106719600bf0b..ee53f78e218f9 100644 --- a/pkg/planner/core/rule_constant_propagation.go +++ b/pkg/planner/core/rule_constant_propagation.go @@ -19,6 +19,7 @@ import ( "github.com/pingcap/tidb/pkg/expression" "github.com/pingcap/tidb/pkg/parser/ast" + "github.com/pingcap/tidb/pkg/planner/util" ) // constantPropagationSolver can support constant propagated cross-query block. @@ -50,7 +51,7 @@ type constantPropagationSolver struct { // which is mainly implemented in the interface "constantPropagation" of LogicalPlan. // Currently only the Logical Join implements this function. (Used for the subquery in FROM List) // In the future, the Logical Apply will implements this function. (Used for the subquery in WHERE or SELECT list) -func (cp *constantPropagationSolver) optimize(_ context.Context, p LogicalPlan, opt *logicalOptimizeOp) (LogicalPlan, bool, error) { +func (cp *constantPropagationSolver) optimize(_ context.Context, p LogicalPlan, opt *util.LogicalOptimizeOp) (LogicalPlan, bool, error) { planChanged := false // constant propagation root plan newRoot := p.constantPropagation(nil, 0, opt) @@ -67,7 +68,7 @@ func (cp *constantPropagationSolver) optimize(_ context.Context, p LogicalPlan, } // execOptimize optimize constant propagation exclude root plan node -func (cp *constantPropagationSolver) execOptimize(currentPlan LogicalPlan, parentPlan LogicalPlan, currentChildIdx int, opt *logicalOptimizeOp) { +func (cp *constantPropagationSolver) execOptimize(currentPlan LogicalPlan, parentPlan LogicalPlan, currentChildIdx int, opt *util.LogicalOptimizeOp) { if parentPlan == nil { // Attention: The function 'execOptimize' could not handle the root plan, so the parent plan could not be nil. return @@ -84,7 +85,7 @@ func (*constantPropagationSolver) name() string { return "constant_propagation" } -func (*baseLogicalPlan) constantPropagation(_ LogicalPlan, _ int, _ *logicalOptimizeOp) (newRoot LogicalPlan) { +func (*baseLogicalPlan) constantPropagation(_ LogicalPlan, _ int, _ *util.LogicalOptimizeOp) (newRoot LogicalPlan) { // Only LogicalJoin can apply constant propagation // Other Logical plan do nothing return nil @@ -142,7 +143,7 @@ func (*baseLogicalPlan) constantPropagation(_ LogicalPlan, _ int, _ *logicalOpti */ // Return nil if the root of plan has not been changed // Return new root if the root of plan is changed to selection -func (logicalJoin *LogicalJoin) constantPropagation(parentPlan LogicalPlan, currentChildIdx int, opt *logicalOptimizeOp) (newRoot LogicalPlan) { +func (logicalJoin *LogicalJoin) constantPropagation(parentPlan LogicalPlan, currentChildIdx int, opt *util.LogicalOptimizeOp) (newRoot LogicalPlan) { // step1: get constant predicate from left or right according to the JoinType var getConstantPredicateFromLeft bool var getConstantPredicateFromRight bool @@ -267,7 +268,7 @@ func validCompareConstantPredicate(candidatePredicate expression.Expression) boo // If the currentPlan at the top of query plan, return new root plan (selection) // Else return nil func addCandidateSelection(currentPlan LogicalPlan, currentChildIdx int, parentPlan LogicalPlan, - candidatePredicates []expression.Expression, opt *logicalOptimizeOp) (newRoot LogicalPlan) { + candidatePredicates []expression.Expression, opt *util.LogicalOptimizeOp) (newRoot LogicalPlan) { // generate a new selection for candidatePredicates selection := LogicalSelection{Conditions: candidatePredicates}.Init(currentPlan.SCtx(), currentPlan.QueryBlockOffset()) // add selection above of p diff --git a/pkg/planner/core/rule_decorrelate.go b/pkg/planner/core/rule_decorrelate.go index d99e3669ccdff..48450912d1757 100644 --- a/pkg/planner/core/rule_decorrelate.go +++ b/pkg/planner/core/rule_decorrelate.go @@ -24,6 +24,7 @@ import ( "github.com/pingcap/tidb/pkg/expression/aggregation" "github.com/pingcap/tidb/pkg/parser/ast" "github.com/pingcap/tidb/pkg/parser/mysql" + "github.com/pingcap/tidb/pkg/planner/util" "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util/plancodec" ) @@ -193,7 +194,7 @@ func (*decorrelateSolver) aggDefaultValueMap(agg *LogicalAggregation) map[int]*e } // optimize implements logicalOptRule interface. -func (s *decorrelateSolver) optimize(ctx context.Context, p LogicalPlan, opt *logicalOptimizeOp) (LogicalPlan, bool, error) { +func (s *decorrelateSolver) optimize(ctx context.Context, p LogicalPlan, opt *util.LogicalOptimizeOp) (LogicalPlan, bool, error) { planChanged := false if apply, ok := p.(*LogicalApply); ok { outerPlan := apply.children[0] @@ -463,77 +464,77 @@ func (*decorrelateSolver) name() string { return "decorrelate" } -func appendApplySimplifiedTraceStep(p *LogicalApply, j *LogicalJoin, opt *logicalOptimizeOp) { +func appendApplySimplifiedTraceStep(p *LogicalApply, j *LogicalJoin, opt *util.LogicalOptimizeOp) { action := func() string { return fmt.Sprintf("%v_%v simplified into %v_%v", plancodec.TypeApply, p.ID(), plancodec.TypeJoin, j.ID()) } reason := func() string { return fmt.Sprintf("%v_%v hasn't any corelated column, thus the inner plan is non-correlated", p.TP(), p.ID()) } - opt.appendStepToCurrent(p.ID(), p.TP(), reason, action) + opt.AppendStepToCurrent(p.ID(), p.TP(), reason, action) } -func appendRemoveSelectionTraceStep(p LogicalPlan, s *LogicalSelection, opt *logicalOptimizeOp) { +func appendRemoveSelectionTraceStep(p LogicalPlan, s *LogicalSelection, opt *util.LogicalOptimizeOp) { action := func() string { return fmt.Sprintf("%v_%v removed from plan tree", s.TP(), s.ID()) } reason := func() string { return fmt.Sprintf("%v_%v's conditions have been pushed into %v_%v", s.TP(), s.ID(), p.TP(), p.ID()) } - opt.appendStepToCurrent(s.ID(), s.TP(), reason, action) + opt.AppendStepToCurrent(s.ID(), s.TP(), reason, action) } -func appendRemoveMaxOneRowTraceStep(m *LogicalMaxOneRow, opt *logicalOptimizeOp) { +func appendRemoveMaxOneRowTraceStep(m *LogicalMaxOneRow, opt *util.LogicalOptimizeOp) { action := func() string { return fmt.Sprintf("%v_%v removed from plan tree", m.TP(), m.ID()) } reason := func() string { return "" } - opt.appendStepToCurrent(m.ID(), m.TP(), reason, action) + opt.AppendStepToCurrent(m.ID(), m.TP(), reason, action) } -func appendRemoveLimitTraceStep(limit *LogicalLimit, opt *logicalOptimizeOp) { +func appendRemoveLimitTraceStep(limit *LogicalLimit, opt *util.LogicalOptimizeOp) { action := func() string { return fmt.Sprintf("%v_%v removed from plan tree", limit.TP(), limit.ID()) } reason := func() string { return fmt.Sprintf("%v_%v in 'exists' subquery need to remove in order to keep plan optimal", limit.TP(), limit.ID()) } - opt.appendStepToCurrent(limit.ID(), limit.TP(), reason, action) + opt.AppendStepToCurrent(limit.ID(), limit.TP(), reason, action) } -func appendRemoveProjTraceStep(p *LogicalApply, proj *LogicalProjection, opt *logicalOptimizeOp) { +func appendRemoveProjTraceStep(p *LogicalApply, proj *LogicalProjection, opt *util.LogicalOptimizeOp) { action := func() string { return fmt.Sprintf("%v_%v removed from plan tree", proj.TP(), proj.ID()) } reason := func() string { return fmt.Sprintf("%v_%v's columns all substituted into %v_%v", proj.TP(), proj.ID(), p.TP(), p.ID()) } - opt.appendStepToCurrent(proj.ID(), proj.TP(), reason, action) + opt.AppendStepToCurrent(proj.ID(), proj.TP(), reason, action) } -func appendMoveProjTraceStep(p *LogicalApply, np LogicalPlan, proj *LogicalProjection, opt *logicalOptimizeOp) { +func appendMoveProjTraceStep(p *LogicalApply, np LogicalPlan, proj *LogicalProjection, opt *util.LogicalOptimizeOp) { action := func() string { return fmt.Sprintf("%v_%v is moved as %v_%v's parent", proj.TP(), proj.ID(), np.TP(), np.ID()) } reason := func() string { return fmt.Sprintf("%v_%v's join type is %v, not semi join", p.TP(), p.ID(), p.JoinType.String()) } - opt.appendStepToCurrent(proj.ID(), proj.TP(), reason, action) + opt.AppendStepToCurrent(proj.ID(), proj.TP(), reason, action) } -func appendRemoveSortTraceStep(sort *LogicalSort, opt *logicalOptimizeOp) { +func appendRemoveSortTraceStep(sort *LogicalSort, opt *util.LogicalOptimizeOp) { action := func() string { return fmt.Sprintf("%v_%v removed from plan tree", sort.TP(), sort.ID()) } reason := func() string { return "" } - opt.appendStepToCurrent(sort.ID(), sort.TP(), reason, action) + opt.AppendStepToCurrent(sort.ID(), sort.TP(), reason, action) } -func appendPullUpAggTraceStep(p *LogicalApply, np LogicalPlan, agg *LogicalAggregation, opt *logicalOptimizeOp) { +func appendPullUpAggTraceStep(p *LogicalApply, np LogicalPlan, agg *LogicalAggregation, opt *util.LogicalOptimizeOp) { action := func() string { return fmt.Sprintf("%v_%v pulled up as %v_%v's parent, and %v_%v's join type becomes %v", agg.TP(), agg.ID(), np.TP(), np.ID(), p.TP(), p.ID(), p.JoinType.String()) @@ -542,22 +543,22 @@ func appendPullUpAggTraceStep(p *LogicalApply, np LogicalPlan, agg *LogicalAggre return fmt.Sprintf("%v_%v's functions haven't any group by items and %v_%v's join type isn't %v or %v, and hasn't any conditions", agg.TP(), agg.ID(), p.TP(), p.ID(), InnerJoin.String(), LeftOuterJoin.String()) } - opt.appendStepToCurrent(agg.ID(), agg.TP(), reason, action) + opt.AppendStepToCurrent(agg.ID(), agg.TP(), reason, action) } -func appendAddProjTraceStep(p *LogicalApply, proj *LogicalProjection, opt *logicalOptimizeOp) { +func appendAddProjTraceStep(p *LogicalApply, proj *LogicalProjection, opt *util.LogicalOptimizeOp) { action := func() string { return fmt.Sprintf("%v_%v is added as %v_%v's parent", proj.TP(), proj.ID(), p.TP(), p.ID()) } reason := func() string { return "" } - opt.appendStepToCurrent(proj.ID(), proj.TP(), reason, action) + opt.AppendStepToCurrent(proj.ID(), proj.TP(), reason, action) } func appendModifyAggTraceStep(outerPlan LogicalPlan, p *LogicalApply, agg *LogicalAggregation, sel *LogicalSelection, appendedGroupByCols *expression.Schema, appendedAggFuncs []*aggregation.AggFuncDesc, - eqCondWithCorCol []*expression.ScalarFunction, opt *logicalOptimizeOp) { + eqCondWithCorCol []*expression.ScalarFunction, opt *util.LogicalOptimizeOp) { action := func() string { buffer := bytes.NewBufferString(fmt.Sprintf("%v_%v's groupby items added [", agg.TP(), agg.ID())) for i, col := range appendedGroupByCols.Columns { @@ -595,5 +596,5 @@ func appendModifyAggTraceStep(outerPlan LogicalPlan, p *LogicalApply, agg *Logic outerPlan.TP(), outerPlan.ID(), p.TP(), p.ID()) return buffer.String() } - opt.appendStepToCurrent(agg.ID(), agg.TP(), reason, action) + opt.AppendStepToCurrent(agg.ID(), agg.TP(), reason, action) } diff --git a/pkg/planner/core/rule_derive_topn_from_window.go b/pkg/planner/core/rule_derive_topn_from_window.go index 2633fec00039b..869de3d066c46 100644 --- a/pkg/planner/core/rule_derive_topn_from_window.go +++ b/pkg/planner/core/rule_derive_topn_from_window.go @@ -28,7 +28,7 @@ import ( type deriveTopNFromWindow struct { } -func appendDerivedTopNTrace(topN LogicalPlan, opt *logicalOptimizeOp) { +func appendDerivedTopNTrace(topN LogicalPlan, opt *util.LogicalOptimizeOp) { child := topN.Children()[0] action := func() string { return fmt.Sprintf("%v_%v top N added below %v_%v ", topN.TP(), topN.ID(), child.TP(), child.ID()) @@ -36,7 +36,7 @@ func appendDerivedTopNTrace(topN LogicalPlan, opt *logicalOptimizeOp) { reason := func() string { return fmt.Sprintf("%v filter on row number", topN.TP()) } - opt.appendStepToCurrent(topN.ID(), topN.TP(), reason, action) + opt.AppendStepToCurrent(topN.ID(), topN.TP(), reason, action) } // checkPartitionBy mainly checks if partition by of window function is a prefix of @@ -116,12 +116,12 @@ func windowIsTopN(p *LogicalSelection) (bool, uint64) { return false, 0 } -func (*deriveTopNFromWindow) optimize(_ context.Context, p LogicalPlan, opt *logicalOptimizeOp) (LogicalPlan, bool, error) { +func (*deriveTopNFromWindow) optimize(_ context.Context, p LogicalPlan, opt *util.LogicalOptimizeOp) (LogicalPlan, bool, error) { planChanged := false return p.deriveTopN(opt), planChanged, nil } -func (s *baseLogicalPlan) deriveTopN(opt *logicalOptimizeOp) LogicalPlan { +func (s *baseLogicalPlan) deriveTopN(opt *util.LogicalOptimizeOp) LogicalPlan { p := s.self if p.SCtx().GetSessionVars().AllowDeriveTopN { for i, child := range p.Children() { @@ -132,7 +132,7 @@ func (s *baseLogicalPlan) deriveTopN(opt *logicalOptimizeOp) LogicalPlan { return p } -func (s *LogicalSelection) deriveTopN(opt *logicalOptimizeOp) LogicalPlan { +func (s *LogicalSelection) deriveTopN(opt *util.LogicalOptimizeOp) LogicalPlan { p := s.self.(*LogicalSelection) windowIsTopN, limitValue := windowIsTopN(p) if windowIsTopN { diff --git a/pkg/planner/core/rule_eliminate_projection.go b/pkg/planner/core/rule_eliminate_projection.go index 55b6c5bbfe9fe..f821ac440154e 100644 --- a/pkg/planner/core/rule_eliminate_projection.go +++ b/pkg/planner/core/rule_eliminate_projection.go @@ -23,6 +23,7 @@ import ( "github.com/pingcap/tidb/pkg/expression" "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/parser/mysql" + "github.com/pingcap/tidb/pkg/planner/util" ) // canProjectionBeEliminatedLoose checks whether a projection can be eliminated, @@ -167,14 +168,14 @@ type projectionEliminator struct { } // optimize implements the logicalOptRule interface. -func (pe *projectionEliminator) optimize(_ context.Context, lp LogicalPlan, opt *logicalOptimizeOp) (LogicalPlan, bool, error) { +func (pe *projectionEliminator) optimize(_ context.Context, lp LogicalPlan, opt *util.LogicalOptimizeOp) (LogicalPlan, bool, error) { planChanged := false root := pe.eliminate(lp, make(map[string]*expression.Column), false, opt) return root, planChanged, nil } // eliminate eliminates the redundant projection in a logical plan. -func (pe *projectionEliminator) eliminate(p LogicalPlan, replace map[string]*expression.Column, canEliminate bool, opt *logicalOptimizeOp) LogicalPlan { +func (pe *projectionEliminator) eliminate(p LogicalPlan, replace map[string]*expression.Column, canEliminate bool, opt *util.LogicalOptimizeOp) LogicalPlan { // LogicalCTE's logical optimization is independent. if _, ok := p.(*LogicalCTE); ok { return p @@ -338,7 +339,7 @@ func (*projectionEliminator) name() string { return "projection_eliminate" } -func appendDupProjEliminateTraceStep(parent, child *LogicalProjection, opt *logicalOptimizeOp) { +func appendDupProjEliminateTraceStep(parent, child *LogicalProjection, opt *util.LogicalOptimizeOp) { action := func() string { buffer := bytes.NewBufferString( fmt.Sprintf("%v_%v is eliminated, %v_%v's expressions changed into[", child.TP(), child.ID(), parent.TP(), parent.ID())) @@ -354,15 +355,15 @@ func appendDupProjEliminateTraceStep(parent, child *LogicalProjection, opt *logi reason := func() string { return fmt.Sprintf("%v_%v's child %v_%v is redundant", parent.TP(), parent.ID(), child.TP(), child.ID()) } - opt.appendStepToCurrent(child.ID(), child.TP(), reason, action) + opt.AppendStepToCurrent(child.ID(), child.TP(), reason, action) } -func appendProjEliminateTraceStep(proj *LogicalProjection, opt *logicalOptimizeOp) { +func appendProjEliminateTraceStep(proj *LogicalProjection, opt *util.LogicalOptimizeOp) { reason := func() string { return fmt.Sprintf("%v_%v's Exprs are all Columns", proj.TP(), proj.ID()) } action := func() string { return fmt.Sprintf("%v_%v is eliminated", proj.TP(), proj.ID()) } - opt.appendStepToCurrent(proj.ID(), proj.TP(), reason, action) + opt.AppendStepToCurrent(proj.ID(), proj.TP(), reason, action) } diff --git a/pkg/planner/core/rule_generate_column_substitute.go b/pkg/planner/core/rule_generate_column_substitute.go index 1f30c3e39afe5..cea9a393798aa 100644 --- a/pkg/planner/core/rule_generate_column_substitute.go +++ b/pkg/planner/core/rule_generate_column_substitute.go @@ -20,6 +20,7 @@ import ( "github.com/pingcap/tidb/pkg/expression" "github.com/pingcap/tidb/pkg/parser/ast" + "github.com/pingcap/tidb/pkg/planner/util" "github.com/pingcap/tidb/pkg/types" h "github.com/pingcap/tidb/pkg/util/hint" ) @@ -37,7 +38,7 @@ type ExprColumnMap map[expression.Expression]*expression.Column // For example: select a+1 from t order by a+1, with a virtual generate column c as (a+1) and // an index on c. We need to replace a+1 with c so that we can use the index on c. // See also https://dev.mysql.com/doc/refman/8.0/en/generated-column-index-optimizations.html -func (gc *gcSubstituter) optimize(ctx context.Context, lp LogicalPlan, opt *logicalOptimizeOp) (LogicalPlan, bool, error) { +func (gc *gcSubstituter) optimize(ctx context.Context, lp LogicalPlan, opt *util.LogicalOptimizeOp) (LogicalPlan, bool, error) { planChanged := false exprToColumn := make(ExprColumnMap) collectGenerateColumn(lp, exprToColumn) @@ -84,7 +85,7 @@ func collectGenerateColumn(lp LogicalPlan, exprToColumn ExprColumnMap) { } } -func tryToSubstituteExpr(expr *expression.Expression, lp LogicalPlan, candidateExpr expression.Expression, tp types.EvalType, schema *expression.Schema, col *expression.Column, opt *logicalOptimizeOp) bool { +func tryToSubstituteExpr(expr *expression.Expression, lp LogicalPlan, candidateExpr expression.Expression, tp types.EvalType, schema *expression.Schema, col *expression.Column, opt *util.LogicalOptimizeOp) bool { changed := false if (*expr).Equal(lp.SCtx().GetExprCtx(), candidateExpr) && candidateExpr.GetType().EvalType() == tp && schema.ColumnIndex(col) != -1 { @@ -95,7 +96,7 @@ func tryToSubstituteExpr(expr *expression.Expression, lp LogicalPlan, candidateE return changed } -func appendSubstituteColumnStep(lp LogicalPlan, candidateExpr expression.Expression, col *expression.Column, opt *logicalOptimizeOp) { +func appendSubstituteColumnStep(lp LogicalPlan, candidateExpr expression.Expression, col *expression.Column, opt *util.LogicalOptimizeOp) { reason := func() string { return "" } action := func() string { buffer := bytes.NewBufferString("expression:") @@ -105,15 +106,15 @@ func appendSubstituteColumnStep(lp LogicalPlan, candidateExpr expression.Express buffer.WriteString(col.String()) return buffer.String() } - opt.appendStepToCurrent(lp.ID(), lp.TP(), reason, action) + opt.AppendStepToCurrent(lp.ID(), lp.TP(), reason, action) } // SubstituteExpression is Exported for bench -func SubstituteExpression(cond expression.Expression, lp LogicalPlan, exprToColumn ExprColumnMap, schema *expression.Schema, opt *logicalOptimizeOp) bool { +func SubstituteExpression(cond expression.Expression, lp LogicalPlan, exprToColumn ExprColumnMap, schema *expression.Schema, opt *util.LogicalOptimizeOp) bool { return substituteExpression(cond, lp, exprToColumn, schema, opt) } -func substituteExpression(cond expression.Expression, lp LogicalPlan, exprToColumn ExprColumnMap, schema *expression.Schema, opt *logicalOptimizeOp) bool { +func substituteExpression(cond expression.Expression, lp LogicalPlan, exprToColumn ExprColumnMap, schema *expression.Schema, opt *util.LogicalOptimizeOp) bool { sf, ok := cond.(*expression.ScalarFunction) if !ok { return false @@ -172,7 +173,7 @@ func substituteExpression(cond expression.Expression, lp LogicalPlan, exprToColu return changed } -func (gc *gcSubstituter) substitute(ctx context.Context, lp LogicalPlan, exprToColumn ExprColumnMap, opt *logicalOptimizeOp) LogicalPlan { +func (gc *gcSubstituter) substitute(ctx context.Context, lp LogicalPlan, exprToColumn ExprColumnMap, opt *util.LogicalOptimizeOp) LogicalPlan { var tp types.EvalType switch x := lp.(type) { case *LogicalSelection: diff --git a/pkg/planner/core/rule_join_elimination.go b/pkg/planner/core/rule_join_elimination.go index c5330a1f6b4d6..2a71714a78a48 100644 --- a/pkg/planner/core/rule_join_elimination.go +++ b/pkg/planner/core/rule_join_elimination.go @@ -21,6 +21,7 @@ import ( "github.com/pingcap/tidb/pkg/expression" "github.com/pingcap/tidb/pkg/parser/ast" + "github.com/pingcap/tidb/pkg/planner/util" "github.com/pingcap/tidb/pkg/util/set" ) @@ -34,7 +35,7 @@ type outerJoinEliminator struct { // 2. outer join elimination with duplicate agnostic aggregate functions: For example left outer join. // If the parent only use the columns from left table with 'distinct' label. The left outer join can // be eliminated. -func (o *outerJoinEliminator) tryToEliminateOuterJoin(p *LogicalJoin, aggCols []*expression.Column, parentCols []*expression.Column, opt *logicalOptimizeOp) (LogicalPlan, bool, error) { +func (o *outerJoinEliminator) tryToEliminateOuterJoin(p *LogicalJoin, aggCols []*expression.Column, parentCols []*expression.Column, opt *util.LogicalOptimizeOp) (LogicalPlan, bool, error) { var innerChildIdx int switch p.JoinType { case LeftOuterJoin: @@ -191,7 +192,7 @@ func GetDupAgnosticAggCols( return true, newAggCols } -func (o *outerJoinEliminator) doOptimize(p LogicalPlan, aggCols []*expression.Column, parentCols []*expression.Column, opt *logicalOptimizeOp) (LogicalPlan, error) { +func (o *outerJoinEliminator) doOptimize(p LogicalPlan, aggCols []*expression.Column, parentCols []*expression.Column, opt *util.LogicalOptimizeOp) (LogicalPlan, error) { // CTE's logical optimization is independent. if _, ok := p.(*LogicalCTE); ok { return p, nil @@ -245,7 +246,7 @@ func (o *outerJoinEliminator) doOptimize(p LogicalPlan, aggCols []*expression.Co return p, nil } -func (o *outerJoinEliminator) optimize(_ context.Context, p LogicalPlan, opt *logicalOptimizeOp) (LogicalPlan, bool, error) { +func (o *outerJoinEliminator) optimize(_ context.Context, p LogicalPlan, opt *util.LogicalOptimizeOp) (LogicalPlan, bool, error) { planChanged := false p, err := o.doOptimize(p, nil, nil, opt) return p, planChanged, err @@ -256,7 +257,7 @@ func (*outerJoinEliminator) name() string { } func appendOuterJoinEliminateTraceStep(join *LogicalJoin, outerPlan LogicalPlan, parentCols []*expression.Column, - innerJoinKeys *expression.Schema, opt *logicalOptimizeOp) { + innerJoinKeys *expression.Schema, opt *util.LogicalOptimizeOp) { reason := func() string { buffer := bytes.NewBufferString("The columns[") for i, col := range parentCols { @@ -278,10 +279,10 @@ func appendOuterJoinEliminateTraceStep(join *LogicalJoin, outerPlan LogicalPlan, action := func() string { return fmt.Sprintf("Outer %v_%v is eliminated and become %v_%v", join.TP(), join.ID(), outerPlan.TP(), outerPlan.ID()) } - opt.appendStepToCurrent(join.ID(), join.TP(), reason, action) + opt.AppendStepToCurrent(join.ID(), join.TP(), reason, action) } -func appendOuterJoinEliminateAggregationTraceStep(join *LogicalJoin, outerPlan LogicalPlan, aggCols []*expression.Column, opt *logicalOptimizeOp) { +func appendOuterJoinEliminateAggregationTraceStep(join *LogicalJoin, outerPlan LogicalPlan, aggCols []*expression.Column, opt *util.LogicalOptimizeOp) { reason := func() string { buffer := bytes.NewBufferString("The columns[") for i, col := range aggCols { @@ -296,5 +297,5 @@ func appendOuterJoinEliminateAggregationTraceStep(join *LogicalJoin, outerPlan L action := func() string { return fmt.Sprintf("Outer %v_%v is eliminated and become %v_%v", join.TP(), join.ID(), outerPlan.TP(), outerPlan.ID()) } - opt.appendStepToCurrent(join.ID(), join.TP(), reason, action) + opt.AppendStepToCurrent(join.ID(), join.TP(), reason, action) } diff --git a/pkg/planner/core/rule_join_reorder.go b/pkg/planner/core/rule_join_reorder.go index a7182d38a6b42..6acdb08a21381 100644 --- a/pkg/planner/core/rule_join_reorder.go +++ b/pkg/planner/core/rule_join_reorder.go @@ -22,6 +22,7 @@ import ( "github.com/pingcap/tidb/pkg/expression" "github.com/pingcap/tidb/pkg/parser/ast" + "github.com/pingcap/tidb/pkg/planner/util" h "github.com/pingcap/tidb/pkg/util/hint" "github.com/pingcap/tidb/pkg/util/plancodec" "github.com/pingcap/tidb/pkg/util/tracing" @@ -222,7 +223,7 @@ type joinTypeWithExtMsg struct { outerBindCondition []expression.Expression } -func (s *joinReOrderSolver) optimize(_ context.Context, p LogicalPlan, opt *logicalOptimizeOp) (LogicalPlan, bool, error) { +func (s *joinReOrderSolver) optimize(_ context.Context, p LogicalPlan, opt *util.LogicalOptimizeOp) (LogicalPlan, bool, error) { planChanged := false tracer := &joinReorderTrace{cost: map[string]float64{}, opt: opt} tracer.traceJoinReorder(p) @@ -662,7 +663,7 @@ func (*joinReOrderSolver) name() string { return "join_reorder" } -func appendJoinReorderTraceStep(tracer *joinReorderTrace, plan LogicalPlan, opt *logicalOptimizeOp) { +func appendJoinReorderTraceStep(tracer *joinReorderTrace, plan LogicalPlan, opt *util.LogicalOptimizeOp) { if len(tracer.initial) < 1 || len(tracer.final) < 1 { return } @@ -685,7 +686,7 @@ func appendJoinReorderTraceStep(tracer *joinReorderTrace, plan LogicalPlan, opt buffer.WriteString("]") return buffer.String() } - opt.appendStepToCurrent(plan.ID(), plan.TP(), reason, action) + opt.AppendStepToCurrent(plan.ID(), plan.TP(), reason, action) } func allJoinOrderToString(tt []*tracing.PlanTrace) string { @@ -772,14 +773,14 @@ func findRoots(t *tracing.PlanTrace) []*tracing.PlanTrace { } type joinReorderTrace struct { - opt *logicalOptimizeOp + opt *util.LogicalOptimizeOp initial string final string cost map[string]float64 } func (t *joinReorderTrace) traceJoinReorder(p LogicalPlan) { - if t == nil || t.opt == nil || t.opt.tracer == nil { + if t == nil || t.opt == nil || t.opt.TracerIsNil() { return } if len(t.initial) > 0 { @@ -790,7 +791,7 @@ func (t *joinReorderTrace) traceJoinReorder(p LogicalPlan) { } func (t *joinReorderTrace) appendLogicalJoinCost(join LogicalPlan, cost float64) { - if t == nil || t.opt == nil || t.opt.tracer == nil { + if t == nil || t.opt == nil || t.opt.TracerIsNil() { return } joinMapKey := allJoinOrderToString(extractJoinAndDataSource(join.BuildPlanTrace())) diff --git a/pkg/planner/core/rule_max_min_eliminate.go b/pkg/planner/core/rule_max_min_eliminate.go index e4ec913d49995..9f523d469ccb7 100644 --- a/pkg/planner/core/rule_max_min_eliminate.go +++ b/pkg/planner/core/rule_max_min_eliminate.go @@ -36,13 +36,13 @@ import ( type maxMinEliminator struct { } -func (a *maxMinEliminator) optimize(_ context.Context, p LogicalPlan, opt *logicalOptimizeOp) (LogicalPlan, bool, error) { +func (a *maxMinEliminator) optimize(_ context.Context, p LogicalPlan, opt *util.LogicalOptimizeOp) (LogicalPlan, bool, error) { planChanged := false return a.eliminateMaxMin(p, opt), planChanged, nil } // composeAggsByInnerJoin composes the scalar aggregations by cartesianJoin. -func (*maxMinEliminator) composeAggsByInnerJoin(originAgg *LogicalAggregation, aggs []*LogicalAggregation, opt *logicalOptimizeOp) (plan LogicalPlan) { +func (*maxMinEliminator) composeAggsByInnerJoin(originAgg *LogicalAggregation, aggs []*LogicalAggregation, opt *util.LogicalOptimizeOp) (plan LogicalPlan) { plan = aggs[0] sctx := plan.SCtx() joins := make([]*LogicalJoin, 0) @@ -138,7 +138,7 @@ func (a *maxMinEliminator) cloneSubPlans(plan LogicalPlan) LogicalPlan { // `select max(a) from t` + `select min(a) from t` + `select max(b) from t`. // Then we check whether `a` and `b` have indices. If any of the used column has no index, we cannot eliminate // this aggregation. -func (a *maxMinEliminator) splitAggFuncAndCheckIndices(agg *LogicalAggregation, opt *logicalOptimizeOp) (aggs []*LogicalAggregation, canEliminate bool) { +func (a *maxMinEliminator) splitAggFuncAndCheckIndices(agg *LogicalAggregation, opt *util.LogicalOptimizeOp) (aggs []*LogicalAggregation, canEliminate bool) { for _, f := range agg.AggFuncs { // We must make sure the args of max/min is a simple single column. col, ok := f.Args[0].(*expression.Column) @@ -170,7 +170,7 @@ func (a *maxMinEliminator) splitAggFuncAndCheckIndices(agg *LogicalAggregation, } // eliminateSingleMaxMin tries to convert a single max/min to Limit+Sort operators. -func (*maxMinEliminator) eliminateSingleMaxMin(agg *LogicalAggregation, opt *logicalOptimizeOp) *LogicalAggregation { +func (*maxMinEliminator) eliminateSingleMaxMin(agg *LogicalAggregation, opt *util.LogicalOptimizeOp) *LogicalAggregation { f := agg.AggFuncs[0] child := agg.Children()[0] ctx := agg.SCtx() @@ -211,7 +211,7 @@ func (*maxMinEliminator) eliminateSingleMaxMin(agg *LogicalAggregation, opt *log } // eliminateMaxMin tries to convert max/min to Limit+Sort operators. -func (a *maxMinEliminator) eliminateMaxMin(p LogicalPlan, opt *logicalOptimizeOp) LogicalPlan { +func (a *maxMinEliminator) eliminateMaxMin(p LogicalPlan, opt *util.LogicalOptimizeOp) LogicalPlan { // CTE's logical optimization is indenpent. if _, ok := p.(*LogicalCTE); ok { return p @@ -261,7 +261,7 @@ func (*maxMinEliminator) name() string { return "max_min_eliminate" } -func appendEliminateSingleMaxMinTrace(agg *LogicalAggregation, sel *LogicalSelection, sort *LogicalSort, limit *LogicalLimit, opt *logicalOptimizeOp) { +func appendEliminateSingleMaxMinTrace(agg *LogicalAggregation, sel *LogicalSelection, sort *LogicalSort, limit *LogicalLimit, opt *util.LogicalOptimizeOp) { action := func() string { buffer := bytes.NewBufferString("") if sel != nil { @@ -283,10 +283,10 @@ func appendEliminateSingleMaxMinTrace(agg *LogicalAggregation, sel *LogicalSelec } return buffer.String() } - opt.appendStepToCurrent(agg.ID(), agg.TP(), reason, action) + opt.AppendStepToCurrent(agg.ID(), agg.TP(), reason, action) } -func appendEliminateMultiMinMaxTraceStep(originAgg *LogicalAggregation, aggs []*LogicalAggregation, joins []*LogicalJoin, opt *logicalOptimizeOp) { +func appendEliminateMultiMinMaxTraceStep(originAgg *LogicalAggregation, aggs []*LogicalAggregation, joins []*LogicalJoin, opt *util.LogicalOptimizeOp) { action := func() string { buffer := bytes.NewBufferString(fmt.Sprintf("%v_%v splited into [", originAgg.TP(), originAgg.ID())) for i, agg := range aggs { @@ -316,5 +316,5 @@ func appendEliminateMultiMinMaxTraceStep(originAgg *LogicalAggregation, aggs []* buffer.WriteString("] and none of them has group by clause") return buffer.String() } - opt.appendStepToCurrent(originAgg.ID(), originAgg.TP(), reason, action) + opt.AppendStepToCurrent(originAgg.ID(), originAgg.TP(), reason, action) } diff --git a/pkg/planner/core/rule_partition_processor.go b/pkg/planner/core/rule_partition_processor.go index 79730d191c4db..850a7efdc3e96 100644 --- a/pkg/planner/core/rule_partition_processor.go +++ b/pkg/planner/core/rule_partition_processor.go @@ -29,6 +29,7 @@ import ( "github.com/pingcap/tidb/pkg/parser/ast" "github.com/pingcap/tidb/pkg/parser/model" "github.com/pingcap/tidb/pkg/parser/mysql" + "github.com/pingcap/tidb/pkg/planner/util" "github.com/pingcap/tidb/pkg/table" "github.com/pingcap/tidb/pkg/table/tables" "github.com/pingcap/tidb/pkg/types" @@ -61,13 +62,13 @@ const FullRange = -1 // partitionProcessor is here because it's easier to prune partition after predicate push down. type partitionProcessor struct{} -func (s *partitionProcessor) optimize(_ context.Context, lp LogicalPlan, opt *logicalOptimizeOp) (LogicalPlan, bool, error) { +func (s *partitionProcessor) optimize(_ context.Context, lp LogicalPlan, opt *util.LogicalOptimizeOp) (LogicalPlan, bool, error) { planChanged := false p, err := s.rewriteDataSource(lp, opt) return p, planChanged, err } -func (s *partitionProcessor) rewriteDataSource(lp LogicalPlan, opt *logicalOptimizeOp) (LogicalPlan, error) { +func (s *partitionProcessor) rewriteDataSource(lp LogicalPlan, opt *util.LogicalOptimizeOp) (LogicalPlan, error) { // Assert there will not be sel -> sel in the ast. switch p := lp.(type) { case *DataSource: @@ -501,7 +502,7 @@ func (*partitionProcessor) reconstructTableColNames(ds *DataSource) ([]*types.Fi return names, nil } -func (s *partitionProcessor) processHashOrKeyPartition(ds *DataSource, pi *model.PartitionInfo, opt *logicalOptimizeOp) (LogicalPlan, error) { +func (s *partitionProcessor) processHashOrKeyPartition(ds *DataSource, pi *model.PartitionInfo, opt *util.LogicalOptimizeOp) (LogicalPlan, error) { names, err := s.reconstructTableColNames(ds) if err != nil { return nil, err @@ -823,7 +824,7 @@ func (s *partitionProcessor) pruneListPartition(ctx PlanContext, tbl table.Table return used, nil } -func (s *partitionProcessor) prune(ds *DataSource, opt *logicalOptimizeOp) (LogicalPlan, error) { +func (s *partitionProcessor) prune(ds *DataSource, opt *util.LogicalOptimizeOp) (LogicalPlan, error) { pi := ds.tableInfo.GetPartitionInfo() if pi == nil { return ds, nil @@ -1036,7 +1037,7 @@ func (s *partitionProcessor) pruneRangePartition(ctx PlanContext, pi *model.Part return result, nil } -func (s *partitionProcessor) processRangePartition(ds *DataSource, pi *model.PartitionInfo, opt *logicalOptimizeOp) (LogicalPlan, error) { +func (s *partitionProcessor) processRangePartition(ds *DataSource, pi *model.PartitionInfo, opt *util.LogicalOptimizeOp) (LogicalPlan, error) { used, err := s.pruneRangePartition(ds.SCtx(), pi, ds.table.(table.PartitionedTable), ds.allConds, ds.TblCols, ds.names) if err != nil { return nil, err @@ -1044,7 +1045,7 @@ func (s *partitionProcessor) processRangePartition(ds *DataSource, pi *model.Par return s.makeUnionAllChildren(ds, pi, used, opt) } -func (s *partitionProcessor) processListPartition(ds *DataSource, pi *model.PartitionInfo, opt *logicalOptimizeOp) (LogicalPlan, error) { +func (s *partitionProcessor) processListPartition(ds *DataSource, pi *model.PartitionInfo, opt *util.LogicalOptimizeOp) (LogicalPlan, error) { used, err := s.pruneListPartition(ds.SCtx(), ds.table, ds.partitionNames, ds.allConds, ds.TblCols) if err != nil { return nil, err @@ -1766,7 +1767,7 @@ func (*partitionProcessor) checkHintsApplicable(ds *DataSource, partitionSet set appendWarnForUnknownPartitions(ds.SCtx(), h.HintReadFromStorage, unknownPartitions) } -func (s *partitionProcessor) makeUnionAllChildren(ds *DataSource, pi *model.PartitionInfo, or partitionRangeOR, opt *logicalOptimizeOp) (LogicalPlan, error) { +func (s *partitionProcessor) makeUnionAllChildren(ds *DataSource, pi *model.PartitionInfo, or partitionRangeOR, opt *util.LogicalOptimizeOp) (LogicalPlan, error) { children := make([]LogicalPlan, 0, len(pi.Definitions)) partitionNameSet := make(set.StringSet) usedDefinition := make(map[int64]model.PartitionDefinition) @@ -2002,8 +2003,8 @@ func (p *rangeColumnsPruner) pruneUseBinarySearch(sctx PlanContext, op string, d return start, end } -func appendMakeUnionAllChildrenTranceStep(origin *DataSource, usedMap map[int64]model.PartitionDefinition, plan LogicalPlan, children []LogicalPlan, opt *logicalOptimizeOp) { - if opt.tracer == nil { +func appendMakeUnionAllChildrenTranceStep(origin *DataSource, usedMap map[int64]model.PartitionDefinition, plan LogicalPlan, children []LogicalPlan, opt *util.LogicalOptimizeOp) { + if opt.TracerIsNil() { return } if len(children) == 0 { @@ -2055,15 +2056,15 @@ func appendMakeUnionAllChildrenTranceStep(origin *DataSource, usedMap map[int64] return buffer.String() } } - opt.appendStepToCurrent(origin.ID(), origin.TP(), reason, action) + opt.AppendStepToCurrent(origin.ID(), origin.TP(), reason, action) } -func appendNoPartitionChildTraceStep(ds *DataSource, dual LogicalPlan, opt *logicalOptimizeOp) { +func appendNoPartitionChildTraceStep(ds *DataSource, dual LogicalPlan, opt *util.LogicalOptimizeOp) { action := func() string { return fmt.Sprintf("%v_%v becomes %v_%v", ds.TP(), ds.ID(), dual.TP(), dual.ID()) } reason := func() string { return fmt.Sprintf("%v_%v doesn't have needed partition table after pruning", ds.TP(), ds.ID()) } - opt.appendStepToCurrent(dual.ID(), dual.TP(), reason, action) + opt.AppendStepToCurrent(dual.ID(), dual.TP(), reason, action) } diff --git a/pkg/planner/core/rule_predicate_push_down.go b/pkg/planner/core/rule_predicate_push_down.go index 317cabfbbd836..9561108586257 100644 --- a/pkg/planner/core/rule_predicate_push_down.go +++ b/pkg/planner/core/rule_predicate_push_down.go @@ -41,13 +41,13 @@ type exprPrefixAdder struct { lengths []int } -func (*ppdSolver) optimize(_ context.Context, lp LogicalPlan, opt *logicalOptimizeOp) (LogicalPlan, bool, error) { +func (*ppdSolver) optimize(_ context.Context, lp LogicalPlan, opt *util.LogicalOptimizeOp) (LogicalPlan, bool, error) { planChanged := false _, p := lp.PredicatePushDown(nil, opt) return p, planChanged, nil } -func addSelection(p LogicalPlan, child LogicalPlan, conditions []expression.Expression, chIdx int, opt *logicalOptimizeOp) { +func addSelection(p LogicalPlan, child LogicalPlan, conditions []expression.Expression, chIdx int, opt *util.LogicalOptimizeOp) { if len(conditions) == 0 { p.Children()[chIdx] = child return @@ -73,7 +73,7 @@ func addSelection(p LogicalPlan, child LogicalPlan, conditions []expression.Expr } // PredicatePushDown implements LogicalPlan interface. -func (p *baseLogicalPlan) PredicatePushDown(predicates []expression.Expression, opt *logicalOptimizeOp) ([]expression.Expression, LogicalPlan) { +func (p *baseLogicalPlan) PredicatePushDown(predicates []expression.Expression, opt *util.LogicalOptimizeOp) ([]expression.Expression, LogicalPlan) { if len(p.children) == 0 { return predicates, p.self } @@ -97,7 +97,7 @@ func splitSetGetVarFunc(filters []expression.Expression) ([]expression.Expressio } // PredicatePushDown implements LogicalPlan PredicatePushDown interface. -func (p *LogicalSelection) PredicatePushDown(predicates []expression.Expression, opt *logicalOptimizeOp) ([]expression.Expression, LogicalPlan) { +func (p *LogicalSelection) PredicatePushDown(predicates []expression.Expression, opt *util.LogicalOptimizeOp) ([]expression.Expression, LogicalPlan) { predicates = DeleteTrueExprs(p, predicates) p.Conditions = DeleteTrueExprs(p, p.Conditions) var child LogicalPlan @@ -123,7 +123,7 @@ func (p *LogicalSelection) PredicatePushDown(predicates []expression.Expression, } // PredicatePushDown implements LogicalPlan PredicatePushDown interface. -func (p *LogicalUnionScan) PredicatePushDown(predicates []expression.Expression, opt *logicalOptimizeOp) ([]expression.Expression, LogicalPlan) { +func (p *LogicalUnionScan) PredicatePushDown(predicates []expression.Expression, opt *util.LogicalOptimizeOp) ([]expression.Expression, LogicalPlan) { retainedPredicates, _ := p.children[0].PredicatePushDown(predicates, opt) p.conditions = make([]expression.Expression, 0, len(predicates)) p.conditions = append(p.conditions, predicates...) @@ -132,7 +132,7 @@ func (p *LogicalUnionScan) PredicatePushDown(predicates []expression.Expression, } // PredicatePushDown implements LogicalPlan PredicatePushDown interface. -func (ds *DataSource) PredicatePushDown(predicates []expression.Expression, opt *logicalOptimizeOp) ([]expression.Expression, LogicalPlan) { +func (ds *DataSource) PredicatePushDown(predicates []expression.Expression, opt *util.LogicalOptimizeOp) ([]expression.Expression, LogicalPlan) { predicates = expression.PropagateConstant(ds.SCtx().GetExprCtx(), predicates) predicates = DeleteTrueExprs(ds, predicates) // Add tidb_shard() prefix to the condtion for shard index in some scenarios @@ -145,12 +145,12 @@ func (ds *DataSource) PredicatePushDown(predicates []expression.Expression, opt } // PredicatePushDown implements LogicalPlan PredicatePushDown interface. -func (p *LogicalTableDual) PredicatePushDown(predicates []expression.Expression, _ *logicalOptimizeOp) ([]expression.Expression, LogicalPlan) { +func (p *LogicalTableDual) PredicatePushDown(predicates []expression.Expression, _ *util.LogicalOptimizeOp) ([]expression.Expression, LogicalPlan) { return predicates, p } // PredicatePushDown implements LogicalPlan PredicatePushDown interface. -func (p *LogicalJoin) PredicatePushDown(predicates []expression.Expression, opt *logicalOptimizeOp) (ret []expression.Expression, retPlan LogicalPlan) { +func (p *LogicalJoin) PredicatePushDown(predicates []expression.Expression, opt *util.LogicalOptimizeOp) (ret []expression.Expression, retPlan LogicalPlan) { simplifyOuterJoin(p, predicates) var equalCond []*expression.ScalarFunction var leftPushCond, rightPushCond, otherCond, leftCond, rightCond []expression.Expression @@ -493,7 +493,7 @@ func specialNullRejectedCase1(ctx PlanContext, schema *expression.Schema, expr e } // PredicatePushDown implements LogicalPlan PredicatePushDown interface. -func (p *LogicalExpand) PredicatePushDown(predicates []expression.Expression, opt *logicalOptimizeOp) (ret []expression.Expression, retPlan LogicalPlan) { +func (p *LogicalExpand) PredicatePushDown(predicates []expression.Expression, opt *util.LogicalOptimizeOp) (ret []expression.Expression, retPlan LogicalPlan) { // Note that, grouping column related predicates can't be pushed down, since grouping column has nullability change after Expand OP itself. // condition related with grouping column shouldn't be pushed down through it. // currently, since expand is adjacent to aggregate, any filter above aggregate wanted to be push down through expand only have two cases: @@ -505,7 +505,7 @@ func (p *LogicalExpand) PredicatePushDown(predicates []expression.Expression, op } // PredicatePushDown implements LogicalPlan PredicatePushDown interface. -func (p *LogicalProjection) PredicatePushDown(predicates []expression.Expression, opt *logicalOptimizeOp) (ret []expression.Expression, retPlan LogicalPlan) { +func (p *LogicalProjection) PredicatePushDown(predicates []expression.Expression, opt *util.LogicalOptimizeOp) (ret []expression.Expression, retPlan LogicalPlan) { canBePushed := make([]expression.Expression, 0, len(predicates)) canNotBePushed := make([]expression.Expression, 0, len(predicates)) for _, expr := range p.Exprs { @@ -528,7 +528,7 @@ func (p *LogicalProjection) PredicatePushDown(predicates []expression.Expression } // PredicatePushDown implements LogicalPlan PredicatePushDown interface. -func (p *LogicalUnionAll) PredicatePushDown(predicates []expression.Expression, opt *logicalOptimizeOp) (ret []expression.Expression, retPlan LogicalPlan) { +func (p *LogicalUnionAll) PredicatePushDown(predicates []expression.Expression, opt *util.LogicalOptimizeOp) (ret []expression.Expression, retPlan LogicalPlan) { for i, proj := range p.children { newExprs := make([]expression.Expression, 0, len(predicates)) newExprs = append(newExprs, predicates...) @@ -629,7 +629,7 @@ func (la *LogicalAggregation) pushDownDNFPredicatesForAggregation(cond expressio } // PredicatePushDown implements LogicalPlan PredicatePushDown interface. -func (la *LogicalAggregation) PredicatePushDown(predicates []expression.Expression, opt *logicalOptimizeOp) (ret []expression.Expression, retPlan LogicalPlan) { +func (la *LogicalAggregation) PredicatePushDown(predicates []expression.Expression, opt *util.LogicalOptimizeOp) (ret []expression.Expression, retPlan LogicalPlan) { var condsToPush []expression.Expression exprsOriginal := make([]expression.Expression, 0, len(la.AggFuncs)) for _, fun := range la.AggFuncs { @@ -651,14 +651,14 @@ func (la *LogicalAggregation) PredicatePushDown(predicates []expression.Expressi } // PredicatePushDown implements LogicalPlan PredicatePushDown interface. -func (p *LogicalLimit) PredicatePushDown(predicates []expression.Expression, opt *logicalOptimizeOp) ([]expression.Expression, LogicalPlan) { +func (p *LogicalLimit) PredicatePushDown(predicates []expression.Expression, opt *util.LogicalOptimizeOp) ([]expression.Expression, LogicalPlan) { // Limit forbids any condition to push down. p.baseLogicalPlan.PredicatePushDown(nil, opt) return predicates, p } // PredicatePushDown implements LogicalPlan PredicatePushDown interface. -func (p *LogicalMaxOneRow) PredicatePushDown(predicates []expression.Expression, opt *logicalOptimizeOp) ([]expression.Expression, LogicalPlan) { +func (p *LogicalMaxOneRow) PredicatePushDown(predicates []expression.Expression, opt *util.LogicalOptimizeOp) ([]expression.Expression, LogicalPlan) { // MaxOneRow forbids any condition to push down. p.baseLogicalPlan.PredicatePushDown(nil, opt) return predicates, p @@ -809,7 +809,7 @@ func (p *LogicalWindow) GetPartitionByCols() []*expression.Column { } // PredicatePushDown implements LogicalPlan PredicatePushDown interface. -func (p *LogicalWindow) PredicatePushDown(predicates []expression.Expression, opt *logicalOptimizeOp) ([]expression.Expression, LogicalPlan) { +func (p *LogicalWindow) PredicatePushDown(predicates []expression.Expression, opt *util.LogicalOptimizeOp) ([]expression.Expression, LogicalPlan) { canBePushed := make([]expression.Expression, 0, len(predicates)) canNotBePushed := make([]expression.Expression, 0, len(predicates)) partitionCols := expression.NewSchema(p.GetPartitionByCols()...) @@ -827,7 +827,7 @@ func (p *LogicalWindow) PredicatePushDown(predicates []expression.Expression, op } // PredicatePushDown implements LogicalPlan PredicatePushDown interface. -func (p *LogicalMemTable) PredicatePushDown(predicates []expression.Expression, _ *logicalOptimizeOp) ([]expression.Expression, LogicalPlan) { +func (p *LogicalMemTable) PredicatePushDown(predicates []expression.Expression, _ *util.LogicalOptimizeOp) ([]expression.Expression, LogicalPlan) { if p.Extractor != nil { predicates = p.Extractor.Extract(p.SCtx(), p.schema, p.names, predicates) } @@ -838,7 +838,7 @@ func (*ppdSolver) name() string { return "predicate_push_down" } -func appendTableDualTraceStep(replaced LogicalPlan, dual LogicalPlan, conditions []expression.Expression, opt *logicalOptimizeOp) { +func appendTableDualTraceStep(replaced LogicalPlan, dual LogicalPlan, conditions []expression.Expression, opt *util.LogicalOptimizeOp) { action := func() string { return fmt.Sprintf("%v_%v is replaced by %v_%v", replaced.TP(), replaced.ID(), dual.TP(), dual.ID()) } @@ -853,10 +853,10 @@ func appendTableDualTraceStep(replaced LogicalPlan, dual LogicalPlan, conditions buffer.WriteString("] are constant false or null") return buffer.String() } - opt.appendStepToCurrent(dual.ID(), dual.TP(), reason, action) + opt.AppendStepToCurrent(dual.ID(), dual.TP(), reason, action) } -func appendSelectionPredicatePushDownTraceStep(p *LogicalSelection, conditions []expression.Expression, opt *logicalOptimizeOp) { +func appendSelectionPredicatePushDownTraceStep(p *LogicalSelection, conditions []expression.Expression, opt *util.LogicalOptimizeOp) { action := func() string { return fmt.Sprintf("%v_%v is removed", p.TP(), p.ID()) } @@ -876,10 +876,10 @@ func appendSelectionPredicatePushDownTraceStep(p *LogicalSelection, conditions [ return buffer.String() } } - opt.appendStepToCurrent(p.ID(), p.TP(), reason, action) + opt.AppendStepToCurrent(p.ID(), p.TP(), reason, action) } -func appendDataSourcePredicatePushDownTraceStep(ds *DataSource, opt *logicalOptimizeOp) { +func appendDataSourcePredicatePushDownTraceStep(ds *DataSource, opt *util.LogicalOptimizeOp) { if len(ds.pushedDownConds) < 1 { return } @@ -897,17 +897,17 @@ func appendDataSourcePredicatePushDownTraceStep(ds *DataSource, opt *logicalOpti fmt.Fprintf(buffer, "] are pushed down across %v_%v", ds.TP(), ds.ID()) return buffer.String() } - opt.appendStepToCurrent(ds.ID(), ds.TP(), reason, action) + opt.AppendStepToCurrent(ds.ID(), ds.TP(), reason, action) } -func appendAddSelectionTraceStep(p LogicalPlan, child LogicalPlan, sel *LogicalSelection, opt *logicalOptimizeOp) { +func appendAddSelectionTraceStep(p LogicalPlan, child LogicalPlan, sel *LogicalSelection, opt *util.LogicalOptimizeOp) { reason := func() string { return "" } action := func() string { return fmt.Sprintf("add %v_%v to connect %v_%v and %v_%v", sel.TP(), sel.ID(), p.TP(), p.ID(), child.TP(), child.ID()) } - opt.appendStepToCurrent(sel.ID(), sel.TP(), reason, action) + opt.AppendStepToCurrent(sel.ID(), sel.TP(), reason, action) } // AddPrefix4ShardIndexes add expression prefix for shard index. e.g. an index is test.uk(tidb_shard(a), a). @@ -1031,7 +1031,7 @@ func (adder *exprPrefixAdder) addExprPrefix4DNFCond(condition *expression.Scalar } // PredicatePushDown implements LogicalPlan PredicatePushDown interface. -func (p *LogicalCTE) PredicatePushDown(predicates []expression.Expression, _ *logicalOptimizeOp) ([]expression.Expression, LogicalPlan) { +func (p *LogicalCTE) PredicatePushDown(predicates []expression.Expression, _ *util.LogicalOptimizeOp) ([]expression.Expression, LogicalPlan) { if p.cte.recursivePartLogicalPlan != nil { // Doesn't support recursive CTE yet. return predicates, p.self @@ -1067,7 +1067,7 @@ func (p *LogicalCTE) PredicatePushDown(predicates []expression.Expression, _ *lo // PredicatePushDown implements the LogicalPlan interface. // Currently, we only maintain the main query tree. -func (p *LogicalSequence) PredicatePushDown(predicates []expression.Expression, op *logicalOptimizeOp) ([]expression.Expression, LogicalPlan) { +func (p *LogicalSequence) PredicatePushDown(predicates []expression.Expression, op *util.LogicalOptimizeOp) ([]expression.Expression, LogicalPlan) { lastIdx := len(p.children) - 1 remained, newLastChild := p.children[lastIdx].PredicatePushDown(predicates, op) p.SetChild(lastIdx, newLastChild) diff --git a/pkg/planner/core/rule_predicate_simplification.go b/pkg/planner/core/rule_predicate_simplification.go index 387898cb57a78..dabc4a85b55de 100644 --- a/pkg/planner/core/rule_predicate_simplification.go +++ b/pkg/planner/core/rule_predicate_simplification.go @@ -21,6 +21,7 @@ import ( "github.com/pingcap/tidb/pkg/expression" "github.com/pingcap/tidb/pkg/parser/ast" + "github.com/pingcap/tidb/pkg/planner/util" ) // predicateSimplification consolidates different predcicates on a column and its equivalence classes. Initial out is for @@ -64,12 +65,12 @@ func findPredicateType(expr expression.Expression) (*expression.Column, predicat return nil, otherPredicate } -func (*predicateSimplification) optimize(_ context.Context, p LogicalPlan, opt *logicalOptimizeOp) (LogicalPlan, bool, error) { +func (*predicateSimplification) optimize(_ context.Context, p LogicalPlan, opt *util.LogicalOptimizeOp) (LogicalPlan, bool, error) { planChanged := false return p.predicateSimplification(opt), planChanged, nil } -func (s *baseLogicalPlan) predicateSimplification(opt *logicalOptimizeOp) LogicalPlan { +func (s *baseLogicalPlan) predicateSimplification(opt *util.LogicalOptimizeOp) LogicalPlan { p := s.self for i, child := range p.Children() { newChild := child.predicateSimplification(opt) @@ -154,7 +155,7 @@ func applyPredicateSimplification(sctx PlanContext, predicates []expression.Expr return newValues } -func (ds *DataSource) predicateSimplification(*logicalOptimizeOp) LogicalPlan { +func (ds *DataSource) predicateSimplification(*util.LogicalOptimizeOp) LogicalPlan { p := ds.self.(*DataSource) p.pushedDownConds = applyPredicateSimplification(p.SCtx(), p.pushedDownConds) p.allConds = applyPredicateSimplification(p.SCtx(), p.allConds) diff --git a/pkg/planner/core/rule_push_down_sequence.go b/pkg/planner/core/rule_push_down_sequence.go index ab196860b26a0..17964353c5b52 100644 --- a/pkg/planner/core/rule_push_down_sequence.go +++ b/pkg/planner/core/rule_push_down_sequence.go @@ -14,7 +14,11 @@ package core -import "context" +import ( + "context" + + "github.com/pingcap/tidb/pkg/planner/util" +) type pushDownSequenceSolver struct { } @@ -23,7 +27,7 @@ func (*pushDownSequenceSolver) name() string { return "push_down_sequence" } -func (pdss *pushDownSequenceSolver) optimize(_ context.Context, lp LogicalPlan, _ *logicalOptimizeOp) (LogicalPlan, bool, error) { +func (pdss *pushDownSequenceSolver) optimize(_ context.Context, lp LogicalPlan, _ *util.LogicalOptimizeOp) (LogicalPlan, bool, error) { planChanged := false return pdss.recursiveOptimize(nil, lp), planChanged, nil } diff --git a/pkg/planner/core/rule_resolve_grouping_expand.go b/pkg/planner/core/rule_resolve_grouping_expand.go index cfd3a89939793..53e5892aaea05 100644 --- a/pkg/planner/core/rule_resolve_grouping_expand.go +++ b/pkg/planner/core/rule_resolve_grouping_expand.go @@ -16,6 +16,8 @@ package core import ( "context" + + "github.com/pingcap/tidb/pkg/planner/util" ) // For normal rollup Expand construction, its logical Expand should be bound @@ -72,7 +74,7 @@ type resolveExpand struct { // (upper required) (grouping sets columns appended) // // Expand operator itself is kind like a projection, while difference is that it has a multi projection list, named as leveled projection. -func (*resolveExpand) optimize(_ context.Context, p LogicalPlan, opt *logicalOptimizeOp) (LogicalPlan, bool, error) { +func (*resolveExpand) optimize(_ context.Context, p LogicalPlan, opt *util.LogicalOptimizeOp) (LogicalPlan, bool, error) { planChanged := false // As you see, Expand's leveled projection should be built after all column-prune is done. So we just make generating-leveled-projection // as the last rule of logical optimization, which is more clear. (spark has column prune action before building expand) @@ -84,7 +86,7 @@ func (*resolveExpand) name() string { return "resolve_expand" } -func genExpand(p LogicalPlan, opt *logicalOptimizeOp) (LogicalPlan, error) { +func genExpand(p LogicalPlan, opt *util.LogicalOptimizeOp) (LogicalPlan, error) { for i, child := range p.Children() { np, err := genExpand(child, opt) if err != nil { diff --git a/pkg/planner/core/rule_result_reorder.go b/pkg/planner/core/rule_result_reorder.go index be1bcd26fd5ea..b503a6ede7326 100644 --- a/pkg/planner/core/rule_result_reorder.go +++ b/pkg/planner/core/rule_result_reorder.go @@ -39,7 +39,7 @@ This rule reorders results by modifying or injecting a Sort operator: type resultReorder struct { } -func (rs *resultReorder) optimize(_ context.Context, lp LogicalPlan, _ *logicalOptimizeOp) (LogicalPlan, bool, error) { +func (rs *resultReorder) optimize(_ context.Context, lp LogicalPlan, _ *util.LogicalOptimizeOp) (LogicalPlan, bool, error) { planChanged := false ordered := rs.completeSort(lp) if !ordered { diff --git a/pkg/planner/core/rule_semi_join_rewrite.go b/pkg/planner/core/rule_semi_join_rewrite.go index 1209b3a3ed60a..0c78e6ec6e921 100644 --- a/pkg/planner/core/rule_semi_join_rewrite.go +++ b/pkg/planner/core/rule_semi_join_rewrite.go @@ -20,6 +20,7 @@ import ( "github.com/pingcap/tidb/pkg/expression" "github.com/pingcap/tidb/pkg/expression/aggregation" "github.com/pingcap/tidb/pkg/parser/ast" + "github.com/pingcap/tidb/pkg/planner/util" h "github.com/pingcap/tidb/pkg/util/hint" ) @@ -36,7 +37,7 @@ import ( type semiJoinRewriter struct { } -func (smj *semiJoinRewriter) optimize(_ context.Context, p LogicalPlan, _ *logicalOptimizeOp) (LogicalPlan, bool, error) { +func (smj *semiJoinRewriter) optimize(_ context.Context, p LogicalPlan, _ *util.LogicalOptimizeOp) (LogicalPlan, bool, error) { planChanged := false newLogicalPlan, err := smj.recursivePlan(p) return newLogicalPlan, planChanged, err diff --git a/pkg/planner/core/rule_topn_push_down.go b/pkg/planner/core/rule_topn_push_down.go index e8f9c672c982f..830995f55387d 100644 --- a/pkg/planner/core/rule_topn_push_down.go +++ b/pkg/planner/core/rule_topn_push_down.go @@ -27,12 +27,12 @@ import ( type pushDownTopNOptimizer struct { } -func (*pushDownTopNOptimizer) optimize(_ context.Context, p LogicalPlan, opt *logicalOptimizeOp) (LogicalPlan, bool, error) { +func (*pushDownTopNOptimizer) optimize(_ context.Context, p LogicalPlan, opt *util.LogicalOptimizeOp) (LogicalPlan, bool, error) { planChanged := false return p.pushDownTopN(nil, opt), planChanged, nil } -func (s *baseLogicalPlan) pushDownTopN(topN *LogicalTopN, opt *logicalOptimizeOp) LogicalPlan { +func (s *baseLogicalPlan) pushDownTopN(topN *LogicalTopN, opt *util.LogicalOptimizeOp) LogicalPlan { p := s.self for i, child := range p.Children() { p.Children()[i] = child.pushDownTopN(nil, opt) @@ -43,7 +43,7 @@ func (s *baseLogicalPlan) pushDownTopN(topN *LogicalTopN, opt *logicalOptimizeOp return p } -func (p *LogicalCTE) pushDownTopN(topN *LogicalTopN, opt *logicalOptimizeOp) LogicalPlan { +func (p *LogicalCTE) pushDownTopN(topN *LogicalTopN, opt *util.LogicalOptimizeOp) LogicalPlan { if topN != nil { return topN.setChild(p, opt) } @@ -51,7 +51,7 @@ func (p *LogicalCTE) pushDownTopN(topN *LogicalTopN, opt *logicalOptimizeOp) Log } // setChild set p as topn's child. -func (lt *LogicalTopN) setChild(p LogicalPlan, opt *logicalOptimizeOp) LogicalPlan { +func (lt *LogicalTopN) setChild(p LogicalPlan, opt *util.LogicalOptimizeOp) LogicalPlan { // Remove this TopN if its child is a TableDual. dual, isDual := p.(*LogicalTableDual) if isDual { @@ -81,7 +81,7 @@ func (lt *LogicalTopN) setChild(p LogicalPlan, opt *logicalOptimizeOp) LogicalPl return lt } -func (ls *LogicalSort) pushDownTopN(topN *LogicalTopN, opt *logicalOptimizeOp) LogicalPlan { +func (ls *LogicalSort) pushDownTopN(topN *LogicalTopN, opt *util.LogicalOptimizeOp) LogicalPlan { if topN == nil { return ls.baseLogicalPlan.pushDownTopN(nil, opt) } else if topN.isLimit() { @@ -93,13 +93,13 @@ func (ls *LogicalSort) pushDownTopN(topN *LogicalTopN, opt *logicalOptimizeOp) L return ls.children[0].pushDownTopN(topN, opt) } -func (p *LogicalLimit) convertToTopN(opt *logicalOptimizeOp) *LogicalTopN { +func (p *LogicalLimit) convertToTopN(opt *util.LogicalOptimizeOp) *LogicalTopN { topn := LogicalTopN{Offset: p.Offset, Count: p.Count, PreferLimitToCop: p.PreferLimitToCop}.Init(p.SCtx(), p.QueryBlockOffset()) appendConvertTopNTraceStep(p, topn, opt) return topn } -func (p *LogicalLimit) pushDownTopN(topN *LogicalTopN, opt *logicalOptimizeOp) LogicalPlan { +func (p *LogicalLimit) pushDownTopN(topN *LogicalTopN, opt *util.LogicalOptimizeOp) LogicalPlan { child := p.children[0].pushDownTopN(p.convertToTopN(opt), opt) if topN != nil { return topN.setChild(child, opt) @@ -107,7 +107,7 @@ func (p *LogicalLimit) pushDownTopN(topN *LogicalTopN, opt *logicalOptimizeOp) L return child } -func (p *LogicalUnionAll) pushDownTopN(topN *LogicalTopN, opt *logicalOptimizeOp) LogicalPlan { +func (p *LogicalUnionAll) pushDownTopN(topN *LogicalTopN, opt *util.LogicalOptimizeOp) LogicalPlan { for i, child := range p.children { var newTopN *LogicalTopN if topN != nil { @@ -126,7 +126,7 @@ func (p *LogicalUnionAll) pushDownTopN(topN *LogicalTopN, opt *logicalOptimizeOp return p } -func (p *LogicalProjection) pushDownTopN(topN *LogicalTopN, opt *logicalOptimizeOp) LogicalPlan { +func (p *LogicalProjection) pushDownTopN(topN *LogicalTopN, opt *util.LogicalOptimizeOp) LogicalPlan { for _, expr := range p.Exprs { if expression.HasAssignSetVarFunc(expr) { return p.baseLogicalPlan.pushDownTopN(topN, opt) @@ -164,7 +164,7 @@ func (p *LogicalProjection) pushDownTopN(topN *LogicalTopN, opt *logicalOptimize return p } -func (p *LogicalLock) pushDownTopN(topN *LogicalTopN, opt *logicalOptimizeOp) LogicalPlan { +func (p *LogicalLock) pushDownTopN(topN *LogicalTopN, opt *util.LogicalOptimizeOp) LogicalPlan { if topN != nil { p.children[0] = p.children[0].pushDownTopN(topN, opt) } @@ -172,7 +172,7 @@ func (p *LogicalLock) pushDownTopN(topN *LogicalTopN, opt *logicalOptimizeOp) Lo } // pushDownTopNToChild will push a topN to one child of join. The idx stands for join child index. 0 is for left child. -func (p *LogicalJoin) pushDownTopNToChild(topN *LogicalTopN, idx int, opt *logicalOptimizeOp) LogicalPlan { +func (p *LogicalJoin) pushDownTopNToChild(topN *LogicalTopN, idx int, opt *util.LogicalOptimizeOp) LogicalPlan { if topN == nil { return p.children[idx].pushDownTopN(nil, opt) } @@ -198,7 +198,7 @@ func (p *LogicalJoin) pushDownTopNToChild(topN *LogicalTopN, idx int, opt *logic return p.children[idx].pushDownTopN(newTopN, opt) } -func (p *LogicalJoin) pushDownTopN(topN *LogicalTopN, opt *logicalOptimizeOp) LogicalPlan { +func (p *LogicalJoin) pushDownTopN(topN *LogicalTopN, opt *util.LogicalOptimizeOp) LogicalPlan { switch p.JoinType { case LeftOuterJoin, LeftOuterSemiJoin, AntiLeftOuterSemiJoin: p.children[0] = p.pushDownTopNToChild(topN, 0, opt) @@ -221,17 +221,17 @@ func (*pushDownTopNOptimizer) name() string { return "topn_push_down" } -func appendTopNPushDownTraceStep(parent LogicalPlan, child LogicalPlan, opt *logicalOptimizeOp) { +func appendTopNPushDownTraceStep(parent LogicalPlan, child LogicalPlan, opt *util.LogicalOptimizeOp) { action := func() string { return fmt.Sprintf("%v_%v is added as %v_%v's parent", parent.TP(), parent.ID(), child.TP(), child.ID()) } reason := func() string { return fmt.Sprintf("%v is pushed down", parent.TP()) } - opt.appendStepToCurrent(parent.ID(), parent.TP(), reason, action) + opt.AppendStepToCurrent(parent.ID(), parent.TP(), reason, action) } -func appendTopNPushDownJoinTraceStep(p *LogicalJoin, topN *LogicalTopN, idx int, opt *logicalOptimizeOp) { +func appendTopNPushDownJoinTraceStep(p *LogicalJoin, topN *LogicalTopN, idx int, opt *util.LogicalOptimizeOp) { action := func() string { buffer := bytes.NewBufferString(fmt.Sprintf("%v_%v is added and pushed into %v_%v's ", topN.TP(), topN.ID(), p.TP(), p.ID())) @@ -260,10 +260,10 @@ func appendTopNPushDownJoinTraceStep(p *LogicalJoin, topN *LogicalTopN, idx int, buffer.WriteString("table") return buffer.String() } - opt.appendStepToCurrent(p.ID(), p.TP(), reason, action) + opt.AppendStepToCurrent(p.ID(), p.TP(), reason, action) } -func appendSortPassByItemsTraceStep(sort *LogicalSort, topN *LogicalTopN, opt *logicalOptimizeOp) { +func appendSortPassByItemsTraceStep(sort *LogicalSort, topN *LogicalTopN, opt *util.LogicalOptimizeOp) { action := func() string { buffer := bytes.NewBufferString(fmt.Sprintf("%v_%v passes ByItems[", sort.TP(), sort.ID())) for i, item := range sort.ByItems { @@ -278,25 +278,25 @@ func appendSortPassByItemsTraceStep(sort *LogicalSort, topN *LogicalTopN, opt *l reason := func() string { return fmt.Sprintf("%v_%v is Limit originally", topN.TP(), topN.ID()) } - opt.appendStepToCurrent(sort.ID(), sort.TP(), reason, action) + opt.AppendStepToCurrent(sort.ID(), sort.TP(), reason, action) } -func appendNewTopNTraceStep(topN *LogicalTopN, union *LogicalUnionAll, opt *logicalOptimizeOp) { +func appendNewTopNTraceStep(topN *LogicalTopN, union *LogicalUnionAll, opt *util.LogicalOptimizeOp) { reason := func() string { return "" } action := func() string { return fmt.Sprintf("%v_%v is added and pushed down across %v_%v", topN.TP(), topN.ID(), union.TP(), union.ID()) } - opt.appendStepToCurrent(topN.ID(), topN.TP(), reason, action) + opt.AppendStepToCurrent(topN.ID(), topN.TP(), reason, action) } -func appendConvertTopNTraceStep(p LogicalPlan, topN *LogicalTopN, opt *logicalOptimizeOp) { +func appendConvertTopNTraceStep(p LogicalPlan, topN *LogicalTopN, opt *util.LogicalOptimizeOp) { reason := func() string { return "" } action := func() string { return fmt.Sprintf("%v_%v is converted into %v_%v", p.TP(), p.ID(), topN.TP(), topN.ID()) } - opt.appendStepToCurrent(topN.ID(), topN.TP(), reason, action) + opt.AppendStepToCurrent(topN.ID(), topN.TP(), reason, action) } diff --git a/pkg/planner/core/util.go b/pkg/planner/core/util.go index d40b98acd2ff9..349cd9005abca 100644 --- a/pkg/planner/core/util.go +++ b/pkg/planner/core/util.go @@ -25,6 +25,7 @@ import ( "github.com/pingcap/tidb/pkg/parser/model" "github.com/pingcap/tidb/pkg/parser/mysql" "github.com/pingcap/tidb/pkg/planner/core/internal/base" + "github.com/pingcap/tidb/pkg/planner/util" "github.com/pingcap/tidb/pkg/sessionctx" "github.com/pingcap/tidb/pkg/table" "github.com/pingcap/tidb/pkg/tablecodec" @@ -136,7 +137,7 @@ func (s *logicalSchemaProducer) setSchemaAndNames(schema *expression.Schema, nam } // inlineProjection prunes unneeded columns inline a executor. -func (s *logicalSchemaProducer) inlineProjection(parentUsedCols []*expression.Column, opt *logicalOptimizeOp) { +func (s *logicalSchemaProducer) inlineProjection(parentUsedCols []*expression.Column, opt *util.LogicalOptimizeOp) { prunedColumns := make([]*expression.Column, 0) used := expression.GetUsedList(s.SCtx().GetExprCtx(), parentUsedCols, s.Schema()) for i := len(used) - 1; i >= 0; i-- { diff --git a/pkg/planner/util/BUILD.bazel b/pkg/planner/util/BUILD.bazel index af04ffaa5c027..e1ab66724eb81 100644 --- a/pkg/planner/util/BUILD.bazel +++ b/pkg/planner/util/BUILD.bazel @@ -6,6 +6,7 @@ go_library( "byitem.go", "expression.go", "misc.go", + "optTracer.go", "path.go", ], importpath = "github.com/pingcap/tidb/pkg/planner/util", @@ -20,6 +21,7 @@ go_library( "//pkg/util/collate", "//pkg/util/ranger", "//pkg/util/size", + "//pkg/util/tracing", ], ) diff --git a/pkg/planner/util/optTracer.go b/pkg/planner/util/optTracer.go new file mode 100644 index 0000000000000..79fa275f93ff8 --- /dev/null +++ b/pkg/planner/util/optTracer.go @@ -0,0 +1,68 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import "github.com/pingcap/tidb/pkg/util/tracing" + +// optTracer define those basic element for logical optimizing trace and physical optimizing trace. +// logicalOptRule inside the accommodated pkg `util` should only be depended on by logical `rule` pkg. +// +// rule related -----> core/util + +// LogicalOptimizeOp is logical optimizing option for tracing. +type LogicalOptimizeOp struct { + // tracer is goring to track optimize steps during rule optimizing + tracer *tracing.LogicalOptimizeTracer +} + +// TracerIsNil returns whether inside tracer is nil +func (op *LogicalOptimizeOp) TracerIsNil() bool { + return op.tracer == nil +} + +// DefaultLogicalOptimizeOption returns the default LogicalOptimizeOp. +func DefaultLogicalOptimizeOption() *LogicalOptimizeOp { + return &LogicalOptimizeOp{} +} + +// WithEnableOptimizeTracer attach the customized tracer to current LogicalOptimizeOp. +func (op *LogicalOptimizeOp) WithEnableOptimizeTracer(tracer *tracing.LogicalOptimizeTracer) *LogicalOptimizeOp { + op.tracer = tracer + return op +} + +// AppendBeforeRuleOptimize just appends a before-rule plan tracer. +func (op *LogicalOptimizeOp) AppendBeforeRuleOptimize(index int, name string, build func() *tracing.PlanTrace) { + if op == nil || op.tracer == nil { + return + } + op.tracer.AppendRuleTracerBeforeRuleOptimize(index, name, build()) +} + +// AppendStepToCurrent appends a step of current action. +func (op *LogicalOptimizeOp) AppendStepToCurrent(id int, tp string, reason, action func() string) { + if op == nil || op.tracer == nil { + return + } + op.tracer.AppendRuleTracerStepToCurrent(id, tp, reason(), action()) +} + +// RecordFinalLogicalPlan records the final logical plan. +func (op *LogicalOptimizeOp) RecordFinalLogicalPlan(build func() *tracing.PlanTrace) { + if op == nil || op.tracer == nil { + return + } + op.tracer.RecordFinalLogicalPlan(build()) +}