Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

planner: fix index merge plan when expr cannot be pushed to tikv #30341

Merged
merged 32 commits into from
Dec 23, 2021
Merged

planner: fix index merge plan when expr cannot be pushed to tikv #30341

Show file tree
Hide file tree
Changes from 31 commits
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
56bf325
planner: fix index merge plan when expr has not been pushed to tikv.
guo-shaoge Dec 1, 2021
c5d2ddb
Merge branch 'master' of https://github.com/pingcap/tidb into fix_ind…
guo-shaoge Dec 1, 2021
c4dc91f
Merge branch 'master' into fix_index_merge_sel_plan
qw4990 Dec 3, 2021
b60786e
Merge branch 'master' into fix_index_merge_sel_plan
guo-shaoge Dec 7, 2021
4800457
fix comment
guo-shaoge Dec 8, 2021
7de60f4
Merge branch 'fix_index_merge_sel_plan' of github.com:guo-shaoge/tidb…
guo-shaoge Dec 8, 2021
b4654ea
fix comment
guo-shaoge Dec 11, 2021
cc74a3e
fix cannot be pushed expr in partial_path.IndexFilters
guo-shaoge Dec 12, 2021
594b734
Merge branch 'master' into fix_index_merge_sel_plan
guo-shaoge Dec 12, 2021
bc26b95
Merge branch 'fix_index_merge_sel_plan' of github.com:guo-shaoge/tidb…
guo-shaoge Dec 12, 2021
5d98c2c
fix cost
guo-shaoge Dec 12, 2021
7f00244
Update planner/core/find_best_task.go
guo-shaoge Dec 13, 2021
eba2cbd
Update planner/core/find_best_task.go
guo-shaoge Dec 13, 2021
9b4ec81
Update planner/core/find_best_task.go
guo-shaoge Dec 13, 2021
b59caf3
check partial path's TableFilters can be pushed to TiKV or not.
guo-shaoge Dec 14, 2021
3630fdb
Merge branch 'master' of https://github.com/pingcap/tidb into fix_ind…
guo-shaoge Dec 14, 2021
bc8c836
fix
guo-shaoge Dec 14, 2021
9187957
fix didn't push not expr
guo-shaoge Dec 14, 2021
b89e269
fix est row
guo-shaoge Dec 19, 2021
fb21ec4
check expr can be pushed in ds.pushedDownConds
guo-shaoge Dec 19, 2021
19ffe55
add more comment
guo-shaoge Dec 19, 2021
e3f3e2e
fix index on virtual column for index merge
guo-shaoge Dec 19, 2021
e249201
fix typo
guo-shaoge Dec 19, 2021
cfe8309
Merge branch 'master' into fix_index_merge_sel_plan
guo-shaoge Dec 21, 2021
f47ebc9
fix case in index_merge_reader_test.go
guo-shaoge Dec 21, 2021
11e5179
fix case
guo-shaoge Dec 21, 2021
887bccd
a simple way to do same thing
guo-shaoge Dec 21, 2021
1d6abd4
fix warnings and case
guo-shaoge Dec 21, 2021
b75d087
fix comment
guo-shaoge Dec 22, 2021
ff0fb07
add index merge warnings
guo-shaoge Dec 22, 2021
8b50eec
fix warning case
guo-shaoge Dec 23, 2021
a78dc67
Merge branch 'master' into fix_index_merge_sel_plan
ti-chi-bot Dec 23, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 30 additions & 30 deletions cmd/explaintest/r/index_merge.result
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -237,11 +237,11 @@ insert into t1(c1, c2) values(1, 1), (2, 2), (3, 3), (4, 4), (5, 5);
explain select /*+ use_index_merge(t1) */ * from t1 where c1 < 10 or c2 < 10 and c3 < 10 order by 1;
id estRows task access object operator info
Sort_5 4060.74 root test.t1.c1
└─IndexMerge_12 2250.55 root
├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─Selection_11(Probe) 2250.55 cop[tikv] or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), lt(test.t1.c3, 10)))
└─TableRowIDScan_10 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo
└─Selection_12 2250.55 root or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), lt(test.t1.c3, 10)))
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why Selection_12 not push down: c3 is generated column.

└─IndexMerge_11 5542.21 root
├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─TableRowIDScan_10(Probe) 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo
select /*+ use_index_merge(t1) */ * from t1 where c1 < 10 or c2 < 10 and c3 < 10 order by 1;
c1 c2 c3
1 1 2
Expand All @@ -252,11 +252,11 @@ c1 c2 c3
explain select /*+ use_index_merge(t1) */ * from t1 where c1 < 10 or c2 < 10 and c3 = c1 + c2 order by 1;
id estRows task access object operator info
Sort_5 5098.44 root test.t1.c1
└─IndexMerge_12 2825.66 root
├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─Selection_11(Probe) 2825.66 cop[tikv] or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), eq(test.t1.c3, plus(test.t1.c1, test.t1.c2))))
└─TableRowIDScan_10 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo
└─Selection_12 2825.66 root or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), eq(test.t1.c3, plus(test.t1.c1, test.t1.c2))))
└─IndexMerge_11 5542.21 root
├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─TableRowIDScan_10(Probe) 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo
select /*+ use_index_merge(t1) */ * from t1 where c1 < 10 or c2 < 10 and c3 = c1 + c2 order by 1;
c1 c2 c3
1 1 2
Expand All @@ -267,11 +267,11 @@ c1 c2 c3
explain select /*+ use_index_merge(t1) */ * from t1 where c1 < 10 or c2 < 10 and substring(c3, c2) order by 1;
id estRows task access object operator info
Sort_5 5098.44 root test.t1.c1
└─IndexMerge_12 2825.66 root
├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─Selection_11(Probe) 2825.66 cop[tikv] or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), istrue_with_null(cast(substring(cast(test.t1.c3, var_string(20)), test.t1.c2), double BINARY))))
└─TableRowIDScan_10 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo
└─Selection_12 2825.66 root or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), istrue_with_null(cast(substring(cast(test.t1.c3, var_string(20)), test.t1.c2), double BINARY))))
└─IndexMerge_11 5542.21 root
├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─TableRowIDScan_10(Probe) 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo
select /*+ use_index_merge(t1) */ * from t1 where c1 < 10 or c2 < 10 and substring(c3, c2) order by 1;
c1 c2 c3
1 1 2
Expand All @@ -282,11 +282,11 @@ c1 c2 c3
explain select /*+ use_index_merge(t1) */ * from t1 where c1 < 10 or c2 < 10 and c3 order by 1;
id estRows task access object operator info
Sort_5 4800.37 root test.t1.c1
└─IndexMerge_12 2660.47 root
├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─Selection_11(Probe) 2660.47 cop[tikv] or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), test.t1.c3))
└─TableRowIDScan_10 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo
└─Selection_12 2660.47 root or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), test.t1.c3))
└─IndexMerge_11 5542.21 root
├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─TableRowIDScan_10(Probe) 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo
select /*+ use_index_merge(t1) */ * from t1 where c1 < 10 or c2 < 10 and c3 order by 1;
c1 c2 c3
1 1 2
Expand All @@ -302,11 +302,11 @@ select /*+ use_index_merge(t1) */ * from t1 where c1 < 10 or c2 < 10 and c3 < 10
explain select * from t1 where c1 < 10 or c2 < 10 and c3 < 10 order by 1;
id estRows task access object operator info
Sort_5 4060.74 root test.t1.c1
└─IndexMerge_12 2250.55 root
├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─Selection_11(Probe) 2250.55 cop[tikv] or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), lt(test.t1.c3, 10)))
└─TableRowIDScan_10 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo
└─Selection_12 2250.55 root or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), lt(test.t1.c3, 10)))
└─IndexMerge_11 5542.21 root
├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─TableRowIDScan_10(Probe) 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo
select * from t1 where c1 < 10 or c2 < 10 and c3 < 10 order by 1;
c1 c2 c3
1 1 2
Expand Down Expand Up @@ -722,11 +722,11 @@ c1 c2 c3 c4 c5
explain select /*+ use_index_merge(t1) */ * from t1 where (c1 < 10 or c2 < 10) and substring(c3, 1, 1) = '1' order by 1;
id estRows task access object operator info
Sort_5 4433.77 root test.t1.c1
└─IndexMerge_12 4433.77 root
├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─Selection_11(Probe) 4433.77 cop[tikv] eq(substring(cast(test.t1.c3, var_string(20)), 1, 1), "1")
└─TableRowIDScan_10 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo
└─Selection_12 4433.77 root eq(substring(cast(test.t1.c3, var_string(20)), 1, 1), "1")
└─IndexMerge_11 5542.21 root
├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─TableRowIDScan_10(Probe) 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo
select /*+ use_index_merge(t1) */ * from t1 where (c1 < 10 or c2 < 10) and substring(c3, 1, 1) = '1' order by 1;
c1 c2 c3 c4 c5
1 1 1 1 1
Expand Down
57 changes: 45 additions & 12 deletions planner/core/find_best_task.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"github.com/pingcap/tidb/planner/property"
"github.com/pingcap/tidb/planner/util"
"github.com/pingcap/tidb/sessionctx"
"github.com/pingcap/tidb/sessionctx/stmtctx"
"github.com/pingcap/tidb/statistics"
"github.com/pingcap/tidb/types"
tidbutil "github.com/pingcap/tidb/util"
Expand Down Expand Up @@ -973,14 +974,17 @@ func (ds *DataSource) convertToIndexMergeScan(prop *property.PhysicalProperty, c
if prop.ExpectedCnt < ds.stats.RowCount {
totalRowCount *= prop.ExpectedCnt / ds.stats.RowCount
}
ts, partialCost, err := ds.buildIndexMergeTableScan(prop, path.TableFilters, totalRowCount)
ts, partialCost, remainingFilters, err := ds.buildIndexMergeTableScan(prop, path.TableFilters, totalRowCount)
if err != nil {
return nil, err
}
totalCost += partialCost
cop.tablePlan = ts
cop.idxMergePartPlans = scans
cop.cst = totalCost
if remainingFilters != nil {
cop.rootTaskConds = remainingFilters
}
task = cop.convertToRootTask(ds.ctx)
ds.addSelection4PlanCache(task.(*rootTask), ds.tableStats.ScaleByExpectCnt(totalRowCount), prop)
return task, nil
Expand Down Expand Up @@ -1092,8 +1096,10 @@ func setIndexMergeTableScanHandleCols(ds *DataSource, ts *PhysicalTableScan) (er
return
}

// buildIndexMergeTableScan() returns Selection that will be pushed to TiKV.
// Filters that cannot be pushed to TiKV are also returned, and an extra Selection above IndexMergeReader will be constructed later.
func (ds *DataSource) buildIndexMergeTableScan(prop *property.PhysicalProperty, tableFilters []expression.Expression,
totalRowCount float64) (PhysicalPlan, float64, error) {
totalRowCount float64) (PhysicalPlan, float64, []expression.Expression, error) {
var partialCost float64
sessVars := ds.ctx.GetSessionVars()
ts := PhysicalTableScan{
Expand All @@ -1108,7 +1114,7 @@ func (ds *DataSource) buildIndexMergeTableScan(prop *property.PhysicalProperty,
ts.SetSchema(ds.schema.Clone())
err := setIndexMergeTableScanHandleCols(ds, ts)
if err != nil {
return nil, 0, err
return nil, 0, nil, err
}
if ts.Table.PKIsHandle {
if pkColInfo := ts.Table.GetPkColInfo(); pkColInfo != nil {
Expand All @@ -1124,17 +1130,44 @@ func (ds *DataSource) buildIndexMergeTableScan(prop *property.PhysicalProperty,
ts.stats.StatsVersion = statistics.PseudoVersion
}
if len(tableFilters) > 0 {
partialCost += totalRowCount * sessVars.CopCPUFactor
selectivity, _, err := ds.tableStats.HistColl.Selectivity(ds.ctx, tableFilters, nil)
if err != nil {
logutil.BgLogger().Debug("calculate selectivity failed, use selection factor", zap.Error(err))
selectivity = SelectionFactor
pushedFilters, remainingFilters := extractFiltersForIndexMerge(sessVars.StmtCtx, ds.ctx.GetClient(), tableFilters)
pushedFilters1, remainingFilters1 := SplitSelCondsWithVirtualColumn(pushedFilters)
pushedFilters = pushedFilters1
remainingFilters = append(remainingFilters, remainingFilters1...)
if len(pushedFilters) != 0 {
partialCost += totalRowCount * sessVars.CopCPUFactor
selectivity, _, err := ds.tableStats.HistColl.Selectivity(ds.ctx, pushedFilters, nil)
if err != nil {
logutil.BgLogger().Debug("calculate selectivity failed, use selection factor", zap.Error(err))
selectivity = SelectionFactor
}
sel := PhysicalSelection{Conditions: pushedFilters}.Init(ts.ctx, ts.stats.ScaleByExpectCnt(selectivity*totalRowCount), ts.blockOffset)
sel.SetChildren(ts)
return sel, partialCost, remainingFilters, nil
}
return ts, partialCost, remainingFilters, nil
}
return ts, partialCost, nil, nil
}

// extractFiltersForIndexMerge returns:
// `pushed`: exprs that can be pushed to TiKV.
// `remaining`: exprs that can NOT be pushed to TiKV but can be pushed to other storage engines.
// Why do we need this func?
// IndexMerge only works on TiKV, so we need to find all exprs that cannot be pushed to TiKV, and add a new Selection above IndexMergeReader.
// But the new Selection should exclude the exprs that can NOT be pushed to ALL the storage engines.
// Because these exprs have already been put in another Selection(check rule_predicate_push_down).
func extractFiltersForIndexMerge(sc *stmtctx.StatementContext, client kv.Client, filters []expression.Expression) (pushed []expression.Expression, remaining []expression.Expression) {
for _, expr := range filters {
if expression.CanExprsPushDown(sc, []expression.Expression{expr}, client, kv.TiKV) {
pushed = append(pushed, expr)
continue
}
if expression.CanExprsPushDown(sc, []expression.Expression{expr}, client, kv.UnSpecified) {
remaining = append(remaining, expr)
}
sel := PhysicalSelection{Conditions: tableFilters}.Init(ts.ctx, ts.stats.ScaleByExpectCnt(selectivity*totalRowCount), ts.blockOffset)
sel.SetChildren(ts)
return sel, partialCost, nil
}
return ts, partialCost, nil
return
}

func indexCoveringCol(col *expression.Column, indexCols []*expression.Column, idxColLens []int) bool {
Expand Down
Loading