Skip to content

Commit

Permalink
planner: fix the issue accessing unnecessary table side caused by col…
Browse files Browse the repository at this point in the history
…umn pruning (#54815)

close #54213
  • Loading branch information
qw4990 authored Jul 23, 2024
1 parent 08785b6 commit 1cd80eb
Show file tree
Hide file tree
Showing 13 changed files with 216 additions and 73 deletions.
17 changes: 7 additions & 10 deletions pkg/planner/core/casetest/hint/testdata/integration_suite_out.json
Original file line number Diff line number Diff line change
Expand Up @@ -799,16 +799,13 @@
"HashAgg 16000.00 root group by:Column#41, funcs:firstrow(Column#41)->Column#41",
"└─Union 1000000010000.00 root ",
" ├─HashJoin 1000000000000.00 root CARTESIAN inner join",
" │ ├─TableReader(Build) 10000.00 root data:TableFullScan",
" │ │ └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo",
" │ └─Projection(Probe) 100000000.00 root 1->Column#55",
" │ └─HashJoin 100000000.00 root CARTESIAN inner join",
" │ ├─Projection(Build) 10000.00 root 1->Column#54",
" │ │ └─IndexReader 10000.00 root index:IndexFullScan",
" │ │ └─IndexFullScan 10000.00 cop[tikv] table:t3, index:idx_a(a) keep order:false, stats:pseudo",
" │ └─Projection(Probe) 10000.00 root 1->Column#53",
" │ └─IndexReader 10000.00 root index:IndexFullScan",
" │ └─IndexFullScan 10000.00 cop[tikv] table:t2, index:idx_a(a) keep order:false, stats:pseudo",
" │ ├─IndexReader(Build) 10000.00 root index:IndexFullScan",
" │ │ └─IndexFullScan 10000.00 cop[tikv] table:t3, index:idx_a(a) keep order:false, stats:pseudo",
" │ └─HashJoin(Probe) 100000000.00 root CARTESIAN inner join",
" │ ├─IndexReader(Build) 10000.00 root index:IndexFullScan",
" │ │ └─IndexFullScan 10000.00 cop[tikv] table:t2, index:idx_a(a) keep order:false, stats:pseudo",
" │ └─TableReader(Probe) 10000.00 root data:TableFullScan",
" │ └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo",
" └─TableReader 10000.00 root data:TableFullScan",
" └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo"
],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8064,9 +8064,9 @@
"SQL": "explain select * from t WHERE TRUE = ALL ( SELECT TRUE GROUP BY 1 LIMIT 1 ) IS NULL IS NOT NULL;",
"Plan": [
"HashJoin_14 10000.00 root CARTESIAN inner join",
"├─StreamAgg_19(Build) 1.00 root funcs:count(1)->Column#13",
"├─StreamAgg_19(Build) 1.00 root funcs:count(1)->Column#12",
"│ └─Limit_22 1.00 root offset:0, count:1",
"│ └─HashAgg_23 1.00 root group by:1, ",
"│ └─HashAgg_23 1.00 root group by:1, funcs:firstrow(1)->Column#13",
"│ └─TableDual_24 1.00 root rows:1",
"└─TableReader_17(Probe) 10000.00 root data:TableFullScan_16",
" └─TableFullScan_16 10000.00 cop[tikv] table:t keep order:false, stats:pseudo"
Expand All @@ -8077,11 +8077,11 @@
"SQL": "explain select * from t WHERE TRUE = ALL ( SELECT TRUE from t GROUP BY 1 LIMIT 1 ) is null is not null;",
"Plan": [
"HashJoin_14 10000.00 root CARTESIAN inner join",
"├─StreamAgg_19(Build) 1.00 root funcs:count(1)->Column#16",
"├─StreamAgg_19(Build) 1.00 root funcs:count(1)->Column#15",
"│ └─Limit_22 1.00 root offset:0, count:1",
"│ └─HashAgg_27 1.00 root group by:Column#17, funcs:firstrow(Column#18)->test.t.a, funcs:firstrow(Column#19)->test.t.b, funcs:firstrow(Column#20)->test.t._tidb_rowid",
"│ └─HashAgg_27 1.00 root group by:Column#17, funcs:firstrow(Column#18)->Column#16",
"│ └─TableReader_28 1.00 root data:HashAgg_23",
"│ └─HashAgg_23 1.00 cop[tikv] group by:1, funcs:firstrow(test.t.a)->Column#18, funcs:firstrow(test.t.b)->Column#19, funcs:firstrow(test.t._tidb_rowid)->Column#20",
"│ └─HashAgg_23 1.00 cop[tikv] group by:1, funcs:firstrow(1)->Column#18",
"│ └─TableFullScan_26 10000.00 cop[tikv] table:t keep order:false, stats:pseudo",
"└─TableReader_17(Probe) 10000.00 root data:TableFullScan_16",
" └─TableFullScan_16 10000.00 cop[tikv] table:t keep order:false, stats:pseudo"
Expand Down
31 changes: 15 additions & 16 deletions pkg/planner/core/casetest/testdata/integration_suite_out.json
Original file line number Diff line number Diff line change
Expand Up @@ -356,22 +356,21 @@
{
"SQL": "explain format = 'verbose' select (2) in (select /*+ read_from_storage(tiflash[t1]) */ count(*) from t1) from (select t.b < (select /*+ read_from_storage(tiflash[t2]) */ t.b from t2 limit 1 ) from t3 t) t; -- we do generate the agg pushed-down plan of mpp, but cost-cmp failed",
"Plan": [
"HashJoin_19 3.00 162366.01 root CARTESIAN left outer semi join",
"├─Selection_36(Build) 0.80 31149.25 root eq(2, Column#18)",
"│ └─StreamAgg_43 1.00 31099.35 root funcs:count(1)->Column#18",
"│ └─TableReader_55 3.00 30949.65 root MppVersion: 2, data:ExchangeSender_54",
"│ └─ExchangeSender_54 3.00 464139.20 mpp[tiflash] ExchangeType: PassThrough",
"│ └─TableFullScan_53 3.00 464139.20 mpp[tiflash] table:t1 keep order:false",
"└─Projection_20(Probe) 3.00 129648.62 root 1->Column#28",
" └─Apply_22 3.00 129648.32 root CARTESIAN left outer join",
" ├─IndexReader_26(Build) 3.00 53.37 root index:IndexFullScan_25",
" │ └─IndexFullScan_25 3.00 610.50 cop[tikv] table:t, index:c(b) keep order:false",
" └─Projection_27(Probe) 3.00 43198.32 root 1->Column#26",
" └─Limit_30 3.00 43198.22 root offset:0, count:1",
" └─TableReader_35 3.00 43198.22 root MppVersion: 2, data:ExchangeSender_34",
" └─ExchangeSender_34 3.00 647920.44 mpp[tiflash] ExchangeType: PassThrough",
" └─Limit_33 3.00 647920.44 mpp[tiflash] offset:0, count:1",
" └─TableFullScan_32 3.00 647920.44 mpp[tiflash] table:t2 keep order:false"
"HashJoin_19 3.00 65286.81 root CARTESIAN left outer semi join",
"├─Selection_35(Build) 0.80 31149.25 root eq(2, Column#18)",
"│ └─StreamAgg_42 1.00 31099.35 root funcs:count(1)->Column#18",
"│ └─TableReader_54 3.00 30949.65 root MppVersion: 2, data:ExchangeSender_53",
"│ └─ExchangeSender_53 3.00 464139.20 mpp[tiflash] ExchangeType: PassThrough",
"│ └─TableFullScan_52 3.00 464139.20 mpp[tiflash] table:t1 keep order:false",
"└─HashJoin_20(Probe) 3.00 32569.41 root CARTESIAN left outer join",
" ├─MaxOneRow_26(Build) 1.00 30937.60 root ",
" │ └─Limit_29 1.00 30937.60 root offset:0, count:1",
" │ └─TableReader_34 1.00 30937.60 root MppVersion: 2, data:ExchangeSender_33",
" │ └─ExchangeSender_33 1.00 464046.40 mpp[tiflash] ExchangeType: PassThrough",
" │ └─Limit_32 1.00 464046.40 mpp[tiflash] offset:0, count:1",
" │ └─TableFullScan_31 1.00 464046.40 mpp[tiflash] table:t2 keep order:false",
" └─IndexReader_25(Probe) 3.00 53.37 root index:IndexFullScan_24",
" └─IndexFullScan_24 3.00 610.50 cop[tikv] table:t, index:c(b) keep order:false"
]
},
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -426,8 +426,8 @@
"Plan": [
" TableReader root ",
" └─ExchangeSender cop[tiflash] ",
" └─Selection cop[tiflash] gt(test.t1.b, ?)",
" └─TableFullScan cop[tiflash] table:t1, range:[?,?], pushed down filter:gt(test.t1.a, ?), gt(test.t1.c, ?), keep order:false"
" └─Selection cop[tiflash] gt(test.t1.a, ?)",
" └─TableFullScan cop[tiflash] table:t1, range:[?,?], pushed down filter:gt(test.t1.b, ?), gt(test.t1.c, ?), keep order:false"
]
},
{
Expand All @@ -443,8 +443,8 @@
"Plan": [
" TableReader root ",
" └─ExchangeSender cop[tiflash] ",
" └─Selection cop[tiflash] gt(test.t1.b, ?), or(lt(test.t1.a, ?), lt(test.t1.b, ?))",
" └─TableFullScan cop[tiflash] table:t1, range:[?,?], pushed down filter:gt(test.t1.a, ?), keep order:false"
" └─Selection cop[tiflash] gt(test.t1.a, ?), or(lt(test.t1.a, ?), lt(test.t1.b, ?))",
" └─TableFullScan cop[tiflash] table:t1, range:[?,?], pushed down filter:gt(test.t1.b, ?), keep order:false"
]
},
{
Expand Down
20 changes: 20 additions & 0 deletions pkg/planner/core/integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2711,6 +2711,26 @@ func TestIssue52472(t *testing.T) {
require.NoError(t, rs.Close())
}

func TestIssue54213(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)

tk.MustExec(`use test`)
tk.MustExec(`CREATE TABLE tb (
object_id bigint(20),
a bigint(20) ,
b bigint(20) ,
c bigint(20) ,
PRIMARY KEY (object_id),
KEY ab (a,b))`)
tk.MustQuery(`explain select count(1) from (select /*+ force_index(tb, ab) */ 1 from tb where a=1 and b=1 limit 100) a`).Check(
testkit.Rows("StreamAgg_11 1.00 root funcs:count(1)->Column#6",
"└─Limit_12 0.10 root offset:0, count:100",
" └─IndexReader_16 0.10 root index:Limit_15",
" └─Limit_15 0.10 cop[tikv] offset:0, count:100",
" └─IndexRangeScan_14 0.10 cop[tikv] table:tb, index:ab(a, b) range:[1 1,1 1], keep order:false, stats:pseudo"))
}

func TestIssue48257(t *testing.T) {
store, dom := testkit.CreateMockStoreAndDomain(t)
tk := testkit.NewTestKit(t, store)
Expand Down
2 changes: 1 addition & 1 deletion pkg/planner/core/logical_plan_trace_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ func TestSingleRuleTraceStep(t *testing.T) {
assertAction: "TableDual_4's columns[test.t.i_date,test.t.h,test.t.g,test.t.f,test.t.e_str,test.t.d_str,test.t.c_str,test.t.e,test.t.d,test.t.c,test.t.b,test.t.a] have been pruned",
},
{
assertAction: "Join_7's columns[Column#28,test.t.a] have been pruned",
assertAction: "Join_7's columns[Column#27] have been pruned",
},
},
},
Expand Down
8 changes: 4 additions & 4 deletions pkg/planner/core/rule_column_pruning.go
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,10 @@ func (p *LogicalProjection) PruneColumns(parentUsedCols []*expression.Column, op
if err != nil {
return nil, err
}
// If its columns are all pruned, we directly use its child. The child will output at least one column.
if p.Schema().Len() == 0 {
return p.Children()[0], nil
}
return p, nil
}

Expand Down Expand Up @@ -628,10 +632,6 @@ func (p *LogicalWindow) extractUsedCols(parentUsedCols []*expression.Column) []*

// PruneColumns implements LogicalPlan interface.
func (p *LogicalLimit) PruneColumns(parentUsedCols []*expression.Column, opt *logicalOptimizeOp) (LogicalPlan, error) {
if len(parentUsedCols) == 0 { // happens when LIMIT appears in UPDATE.
return p, nil
}

savedUsedCols := make([]*expression.Column, len(parentUsedCols))
copy(savedUsedCols, parentUsedCols)

Expand Down
2 changes: 1 addition & 1 deletion pkg/planner/core/testdata/plan_suite_unexported_out.json
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@
"Join{Join{DataScan(t)->DataScan(x)->Aggr(firstrow(test.t.a))}(test.t.a,test.t.a)->Projection->DataScan(x)->Aggr(firstrow(test.t.a))}(test.t.a,test.t.a)->Projection->Projection",
"Apply{DataScan(t1)->DataScan(t2)->Sel([eq(test.t.a, test.t.a)])->Projection->Sort->Limit}->Projection->Sel([eq(test.t.b, test.t.b)])->Projection",
"Apply{DataScan(t2)->DataScan(t1)->Sel([eq(test.t.a, test.t.a)])->Projection}->Projection",
"Join{DataScan(t2)->DataScan(t1)->Aggr(firstrow(test.t.c),count(1))}(test.t.c,test.t.c)->Projection->Aggr(count(1))->Projection"
"Join{DataScan(t2)->DataScan(t1)->Aggr(firstrow(test.t.c),count(1))}(test.t.c,test.t.c)->Aggr(count(1))->Projection"
]
},
{
Expand Down
17 changes: 17 additions & 0 deletions pkg/planner/core/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ package core

import (
"fmt"
"math"
"slices"
"sort"
"strings"
Expand Down Expand Up @@ -139,6 +140,22 @@ func (s *logicalSchemaProducer) setSchemaAndNames(schema *expression.Schema, nam
func (s *logicalSchemaProducer) inlineProjection(parentUsedCols []*expression.Column, opt *logicalOptimizeOp) {
prunedColumns := make([]*expression.Column, 0)
used := expression.GetUsedList(parentUsedCols, s.Schema())
if len(parentUsedCols) == 0 {
// When this operator output no columns, we return its smallest column for safety.
minColLen := math.MaxInt
chosenPos := 0
for i, col := range s.schema.Columns {
flen := col.GetType().GetFlen()
if flen < minColLen {
chosenPos = i
minColLen = flen
}
}
// It should be always true.
if len(used) > 0 {
used[chosenPos] = true
}
}
for i := len(used) - 1; i >= 0; i-- {
if !used[i] {
prunedColumns = append(prunedColumns, s.Schema().Columns[i])
Expand Down
17 changes: 8 additions & 9 deletions tests/integrationtest/r/planner/core/casetest/integration.result
Original file line number Diff line number Diff line change
Expand Up @@ -376,15 +376,14 @@ id estRows task access object operator info
HashJoin 10000.00 root CARTESIAN left outer semi join, other cond:eq(2, planner__core__casetest__integration.t.b)
├─TableReader(Build) 10000.00 root data:TableFullScan
│ └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo
└─Projection(Probe) 10000.00 root 1->Column#27
└─Apply 10000.00 root CARTESIAN left outer join
├─TableReader(Build) 10000.00 root data:TableFullScan
│ └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo
└─Projection(Probe) 10000.00 root 1->Column#25
└─Limit 10000.00 root offset:0, count:1
└─TableReader 10000.00 root data:Limit
└─Limit 10000.00 cop[tikv] offset:0, count:1
└─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo
└─HashJoin(Probe) 10000.00 root CARTESIAN left outer join
├─MaxOneRow(Build) 1.00 root
│ └─Limit 1.00 root offset:0, count:1
│ └─TableReader 1.00 root data:Limit
│ └─Limit 1.00 cop[tikv] offset:0, count:1
│ └─TableFullScan 1.00 cop[tikv] table:t1 keep order:false, stats:pseudo
└─TableReader(Probe) 10000.00 root data:TableFullScan
└─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo
select (2) in (select b from t) from (select t.a < (select t.a from t t1 limit 1) from t) t;
(2) in (select b from t)
1
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1879,31 +1879,31 @@ HashJoin 304261169.13 root CARTESIAN inner join
explain format = 'brief' select * from ((select /*+ leading(t5, t7) */ t8.a, t8.b from t8, t7, t6, t5 where t5.a = t6.a and t6.b=t7.b) t3 join t4 on t3.a=t4.a) join (t1 join t2 on t1.a=t2.a) on t1.a=t4.a;
id estRows task access object operator info
HashJoin 2431655263674.32 root CARTESIAN inner join
├─HashJoin(Build) 124625374.88 root inner join, equal:[eq(planner__core__casetest__rule__rule_join_reorder.t7.b, planner__core__casetest__rule__rule_join_reorder.t6.b) eq(planner__core__casetest__rule__rule_join_reorder.t5.a, planner__core__casetest__rule__rule_join_reorder.t6.a)]
│ ├─TableReader(Build) 9980.01 root data:Selection
│ │ └─Selection 9980.01 cop[tikv] not(isnull(planner__core__casetest__rule__rule_join_reorder.t6.a)), not(isnull(planner__core__casetest__rule__rule_join_reorder.t6.b))
│ │ └─TableFullScan 10000.00 cop[tikv] table:t6 keep order:false, stats:pseudo
│ └─HashJoin(Probe) 99800100.00 root CARTESIAN inner join
├─HashJoin(Build) 19511.72 root inner join, equal:[eq(planner__core__casetest__rule__rule_join_reorder.t1.a, planner__core__casetest__rule__rule_join_reorder.t2.a)]
│ ├─TableReader(Build) 9990.00 root data:Selection
│ │ └─Selection 9990.00 cop[tikv] not(isnull(planner__core__casetest__rule__rule_join_reorder.t2.a))
│ │ └─TableFullScan 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo
│ └─HashJoin(Probe) 15609.38 root inner join, equal:[eq(planner__core__casetest__rule__rule_join_reorder.t4.a, planner__core__casetest__rule__rule_join_reorder.t1.a)]
│ ├─TableReader(Build) 9990.00 root data:Selection
│ │ └─Selection 9990.00 cop[tikv] not(isnull(planner__core__casetest__rule__rule_join_reorder.t7.b))
│ │ └─TableFullScan 10000.00 cop[tikv] table:t7 keep order:false, stats:pseudo
│ └─IndexReader(Probe) 9990.00 root index:IndexFullScan
│ └─IndexFullScan 9990.00 cop[tikv] table:t5, index:a(a) keep order:false, stats:pseudo
└─HashJoin(Probe) 19511.72 root inner join, equal:[eq(planner__core__casetest__rule__rule_join_reorder.t1.a, planner__core__casetest__rule__rule_join_reorder.t2.a)]
├─TableReader(Build) 9990.00 root data:Selection
│ └─Selection 9990.00 cop[tikv] not(isnull(planner__core__casetest__rule__rule_join_reorder.t2.a))
│ └─TableFullScan 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo
└─HashJoin(Probe) 15609.38 root inner join, equal:[eq(planner__core__casetest__rule__rule_join_reorder.t4.a, planner__core__casetest__rule__rule_join_reorder.t1.a)]
│ │ └─Selection 9990.00 cop[tikv] not(isnull(planner__core__casetest__rule__rule_join_reorder.t1.a))
│ │ └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo
│ └─HashJoin(Probe) 12487.50 root inner join, equal:[eq(planner__core__casetest__rule__rule_join_reorder.t8.a, planner__core__casetest__rule__rule_join_reorder.t4.a)]
│ ├─TableReader(Build) 9990.00 root data:Selection
│ │ └─Selection 9990.00 cop[tikv] not(isnull(planner__core__casetest__rule__rule_join_reorder.t4.a))
│ │ └─TableFullScan 10000.00 cop[tikv] table:t4 keep order:false, stats:pseudo
│ └─TableReader(Probe) 9990.00 root data:Selection
│ └─Selection 9990.00 cop[tikv] not(isnull(planner__core__casetest__rule__rule_join_reorder.t8.a))
│ └─TableFullScan 10000.00 cop[tikv] table:t8 keep order:false, stats:pseudo
└─HashJoin(Probe) 124625374.88 root inner join, equal:[eq(planner__core__casetest__rule__rule_join_reorder.t7.b, planner__core__casetest__rule__rule_join_reorder.t6.b) eq(planner__core__casetest__rule__rule_join_reorder.t5.a, planner__core__casetest__rule__rule_join_reorder.t6.a)]
├─TableReader(Build) 9980.01 root data:Selection
│ └─Selection 9980.01 cop[tikv] not(isnull(planner__core__casetest__rule__rule_join_reorder.t6.a)), not(isnull(planner__core__casetest__rule__rule_join_reorder.t6.b))
│ └─TableFullScan 10000.00 cop[tikv] table:t6 keep order:false, stats:pseudo
└─HashJoin(Probe) 99800100.00 root CARTESIAN inner join
├─TableReader(Build) 9990.00 root data:Selection
│ └─Selection 9990.00 cop[tikv] not(isnull(planner__core__casetest__rule__rule_join_reorder.t1.a))
│ └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo
└─HashJoin(Probe) 12487.50 root inner join, equal:[eq(planner__core__casetest__rule__rule_join_reorder.t8.a, planner__core__casetest__rule__rule_join_reorder.t4.a)]
├─TableReader(Build) 9990.00 root data:Selection
│ └─Selection 9990.00 cop[tikv] not(isnull(planner__core__casetest__rule__rule_join_reorder.t4.a))
│ └─TableFullScan 10000.00 cop[tikv] table:t4 keep order:false, stats:pseudo
└─TableReader(Probe) 9990.00 root data:Selection
└─Selection 9990.00 cop[tikv] not(isnull(planner__core__casetest__rule__rule_join_reorder.t8.a))
└─TableFullScan 10000.00 cop[tikv] table:t8 keep order:false, stats:pseudo
│ └─Selection 9990.00 cop[tikv] not(isnull(planner__core__casetest__rule__rule_join_reorder.t7.b))
│ └─TableFullScan 10000.00 cop[tikv] table:t7 keep order:false, stats:pseudo
└─IndexReader(Probe) 9990.00 root index:IndexFullScan
└─IndexFullScan 9990.00 cop[tikv] table:t5, index:a(a) keep order:false, stats:pseudo
explain format = 'brief' select * from ((select /*+ leading(t6, t8, t7) */ t8.a, t8.b from t8, t7, t6, t5 where t5.a = t6.a and t6.b=t7.b) t3 join t4 on t3.a=t4.a) join (t1 join t2 on t1.a=t2.a) on t1.a=t4.a;
id estRows task access object operator info
HashJoin 304261169.13 root inner join, equal:[eq(planner__core__casetest__rule__rule_join_reorder.t1.a, planner__core__casetest__rule__rule_join_reorder.t2.a)]
Expand Down
Loading

0 comments on commit 1cd80eb

Please sign in to comment.