From c6391a93166a98490ef76bfd5ad2a623896114ea Mon Sep 17 00:00:00 2001 From: Zhang Jian Date: Fri, 8 Mar 2019 13:25:44 +0800 Subject: [PATCH 1/2] planner: chose outer table based on cost when both tables are specified in TIDB_INLJ (#9579) --- cmd/explaintest/r/index_join.result | 27 ++++++++++++ cmd/explaintest/t/index_join.test | 12 ++++++ executor/index_lookup_join_test.go | 19 +++++++++ planner/core/exhaust_physical_plans.go | 59 ++++++++++++++------------ planner/core/logical_plan_builder.go | 5 +++ planner/core/logical_plans.go | 11 +++-- planner/core/physical_plan_test.go | 4 +- planner/core/planbuilder.go | 13 ++++++ 8 files changed, 117 insertions(+), 33 deletions(-) create mode 100644 cmd/explaintest/r/index_join.result create mode 100644 cmd/explaintest/t/index_join.test diff --git a/cmd/explaintest/r/index_join.result b/cmd/explaintest/r/index_join.result new file mode 100644 index 0000000000000..4f81db898cf7f --- /dev/null +++ b/cmd/explaintest/r/index_join.result @@ -0,0 +1,27 @@ +drop table if exists t1, t2; +create table t1(a bigint, b bigint, index idx(a)); +create table t2(a bigint, b bigint, index idx(a)); +insert into t1 values(1, 1), (1, 1), (1, 1), (1, 1), (1, 1); +insert into t2 values(1, 1); +analyze table t1, t2; +explain select /*+ TIDB_INLJ(t1, t2) */ * from t1 join t2 on t1.a=t2.a; +id count task operator info +IndexJoin_16 5.00 root inner join, inner:IndexLookUp_15, outer key:test.t2.a, inner key:test.t1.a +├─IndexLookUp_15 0.00 root +│ ├─Selection_14 0.00 cop not(isnull(test.t1.a)) +│ │ └─IndexScan_12 5.00 cop table:t1, index:a, range: decided by [test.t2.a], keep order:false +│ └─TableScan_13 0.00 cop table:t1, keep order:false +└─TableReader_19 1.00 root data:Selection_18 + └─Selection_18 1.00 cop not(isnull(test.t2.a)) + └─TableScan_17 1.00 cop table:t2, range:[-inf,+inf], keep order:false +explain select * from t1 join t2 on t1.a=t2.a; +id count task operator info +Projection_6 5.00 root test.t1.a, test.t1.b, test.t2.a, test.t2.b +└─IndexJoin_12 5.00 root inner join, inner:IndexLookUp_11, outer key:test.t2.a, inner key:test.t1.a + ├─TableReader_32 1.00 root data:Selection_31 + │ └─Selection_31 1.00 cop not(isnull(test.t2.a)) + │ └─TableScan_30 1.00 cop table:t2, range:[-inf,+inf], keep order:false + └─IndexLookUp_11 0.00 root + ├─Selection_10 0.00 cop not(isnull(test.t1.a)) + │ └─IndexScan_8 5.00 cop table:t1, index:a, range: decided by [test.t2.a], keep order:false + └─TableScan_9 0.00 cop table:t1, keep order:false diff --git a/cmd/explaintest/t/index_join.test b/cmd/explaintest/t/index_join.test new file mode 100644 index 0000000000000..acd0ae0f47c08 --- /dev/null +++ b/cmd/explaintest/t/index_join.test @@ -0,0 +1,12 @@ +drop table if exists t1, t2; +create table t1(a bigint, b bigint, index idx(a)); +create table t2(a bigint, b bigint, index idx(a)); +insert into t1 values(1, 1), (1, 1), (1, 1), (1, 1), (1, 1); +insert into t2 values(1, 1); + +analyze table t1, t2; + +-- Test https://github.com/pingcap/tidb/issues/9577 +-- we expect the following two SQL chose t2 as the outer table +explain select /*+ TIDB_INLJ(t1, t2) */ * from t1 join t2 on t1.a=t2.a; +explain select * from t1 join t2 on t1.a=t2.a; diff --git a/executor/index_lookup_join_test.go b/executor/index_lookup_join_test.go index f25792324db93..e6fbd8521a786 100644 --- a/executor/index_lookup_join_test.go +++ b/executor/index_lookup_join_test.go @@ -37,3 +37,22 @@ func (s *testSuite) TestIndexLookupJoinHang(c *C) { } rs.Close() } + +func (s *testSuite) TestInapplicableIndexJoinHint(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec(`drop table if exists t1, t2;`) + tk.MustExec(`create table t1(a bigint, b bigint);`) + tk.MustExec(`create table t2(a bigint, b bigint);`) + tk.MustQuery(`select /*+ TIDB_INLJ(t1, t2) */ * from t1, t2;`).Check(testkit.Rows()) + tk.MustQuery(`show warnings;`).Check(testkit.Rows(`Warning 1815 Optimizer Hint /*+ TIDB_INLJ(t1, t2) */ is inapplicable without column equal ON condition`)) + tk.MustQuery(`select /*+ TIDB_INLJ(t1, t2) */ * from t1 join t2 on t1.a=t2.a;`).Check(testkit.Rows()) + tk.MustQuery(`show warnings;`).Check(testkit.Rows(`Warning 1815 Optimizer Hint /*+ TIDB_INLJ(t1, t2) */ is inapplicable`)) + + tk.MustExec(`drop table if exists t1, t2;`) + tk.MustExec(`create table t1(a bigint, b bigint, index idx_a(a));`) + tk.MustExec(`create table t2(a bigint, b bigint);`) + tk.MustQuery(`select /*+ TIDB_INLJ(t1) */ * from t1 left join t2 on t1.a=t2.a;`).Check(testkit.Rows()) + tk.MustQuery(`show warnings;`).Check(testkit.Rows(`Warning 1815 Optimizer Hint /*+ TIDB_INLJ(t1) */ is inapplicable`)) + tk.MustQuery(`select /*+ TIDB_INLJ(t2) */ * from t1 right join t2 on t1.a=t2.a;`).Check(testkit.Rows()) + tk.MustQuery(`show warnings;`).Check(testkit.Rows(`Warning 1815 Optimizer Hint /*+ TIDB_INLJ(t2) */ is inapplicable`)) +} diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index 1369b13bec39e..220b764603be3 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -14,6 +14,7 @@ package core import ( + "fmt" "math" "github.com/pingcap/parser/ast" @@ -614,62 +615,68 @@ func (p *LogicalJoin) buildFakeEqCondsForIndexJoin(keys, idxCols []*expression.C // tryToGetIndexJoin will get index join by hints. If we can generate a valid index join by hint, the second return value // will be true, which means we force to choose this index join. Otherwise we will select a join algorithm with min-cost. -func (p *LogicalJoin) tryToGetIndexJoin(prop *property.PhysicalProperty) ([]PhysicalPlan, bool) { - plans := make([]PhysicalPlan, 0, 2) +func (p *LogicalJoin) tryToGetIndexJoin(prop *property.PhysicalProperty) (indexJoins []PhysicalPlan, forced bool) { rightOuter := (p.preferJoinType & preferLeftAsIndexInner) > 0 leftOuter := (p.preferJoinType & preferRightAsIndexInner) > 0 - if len(p.EqualConditions) == 0 { - if leftOuter || rightOuter { - warning := ErrInternal.GenWithStack("TIDB_INLJ hint is inapplicable without column equal ON condition") + hasIndexJoinHint := leftOuter || rightOuter + + defer func() { + if !forced && hasIndexJoinHint { + // Construct warning message prefix. + errMsg := "Optimizer Hint TIDB_INLJ is inapplicable" + if p.hintInfo != nil { + errMsg = fmt.Sprintf("Optimizer Hint %s is inapplicable", p.hintInfo.restore2IndexJoinHint()) + } + + // Append inapplicable reason. + if len(p.EqualConditions) == 0 { + errMsg += " without column equal ON condition" + } + + // Generate warning message to client. + warning := ErrInternal.GenWithStack(errMsg) p.ctx.GetSessionVars().StmtCtx.AppendWarning(warning) } + }() + + if len(p.EqualConditions) == 0 { return nil, false } + switch p.JoinType { case SemiJoin, AntiSemiJoin, LeftOuterSemiJoin, AntiLeftOuterSemiJoin, LeftOuterJoin: join := p.getIndexJoinByOuterIdx(prop, 0) - if join != nil { - // If the plan is not nil and matches the hint, return it directly. - if leftOuter { - return join, true - } - plans = append(plans, join...) - } + return join, join != nil && leftOuter case RightOuterJoin: join := p.getIndexJoinByOuterIdx(prop, 1) - if join != nil { - // If the plan is not nil and matches the hint, return it directly. - if rightOuter { - return join, true - } - plans = append(plans, join...) - } + return join, join != nil && rightOuter case InnerJoin: lhsCardinality := p.Children()[0].statsInfo().Count() rhsCardinality := p.Children()[1].statsInfo().Count() leftJoins := p.getIndexJoinByOuterIdx(prop, 0) - if leftOuter && leftJoins != nil { + if leftJoins != nil && leftOuter && !rightOuter { return leftJoins, true } rightJoins := p.getIndexJoinByOuterIdx(prop, 1) - if rightOuter && rightJoins != nil { + if rightJoins != nil && rightOuter && !leftOuter { return rightJoins, true } if leftJoins != nil && lhsCardinality < rhsCardinality { - return leftJoins, leftOuter + return leftJoins, hasIndexJoinHint } if rightJoins != nil && rhsCardinality < lhsCardinality { - return rightJoins, rightOuter + return rightJoins, hasIndexJoinHint } - plans = append(plans, leftJoins...) - plans = append(plans, rightJoins...) + joins := append(leftJoins, rightJoins...) + return joins, hasIndexJoinHint && len(joins) != 0 } - return plans, false + + return nil, false } // LogicalJoin can generates hash join, index join and sort merge join. diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index 380feeeaac1a4..2e71162b7641e 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -267,6 +267,11 @@ func (p *LogicalJoin) setPreferredJoinType(hintInfo *tableHintInfo) error { p.preferJoinType |= preferRightAsIndexInner } + // set hintInfo for further usage if this hint info can be used. + if p.preferJoinType != 0 { + p.hintInfo = hintInfo + } + // If there're multiple join types and one of them is not index join hint, // then there is a conflict of join types. if bits.OnesCount(p.preferJoinType) > 1 && (p.preferJoinType^preferRightAsIndexInner^preferLeftAsIndexInner) > 0 { diff --git a/planner/core/logical_plans.go b/planner/core/logical_plans.go index 12a2e5b1171c2..20c912d91eedf 100644 --- a/planner/core/logical_plans.go +++ b/planner/core/logical_plans.go @@ -95,10 +95,13 @@ const ( type LogicalJoin struct { logicalSchemaProducer - JoinType JoinType - reordered bool - cartesianJoin bool - StraightJoin bool + JoinType JoinType + reordered bool + cartesianJoin bool + StraightJoin bool + + // hintInfo stores the join algorithm hint information specified by client. + hintInfo *tableHintInfo preferJoinType uint EqualConditions []*expression.ScalarFunction diff --git a/planner/core/physical_plan_test.go b/planner/core/physical_plan_test.go index 17fc636a8b327..395aeae16e265 100644 --- a/planner/core/physical_plan_test.go +++ b/planner/core/physical_plan_test.go @@ -22,7 +22,6 @@ import ( "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/session" "github.com/pingcap/tidb/sessionctx" - "github.com/pingcap/tidb/terror" "github.com/pingcap/tidb/util/testleak" "golang.org/x/net/context" ) @@ -1302,6 +1301,5 @@ func (s *testPlanSuite) TestIndexLookupCartesianJoin(c *C) { c.Assert(core.ToString(p), Equals, "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}") warnings := se.GetSessionVars().StmtCtx.GetWarnings() lastWarn := warnings[len(warnings)-1] - err = core.ErrInternal.GenWithStack("TIDB_INLJ hint is inapplicable without column equal ON condition") - c.Assert(terror.ErrorEqual(err, lastWarn.Err), IsTrue) + c.Assert(lastWarn.Err.Error(), Equals, "[planner:1815]Optimizer Hint /*+ TIDB_INLJ(t1, t2) */ is inapplicable without column equal ON condition") } diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index d09f99f0da231..b3524133a8822 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -14,6 +14,7 @@ package core import ( + "bytes" "fmt" "strings" @@ -83,6 +84,18 @@ func (info *tableHintInfo) matchTableName(tables []*model.CIStr, tablesInHints [ return false } +func (info *tableHintInfo) restore2IndexJoinHint() string { + buffer := bytes.NewBufferString("/*+ TIDB_INLJ(") + for i, tableName := range info.indexNestedLoopJoinTables { + buffer.WriteString(tableName.O) + if i < len(info.indexNestedLoopJoinTables)-1 { + buffer.WriteString(", ") + } + } + buffer.WriteString(") */") + return buffer.String() +} + // clauseCode indicates in which clause the column is currently. type clauseCode int From 9e60f3f553fb8341cdaf2816785e6798377b659d Mon Sep 17 00:00:00 2001 From: Jian Zhang Date: Sat, 9 Mar 2019 14:09:43 +0800 Subject: [PATCH 2/2] fix explain test --- cmd/explaintest/r/index_join.result | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/cmd/explaintest/r/index_join.result b/cmd/explaintest/r/index_join.result index 4f81db898cf7f..dd5e03a46ddaa 100644 --- a/cmd/explaintest/r/index_join.result +++ b/cmd/explaintest/r/index_join.result @@ -6,22 +6,17 @@ insert into t2 values(1, 1); analyze table t1, t2; explain select /*+ TIDB_INLJ(t1, t2) */ * from t1 join t2 on t1.a=t2.a; id count task operator info -IndexJoin_16 5.00 root inner join, inner:IndexLookUp_15, outer key:test.t2.a, inner key:test.t1.a -├─IndexLookUp_15 0.00 root -│ ├─Selection_14 0.00 cop not(isnull(test.t1.a)) -│ │ └─IndexScan_12 5.00 cop table:t1, index:a, range: decided by [test.t2.a], keep order:false -│ └─TableScan_13 0.00 cop table:t1, keep order:false -└─TableReader_19 1.00 root data:Selection_18 - └─Selection_18 1.00 cop not(isnull(test.t2.a)) - └─TableScan_17 1.00 cop table:t2, range:[-inf,+inf], keep order:false +IndexJoin_14 1.25 root inner join, inner:IndexLookUp_13, outer key:test.t2.a, inner key:test.t1.a +├─IndexLookUp_13 5.00 root +│ ├─IndexScan_11 5.00 cop table:t1, index:a, range: decided by [test.t2.a], keep order:false +│ └─TableScan_12 5.00 cop table:t1, keep order:false +└─TableReader_16 1.00 root data:TableScan_15 + └─TableScan_15 1.00 cop table:t2, range:[-inf,+inf], keep order:false explain select * from t1 join t2 on t1.a=t2.a; id count task operator info -Projection_6 5.00 root test.t1.a, test.t1.b, test.t2.a, test.t2.b -└─IndexJoin_12 5.00 root inner join, inner:IndexLookUp_11, outer key:test.t2.a, inner key:test.t1.a - ├─TableReader_32 1.00 root data:Selection_31 - │ └─Selection_31 1.00 cop not(isnull(test.t2.a)) - │ └─TableScan_30 1.00 cop table:t2, range:[-inf,+inf], keep order:false - └─IndexLookUp_11 0.00 root - ├─Selection_10 0.00 cop not(isnull(test.t1.a)) - │ └─IndexScan_8 5.00 cop table:t1, index:a, range: decided by [test.t2.a], keep order:false - └─TableScan_9 0.00 cop table:t1, keep order:false +IndexJoin_14 1.25 root inner join, inner:IndexLookUp_13, outer key:test.t2.a, inner key:test.t1.a +├─IndexLookUp_13 5.00 root +│ ├─IndexScan_11 5.00 cop table:t1, index:a, range: decided by [test.t2.a], keep order:false +│ └─TableScan_12 5.00 cop table:t1, keep order:false +└─TableReader_26 1.00 root data:TableScan_25 + └─TableScan_25 1.00 cop table:t2, range:[-inf,+inf], keep order:false