Skip to content

Commit

Permalink
expression: enable pushdown substring/char_length to tikv (#30193)
Browse files Browse the repository at this point in the history 
* expression: enable pushdown substring to tikv

Signed-off-by: guo-shaoge <[email protected]>
  • Loading branch information
@guo-shaoge
guo-shaoge authored Nov 28, 2021
1 parent dc07933 commit 3eaff35
Show file tree
Hide file tree
Showing 6 changed files with 70 additions and 11 deletions.
15 changes: 7 additions & 8 deletions cmd/explaintest/r/tpch.result
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1294,14 +1294,13 @@ cntrycode
order by
cntrycode;
id estRows task access object operator info
Sort 1.00 root Column#27
└─Projection 1.00 root Column#27, Column#28, Column#29
└─HashAgg 1.00 root group by:Column#33, funcs:count(1)->Column#28, funcs:sum(Column#31)->Column#29, funcs:firstrow(Column#32)->Column#27
└─Projection 0.00 root tpch.customer.c_acctbal, substring(tpch.customer.c_phone, 1, 2)->Column#32, substring(tpch.customer.c_phone, 1, 2)->Column#33
Sort 1.00 root Column#31
└─Projection 1.00 root Column#31, Column#32, Column#33
└─HashAgg 1.00 root group by:Column#37, funcs:count(1)->Column#32, funcs:sum(Column#35)->Column#33, funcs:firstrow(Column#36)->Column#31
└─Projection 0.00 root tpch.customer.c_acctbal, substring(tpch.customer.c_phone, 1, 2)->Column#36, substring(tpch.customer.c_phone, 1, 2)->Column#37
└─HashJoin 0.00 root anti semi join, equal:[eq(tpch.customer.c_custkey, tpch.orders.o_custkey)]
├─TableReader(Build) 75000000.00 root data:TableFullScan
│ └─TableFullScan 75000000.00 cop[tikv] table:orders keep order:false
└─Selection(Probe) 0.00 root in(substring(tpch.customer.c_phone, 1, 2), "20", "40", "22", "30", "39", "42", "21")
└─TableReader 0.00 root data:Selection
└─Selection 0.00 cop[tikv] gt(tpch.customer.c_acctbal, NULL)
└─TableFullScan 7500000.00 cop[tikv] table:customer keep order:false
└─TableReader(Probe) 0.00 root data:Selection
└─Selection 0.00 cop[tikv] gt(tpch.customer.c_acctbal, NULL), in(substring(tpch.customer.c_phone, 1, 2), "20", "40", "22", "30", "39", "42", "21")
└─TableFullScan 7500000.00 cop[tikv] table:customer keep order:false
1 change: 1 addition & 0 deletions expression/builtin_string.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,7 @@ var (
_ builtinFunc = &builtinBitLengthSig{}
_ builtinFunc = &builtinCharSig{}
_ builtinFunc = &builtinCharLengthUTF8Sig{}
_ builtinFunc = &builtinCharLengthBinarySig{}
_ builtinFunc = &builtinFindInSetSig{}
_ builtinFunc = &builtinMakeSetSig{}
_ builtinFunc = &builtinOctIntSig{}
Expand Down
2 changes: 2 additions & 0 deletions expression/distsql_builtin.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -932,6 +932,8 @@ func getSignatureByPB(ctx sessionctx.Context, sigCode tipb.ScalarFuncSig, tp *ti
f = &builtinCharSig{base}
case tipb.ScalarFuncSig_CharLengthUTF8:
f = &builtinCharLengthUTF8Sig{base}
case tipb.ScalarFuncSig_CharLength:
f = &builtinCharLengthBinarySig{base}
case tipb.ScalarFuncSig_Concat:
f = &builtinConcatSig{base, maxAllowedPacket}
case tipb.ScalarFuncSig_ConcatWS:
Expand Down
4 changes: 2 additions & 2 deletions expression/expr_to_pb_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -956,8 +956,8 @@ func (s *testEvaluatorSuite) TestExprOnlyPushDownToFlash(c *C) {
c.Assert(len(remained), Equals, 0)

pushed, remained = PushDownExprs(sc, exprs, client, kv.TiKV)
c.Assert(len(pushed), Equals, 0)
c.Assert(len(remained), Equals, len(exprs))
c.Assert(len(pushed), Equals, 1)
c.Assert(len(remained), Equals, len(exprs)-1)
}

func (s *testEvaluatorSuite) TestExprOnlyPushDownToTiKV(c *C) {
Expand Down
2 changes: 1 addition & 1 deletion expression/expression.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -955,7 +955,7 @@ func scalarExprSupportedByTiKV(sf *ScalarFunction) bool {

// string functions.
ast.Length, ast.BitLength, ast.Concat, ast.ConcatWS /*ast.Locate,*/, ast.Replace, ast.ASCII, ast.Hex,
ast.Reverse, ast.LTrim, ast.RTrim /*ast.Left,*/, ast.Strcmp, ast.Space, ast.Elt, ast.Field,
ast.Reverse, ast.LTrim, ast.RTrim /*ast.Left,*/, ast.Strcmp, ast.Space, ast.Elt, ast.Field, ast.Substring, ast.CharLength,

// json functions.
ast.JSONType, ast.JSONExtract, ast.JSONObject, ast.JSONArray, ast.JSONMerge, ast.JSONSet,
Expand Down
57 changes: 57 additions & 0 deletions planner/core/integration_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4087,3 +4087,60 @@ func (s *testIntegrationSuite) TestIssues27130(c *C) {
" └─IndexRangeScan 10.00 cop[tikv] table:t3, index:a(a, b, c) range:[1,1], keep order:false, stats:pseudo",
))
}

func (s *testIntegrationSuite) TestIndexMergeExprs(c *C) {
tk := testkit.NewTestKitWithInit(c, s.store)
// varbinary and binary can be used in IndexMerge.
tk.MustExec("drop table if exists t1;")
tk.MustExec("create table t1(c1 varchar(100), c2 varchar(100), c3 varbinary(100), c4 binary(100), key(c1), key(c2));")
tk.MustExec("insert into t1 values('ab', '10', '10', '10');")
tk.MustQuery("explain select /*+ use_index_merge(t1) */ * from t1 where c1 = 'de' or c2 = '10' and substring(c3, 10) = 10;").Check(testkit.Rows(
"IndexMerge_9 0.04 root ",
"├─IndexRangeScan_5(Build) 10.00 cop[tikv] table:t1, index:c1(c1) range:[\"de\",\"de\"], keep order:false, stats:pseudo",
"├─IndexRangeScan_6(Build) 10.00 cop[tikv] table:t1, index:c2(c2) range:[\"10\",\"10\"], keep order:false, stats:pseudo",
"└─Selection_8(Probe) 0.04 cop[tikv] or(eq(test.t1.c1, \"de\"), and(eq(test.t1.c2, \"10\"), eq(cast(substring(test.t1.c3, 10), double BINARY), 10)))",
" └─TableRowIDScan_7 19.99 cop[tikv] table:t1 keep order:false, stats:pseudo"))
tk.MustQuery("explain select /*+ use_index_merge(t1) */ * from t1 where c1 = 'de' or c2 = '10' and substring(c4, 10) = 10;").Check(testkit.Rows(
"IndexMerge_9 0.04 root ",
"├─IndexRangeScan_5(Build) 10.00 cop[tikv] table:t1, index:c1(c1) range:[\"de\",\"de\"], keep order:false, stats:pseudo",
"├─IndexRangeScan_6(Build) 10.00 cop[tikv] table:t1, index:c2(c2) range:[\"10\",\"10\"], keep order:false, stats:pseudo",
"└─Selection_8(Probe) 0.04 cop[tikv] or(eq(test.t1.c1, \"de\"), and(eq(test.t1.c2, \"10\"), eq(cast(substring(test.t1.c4, 10), double BINARY), 10)))",
" └─TableRowIDScan_7 19.99 cop[tikv] table:t1 keep order:false, stats:pseudo"))

tk.MustQuery("select /*+ use_index_merge(t1) */ 1 from t1 where c1 = 'de' or c2 = '10' and substring(c3, 1) = 10;").Check(testkit.Rows("1"))
tk.MustQuery("select /*+ use_index_merge(t1) */ 1 from t1 where c1 = 'de' or c2 = '10' and substring(c4, 1) = 10;").Check(testkit.Rows("1"))
tk.MustQuery("select /*+ use_index_merge(t1) */ 1 from t1 where c1 = 'de' or c2 = '10' and substring(c3, 1) = 1;").Check(testkit.Rows())
tk.MustQuery("select /*+ use_index_merge(t1) */ 1 from t1 where c1 = 'de' or c2 = '10' and substring(c4, 1) = 1;").Check(testkit.Rows())

// Test char_length.
tk.MustExec("insert into t1 values('ab', '10', '1234567ab', '1234567ab');")
tk.MustQuery("explain select /*+ use_index_merge(t1) */ * from t1 where c1 = 'de' or c2 = '10' and substring(c3, char_length(c3) - 5) = '567ab';").Check(testkit.Rows(
"IndexMerge_9 0.04 root ",
"├─IndexRangeScan_5(Build) 10.00 cop[tikv] table:t1, index:c1(c1) range:[\"de\",\"de\"], keep order:false, stats:pseudo",
"├─IndexRangeScan_6(Build) 10.00 cop[tikv] table:t1, index:c2(c2) range:[\"10\",\"10\"], keep order:false, stats:pseudo",
"└─Selection_8(Probe) 0.04 cop[tikv] or(eq(test.t1.c1, \"de\"), and(eq(test.t1.c2, \"10\"), eq(substring(test.t1.c3, minus(char_length(test.t1.c3), 5)), \"567ab\")))",
" └─TableRowIDScan_7 19.99 cop[tikv] table:t1 keep order:false, stats:pseudo"))
tk.MustQuery("explain select /*+ use_index_merge(t1) */ * from t1 where c1 = 'de' or c2 = '10' and substring(c3, char_length(c4) - 5) = '567ab';").Check(testkit.Rows(
"IndexMerge_9 0.04 root ",
"├─IndexRangeScan_5(Build) 10.00 cop[tikv] table:t1, index:c1(c1) range:[\"de\",\"de\"], keep order:false, stats:pseudo",
"├─IndexRangeScan_6(Build) 10.00 cop[tikv] table:t1, index:c2(c2) range:[\"10\",\"10\"], keep order:false, stats:pseudo",
"└─Selection_8(Probe) 0.04 cop[tikv] or(eq(test.t1.c1, \"de\"), and(eq(test.t1.c2, \"10\"), eq(substring(test.t1.c3, minus(char_length(test.t1.c4), 5)), \"567ab\")))",
" └─TableRowIDScan_7 19.99 cop[tikv] table:t1 keep order:false, stats:pseudo"))
tk.MustQuery("select /*+ use_index_merge(t1) */ 1 from t1 where c1 = 'de' or c2 = '10' and substring(c3, char_length(c3) - 5) = '4567ab';").Check(testkit.Rows("1"))
tk.MustQuery("select /*+ use_index_merge(t1) */ 1 from t1 where c1 = 'de' or c2 = '10' and substring(c3, char_length(c3) - 5, 6) = '4567ab';").Check(testkit.Rows("1"))
tk.MustQuery("select /*+ use_index_merge(t1) */ 1 from t1 where c1 = 'de' or c2 = '10' and substring(c3, char_length(c4) - 5) = '4567ab';").Check(testkit.Rows())
tk.MustQuery("select /*+ use_index_merge(t1) */ 1 from t1 where c1 = 'de' or c2 = '10' and substring(c3, char_length(c4) - 5, 6) = '4567ab';").Check(testkit.Rows())

// varchar can be used in IndexMerge.
tk.MustExec("drop table if exists t1;")
tk.MustExec("create table t1(c1 varchar(100), c2 varchar(100), c3 varchar(100), key(c1), key(c2));")
tk.MustExec("insert into t1 values('ab', '10', '10');")
tk.MustQuery("explain select /*+ use_index_merge(t1) */ * from t1 where c1 = 'de' or c2 = '10' and substring(c3, 10) = 10;").Check(testkit.Rows(
"IndexMerge_9 0.04 root ",
"├─IndexRangeScan_5(Build) 10.00 cop[tikv] table:t1, index:c1(c1) range:[\"de\",\"de\"], keep order:false, stats:pseudo",
"├─IndexRangeScan_6(Build) 10.00 cop[tikv] table:t1, index:c2(c2) range:[\"10\",\"10\"], keep order:false, stats:pseudo",
"└─Selection_8(Probe) 0.04 cop[tikv] or(eq(test.t1.c1, \"de\"), and(eq(test.t1.c2, \"10\"), eq(cast(substring(test.t1.c3, 10), double BINARY), 10)))",
" └─TableRowIDScan_7 19.99 cop[tikv] table:t1 keep order:false, stats:pseudo"))
tk.MustQuery("select /*+ use_index_merge(t1) */ 1 from t1 where c1 = 'de' or c2 = '10' and substring(c3, 1) = 10;").Check(testkit.Rows("1"))
tk.MustQuery("select /*+ use_index_merge(t1) */ 1 from t1 where c1 = 'de' or c2 = '10' and substring(c3, 1) = 1;").Check(testkit.Rows())
}

0 comments on commit 3eaff35

Please sign in to comment.