From fccc4bedd0e217f81f933877fe50a3ea3b36a548 Mon Sep 17 00:00:00 2001 From: Song Gao Date: Tue, 1 Mar 2022 15:29:45 +0800 Subject: [PATCH] expr: support push tikv supported functions down (#32553) ref pingcap/tidb#31846 --- .../explain_generate_column_substitute.result | 8 ++--- cmd/explaintest/r/index_merge.result | 10 +++--- expression/distsql_builtin.go | 6 ++++ expression/expr_to_pb_test.go | 34 ++++++++++++++++--- expression/expression.go | 5 +-- planner/core/integration_test.go | 26 ++++++++++++-- 6 files changed, 71 insertions(+), 18 deletions(-) diff --git a/cmd/explaintest/r/explain_generate_column_substitute.result b/cmd/explaintest/r/explain_generate_column_substitute.result index 8272cd691a007..5d598e82ebe74 100644 --- a/cmd/explaintest/r/explain_generate_column_substitute.result +++ b/cmd/explaintest/r/explain_generate_column_substitute.result @@ -447,14 +447,14 @@ StreamAgg 4.80 root group by:upper(test.t.b), funcs:max(upper(test.t.b))->Colum └─IndexFullScan 6.00 cop[tikv] table:t, index:expression_index_2(upper(`b`)) keep order:true desc format = 'brief' select count(upper(b)) from t use index() group by upper(b); id estRows task access object operator info -HashAgg 6.00 root group by:Column#9, funcs:count(Column#8)->Column#7 -└─Projection 6.00 root upper(test.t.b)->Column#8, upper(test.t.b)->Column#9 +HashAgg 6.00 root group by:Column#11, funcs:count(Column#10)->Column#7 +└─Projection 6.00 root upper(test.t.b)->Column#10, upper(test.t.b)->Column#11 └─TableReader 6.00 root data:TableFullScan └─TableFullScan 6.00 cop[tikv] table:t keep order:false desc format = 'brief' select max(upper(b)) from t use index() group by upper(b); id estRows task access object operator info -HashAgg 6.00 root group by:Column#9, funcs:max(Column#8)->Column#7 -└─Projection 6.00 root upper(test.t.b)->Column#8, upper(test.t.b)->Column#9 +HashAgg 6.00 root group by:Column#11, funcs:max(Column#10)->Column#7 +└─Projection 6.00 root upper(test.t.b)->Column#10, upper(test.t.b)->Column#11 └─TableReader 6.00 root data:TableFullScan └─TableFullScan 6.00 cop[tikv] table:t keep order:false drop table if exists t; diff --git a/cmd/explaintest/r/index_merge.result b/cmd/explaintest/r/index_merge.result index 48080df7d23f2..05940cb7c027b 100644 --- a/cmd/explaintest/r/index_merge.result +++ b/cmd/explaintest/r/index_merge.result @@ -653,11 +653,11 @@ c1 c2 c3 c4 c5 explain select /*+ use_index_merge(t1) */ * from t1 where (c1 < 10 or c2 < 10) and greatest(c1, c2, c4) = 1 order by 1; id estRows task access object operator info Sort_5 4433.77 root test.t1.c1 -└─Selection_8 4433.77 root eq(greatest(test.t1.c1, test.t1.c2, test.t1.c4), 1) - └─IndexMerge_12 5542.21 root - ├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo - ├─IndexRangeScan_10(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo - └─TableRowIDScan_11(Probe) 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo +└─IndexMerge_12 4433.77 root + ├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo + ├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo + └─Selection_11(Probe) 4433.77 cop[tikv] eq(greatest(test.t1.c1, test.t1.c2, test.t1.c4), 1) + └─TableRowIDScan_10 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo select /*+ use_index_merge(t1) */ * from t1 where (c1 < 10 or c2 < 10) and greatest(c1, c2, c4) = 1 order by 1; c1 c2 c3 c4 c5 1 1 1 1 1 diff --git a/expression/distsql_builtin.go b/expression/distsql_builtin.go index 3365222e13b1a..bda1492ca3b86 100644 --- a/expression/distsql_builtin.go +++ b/expression/distsql_builtin.go @@ -436,6 +436,8 @@ func getSignatureByPB(ctx sessionctx.Context, sigCode tipb.ScalarFuncSig, tp *ti f = &builtinTruncateRealSig{base} case tipb.ScalarFuncSig_TruncateDecimal: f = &builtinTruncateDecimalSig{base} + case tipb.ScalarFuncSig_TruncateUint: + f = &builtinTruncateUintSig{base} case tipb.ScalarFuncSig_LogicalAnd: f = &builtinLogicAndSig{base} case tipb.ScalarFuncSig_LogicalOr: @@ -1006,6 +1008,8 @@ func getSignatureByPB(ctx sessionctx.Context, sigCode tipb.ScalarFuncSig, tp *ti f = &builtinLocate3ArgsSig{base} case tipb.ScalarFuncSig_Lower: f = &builtinLowerSig{base} + case tipb.ScalarFuncSig_LowerUTF8: + f = &builtinLowerUTF8Sig{base} case tipb.ScalarFuncSig_LpadUTF8: f = &builtinLpadUTF8Sig{base, maxAllowedPacket} case tipb.ScalarFuncSig_Lpad: @@ -1064,6 +1068,8 @@ func getSignatureByPB(ctx sessionctx.Context, sigCode tipb.ScalarFuncSig, tp *ti f = &builtinUnHexSig{base} case tipb.ScalarFuncSig_Upper: f = &builtinUpperSig{base} + case tipb.ScalarFuncSig_UpperUTF8: + f = &builtinUpperUTF8Sig{base} case tipb.ScalarFuncSig_ToBinary: f = &builtinInternalToBinarySig{base} case tipb.ScalarFuncSig_FromBinary: diff --git a/expression/expr_to_pb_test.go b/expression/expr_to_pb_test.go index 8e60fb5a556d7..2de10e5d19bc7 100644 --- a/expression/expr_to_pb_test.go +++ b/expression/expr_to_pb_test.go @@ -1372,6 +1372,36 @@ func TestExprPushDownToTiKV(t *testing.T) { retType: types.NewFieldType(mysql.TypeDate), args: []Expression{dateColumn, dateColumn}, }, + { + functionName: ast.Lower, + retType: types.NewFieldType(mysql.TypeString), + args: []Expression{stringColumn}, + }, + { + functionName: ast.InsertFunc, + retType: types.NewFieldType(mysql.TypeString), + args: []Expression{stringColumn, intColumn, intColumn, stringColumn}, + }, + { + functionName: ast.Greatest, + retType: types.NewFieldType(mysql.TypeInt24), + args: []Expression{intColumn, intColumn}, + }, + { + functionName: ast.Least, + retType: types.NewFieldType(mysql.TypeInt24), + args: []Expression{intColumn, intColumn}, + }, + { + functionName: ast.Upper, + retType: types.NewFieldType(mysql.TypeString), + args: []Expression{stringColumn}, + }, + { + functionName: ast.Mod, + retType: types.NewFieldType(mysql.TypeInt24), + args: []Expression{intColumn, intColumn}, + }, } for _, tc := range testcases { @@ -1380,10 +1410,6 @@ func TestExprPushDownToTiKV(t *testing.T) { exprs = append(exprs, function) } - function, err = NewFunction(mock.NewContext(), ast.Mod, types.NewFieldType(mysql.TypeInt24), intColumn, intColumn) - require.NoError(t, err) - exprs = append(exprs, function) - pushed, remained = PushDownExprs(sc, exprs, client, kv.TiKV) require.Len(t, pushed, len(exprs)) require.Len(t, remained, 0) diff --git a/expression/expression.go b/expression/expression.go index 94c903bc4994a..624431d907ead 100644 --- a/expression/expression.go +++ b/expression/expression.go @@ -950,6 +950,7 @@ func scalarExprSupportedByTiKV(sf *ScalarFunction) bool { // compare functions. ast.LT, ast.LE, ast.EQ, ast.NE, ast.GE, ast.GT, ast.NullEQ, ast.In, ast.IsNull, ast.Like, ast.IsTruthWithoutNull, ast.IsTruthWithNull, ast.IsFalsity, + ast.Greatest, ast.Least, /* ast.Interval */ // arithmetical functions. ast.PI, ast.Truncate, @@ -968,7 +969,7 @@ func scalarExprSupportedByTiKV(sf *ScalarFunction) bool { // string functions. ast.Bin, ast.Unhex, ast.Locate, ast.Ord, ast.Lpad, ast.Rpad, - ast.Trim, ast.FromBase64, ast.ToBase64, /* ast.Upper, ast.Lower, ast.InsertFunc */ + ast.Trim, ast.FromBase64, ast.ToBase64, ast.Upper, ast.Lower, ast.InsertFunc, ast.MakeSet, ast.SubstringIndex, ast.Instr, ast.Quote, ast.Oct, ast.FindInSet, ast.Repeat, ast.Length, ast.BitLength, ast.Concat, ast.ConcatWS, ast.Replace, ast.ASCII, ast.Hex, @@ -988,7 +989,7 @@ func scalarExprSupportedByTiKV(sf *ScalarFunction) bool { ast.MonthName, ast.MakeDate, ast.TimeToSec, ast.MakeTime, ast.DateFormat, ast.Hour, ast.Minute, ast.Second, ast.MicroSecond, ast.Month, - /* ast.DayName */ ast.DayOfMonth, ast.DayOfWeek, ast.DayOfYear, + ast.DayName, ast.DayOfMonth, ast.DayOfWeek, ast.DayOfYear, ast.Weekday, ast.WeekOfYear, ast.Year, ast.FromDays, ast.ToDays, ast.PeriodAdd, ast.PeriodDiff, /*ast.TimestampDiff, ast.DateAdd, ast.FromUnixTime,*/ diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index a2364b0648d27..da12b3b3f2854 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -2725,9 +2725,9 @@ func TestScalarFunctionPushDown(t *testing.T) { tk.MustQuery("explain analyze select /*+read_from_storage(tikv[t])*/ * from t where month(d);"). CheckAt([]int{0, 3, 6}, rows) - //rows[1][2] = "dayname(test.t.d)" - //tk.MustQuery("explain analyze select /*+read_from_storage(tikv[t])*/ * from t where dayname(d);"). - // CheckAt([]int{0, 3, 6}, rows) + rows[1][2] = "dayname(test.t.d)" + tk.MustQuery("explain analyze select /*+read_from_storage(tikv[t])*/ * from t where dayname(d);"). + CheckAt([]int{0, 3, 6}, rows) rows[1][2] = "dayofmonth(test.t.d)" tk.MustQuery("explain analyze select /*+read_from_storage(tikv[t])*/ * from t where dayofmonth(d);"). @@ -2852,6 +2852,26 @@ func TestScalarFunctionPushDown(t *testing.T) { rows[1][2] = "gt(test.t.d, sysdate())" tk.MustQuery("explain analyze select /*+read_from_storage(tikv[t])*/ * from t where d > sysdate()"). CheckAt([]int{0, 3, 6}, rows) + + rows[1][2] = "lower(test.t.c)" + tk.MustQuery("explain analyze select /*+read_from_storage(tikv[t])*/ * from t where lower(c)"). + CheckAt([]int{0, 3, 6}, rows) + + rows[1][2] = "upper(test.t.c)" + tk.MustQuery("explain analyze select /*+read_from_storage(tikv[t])*/ * from t where upper(c)"). + CheckAt([]int{0, 3, 6}, rows) + + rows[1][2] = "insert_func(test.t.c, 1, 1, \"c\")" + tk.MustQuery("explain analyze select /*+read_from_storage(tikv[t])*/ * from t where insert_func(c, 1, 1,'c')"). + CheckAt([]int{0, 3, 6}, rows) + + rows[1][2] = "greatest(test.t.id, 1)" + tk.MustQuery("explain analyze select /*+read_from_storage(tikv[t])*/ * from t where greatest(id, 1)"). + CheckAt([]int{0, 3, 6}, rows) + + rows[1][2] = "least(test.t.id, 1)" + tk.MustQuery("explain analyze select /*+read_from_storage(tikv[t])*/ * from t where least(id, 1)"). + CheckAt([]int{0, 3, 6}, rows) } func TestDistinctScalarFunctionPushDown(t *testing.T) {