From 4d3bd7735ab41aabeb08c14edafa8d273a1ad121 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Sat, 27 Nov 2021 17:30:15 +0800 Subject: [PATCH 1/8] expression: pushdown substring Signed-off-by: guo-shaoge --- expression/expression.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/expression/expression.go b/expression/expression.go index 6bf6320a19579..81e9cd93386d8 100644 --- a/expression/expression.go +++ b/expression/expression.go @@ -963,7 +963,7 @@ func scalarExprSupportedByTiKV(sf *ScalarFunction) bool { // string functions. ast.Length, ast.BitLength, ast.Concat, ast.ConcatWS /*ast.Locate,*/, ast.Replace, ast.ASCII, ast.Hex, - ast.Reverse, ast.LTrim, ast.RTrim /*ast.Left,*/, ast.Strcmp, ast.Space, ast.Elt, ast.Field, + ast.Reverse, ast.LTrim, ast.RTrim /*ast.Left,*/, ast.Strcmp, ast.Space, ast.Elt, ast.Field, ast.Substring, // json functions. ast.JSONType, ast.JSONExtract, ast.JSONObject, ast.JSONArray, ast.JSONMerge, ast.JSONSet, From 316c119fa08eee9a0351e791846f73b7ae744721 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Sat, 27 Nov 2021 19:01:19 +0800 Subject: [PATCH 2/8] fix case Signed-off-by: guo-shaoge --- cmd/explaintest/r/tpch.result | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/cmd/explaintest/r/tpch.result b/cmd/explaintest/r/tpch.result index 319713d946aa2..018434ce255e1 100644 --- a/cmd/explaintest/r/tpch.result +++ b/cmd/explaintest/r/tpch.result @@ -844,7 +844,7 @@ and l_shipdate >= '1996-12-01' and l_shipdate < date_add('1996-12-01', interval '1' month); id estRows task access object operator info Projection 1.00 root div(mul(100.00, Column#27), Column#28)->Column#29 -└─StreamAgg 1.00 root funcs:sum(Column#31)->Column#27, funcs:sum(Column#32)->Column#28 +└─HashAgg 1.00 root funcs:sum(Column#31)->Column#27, funcs:sum(Column#32)->Column#28 └─Projection 4121984.49 root case(like(tpch.part.p_type, PROMO%, 92), mul(tpch.lineitem.l_extendedprice, minus(1, tpch.lineitem.l_discount)), 0)->Column#31, mul(tpch.lineitem.l_extendedprice, minus(1, tpch.lineitem.l_discount))->Column#32 └─IndexJoin 4121984.49 root inner join, inner:TableReader, outer key:tpch.lineitem.l_partkey, inner key:tpch.part.p_partkey, equal cond:eq(tpch.lineitem.l_partkey, tpch.part.p_partkey) ├─TableReader(Build) 4121984.49 root data:Selection @@ -976,7 +976,7 @@ l_partkey = p_partkey ); id estRows task access object operator info Projection 1.00 root div(Column#46, 7.0)->Column#47 -└─StreamAgg 1.00 root funcs:sum(tpch.lineitem.l_extendedprice)->Column#46 +└─HashAgg 1.00 root funcs:sum(tpch.lineitem.l_extendedprice)->Column#46 └─HashJoin 293773.83 root inner join, equal:[eq(tpch.part.p_partkey, tpch.lineitem.l_partkey)], other cond:lt(tpch.lineitem.l_quantity, mul(0.2, Column#44)) ├─HashJoin(Build) 293773.83 root inner join, equal:[eq(tpch.part.p_partkey, tpch.lineitem.l_partkey)] │ ├─TableReader(Build) 9736.49 root data:Selection @@ -1094,7 +1094,7 @@ and l_shipmode in ('AIR', 'AIR REG') and l_shipinstruct = 'DELIVER IN PERSON' ); id estRows task access object operator info -StreamAgg 1.00 root funcs:sum(Column#28)->Column#27 +HashAgg 1.00 root funcs:sum(Column#28)->Column#27 └─Projection 733887.82 root mul(tpch.lineitem.l_extendedprice, minus(1, tpch.lineitem.l_discount))->Column#28 └─HashJoin 733887.82 root inner join, equal:[eq(tpch.part.p_partkey, tpch.lineitem.l_partkey)], other cond:or(and(and(eq(tpch.part.p_brand, "Brand#52"), in(tpch.part.p_container, "SM CASE", "SM BOX", "SM PACK", "SM PKG")), and(ge(tpch.lineitem.l_quantity, 4), and(le(tpch.lineitem.l_quantity, 14), le(tpch.part.p_size, 5)))), or(and(and(eq(tpch.part.p_brand, "Brand#11"), in(tpch.part.p_container, "MED BAG", "MED BOX", "MED PKG", "MED PACK")), and(ge(tpch.lineitem.l_quantity, 18), and(le(tpch.lineitem.l_quantity, 28), le(tpch.part.p_size, 10)))), and(and(eq(tpch.part.p_brand, "Brand#51"), in(tpch.part.p_container, "LG CASE", "LG BOX", "LG PACK", "LG PKG")), and(ge(tpch.lineitem.l_quantity, 29), and(le(tpch.lineitem.l_quantity, 39), le(tpch.part.p_size, 15)))))) ├─TableReader(Build) 24323.12 root data:Selection @@ -1294,14 +1294,13 @@ cntrycode order by cntrycode; id estRows task access object operator info -Sort 1.00 root Column#27 -└─Projection 1.00 root Column#27, Column#28, Column#29 - └─HashAgg 1.00 root group by:Column#33, funcs:count(1)->Column#28, funcs:sum(Column#31)->Column#29, funcs:firstrow(Column#32)->Column#27 - └─Projection 0.00 root tpch.customer.c_acctbal, substring(tpch.customer.c_phone, 1, 2)->Column#32, substring(tpch.customer.c_phone, 1, 2)->Column#33 +Sort 1.00 root Column#31 +└─Projection 1.00 root Column#31, Column#32, Column#33 + └─HashAgg 1.00 root group by:Column#37, funcs:count(1)->Column#32, funcs:sum(Column#35)->Column#33, funcs:firstrow(Column#36)->Column#31 + └─Projection 0.00 root tpch.customer.c_acctbal, substring(tpch.customer.c_phone, 1, 2)->Column#36, substring(tpch.customer.c_phone, 1, 2)->Column#37 └─HashJoin 0.00 root anti semi join, equal:[eq(tpch.customer.c_custkey, tpch.orders.o_custkey)] ├─TableReader(Build) 75000000.00 root data:TableFullScan │ └─TableFullScan 75000000.00 cop[tikv] table:orders keep order:false - └─Selection(Probe) 0.00 root in(substring(tpch.customer.c_phone, 1, 2), "20", "40", "22", "30", "39", "42", "21") - └─TableReader 0.00 root data:Selection - └─Selection 0.00 cop[tikv] gt(tpch.customer.c_acctbal, NULL) - └─TableFullScan 7500000.00 cop[tikv] table:customer keep order:false + └─TableReader(Probe) 0.00 root data:Selection + └─Selection 0.00 cop[tikv] gt(tpch.customer.c_acctbal, NULL), in(substring(tpch.customer.c_phone, 1, 2), "20", "40", "22", "30", "39", "42", "21") + └─TableFullScan 7500000.00 cop[tikv] table:customer keep order:false From 86ef769f3fa17eb6580719f533800083422fe6ef Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Sat, 27 Nov 2021 19:26:20 +0800 Subject: [PATCH 3/8] fix case Signed-off-by: guo-shaoge --- cmd/explaintest/r/tpch.result | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/explaintest/r/tpch.result b/cmd/explaintest/r/tpch.result index 018434ce255e1..b87d1ff6dd042 100644 --- a/cmd/explaintest/r/tpch.result +++ b/cmd/explaintest/r/tpch.result @@ -844,7 +844,7 @@ and l_shipdate >= '1996-12-01' and l_shipdate < date_add('1996-12-01', interval '1' month); id estRows task access object operator info Projection 1.00 root div(mul(100.00, Column#27), Column#28)->Column#29 -└─HashAgg 1.00 root funcs:sum(Column#31)->Column#27, funcs:sum(Column#32)->Column#28 +└─StreamAgg 1.00 root funcs:sum(Column#31)->Column#27, funcs:sum(Column#32)->Column#28 └─Projection 4121984.49 root case(like(tpch.part.p_type, PROMO%, 92), mul(tpch.lineitem.l_extendedprice, minus(1, tpch.lineitem.l_discount)), 0)->Column#31, mul(tpch.lineitem.l_extendedprice, minus(1, tpch.lineitem.l_discount))->Column#32 └─IndexJoin 4121984.49 root inner join, inner:TableReader, outer key:tpch.lineitem.l_partkey, inner key:tpch.part.p_partkey, equal cond:eq(tpch.lineitem.l_partkey, tpch.part.p_partkey) ├─TableReader(Build) 4121984.49 root data:Selection From 3f4bdd7feb10426b4911583c584bd1312d667f04 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Sat, 27 Nov 2021 19:46:21 +0800 Subject: [PATCH 4/8] fix case Signed-off-by: guo-shaoge --- cmd/explaintest/r/tpch.result | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/explaintest/r/tpch.result b/cmd/explaintest/r/tpch.result index b87d1ff6dd042..55e4e36ccedde 100644 --- a/cmd/explaintest/r/tpch.result +++ b/cmd/explaintest/r/tpch.result @@ -976,7 +976,7 @@ l_partkey = p_partkey ); id estRows task access object operator info Projection 1.00 root div(Column#46, 7.0)->Column#47 -└─HashAgg 1.00 root funcs:sum(tpch.lineitem.l_extendedprice)->Column#46 +└─StreamAgg 1.00 root funcs:sum(tpch.lineitem.l_extendedprice)->Column#46 └─HashJoin 293773.83 root inner join, equal:[eq(tpch.part.p_partkey, tpch.lineitem.l_partkey)], other cond:lt(tpch.lineitem.l_quantity, mul(0.2, Column#44)) ├─HashJoin(Build) 293773.83 root inner join, equal:[eq(tpch.part.p_partkey, tpch.lineitem.l_partkey)] │ ├─TableReader(Build) 9736.49 root data:Selection @@ -1094,7 +1094,7 @@ and l_shipmode in ('AIR', 'AIR REG') and l_shipinstruct = 'DELIVER IN PERSON' ); id estRows task access object operator info -HashAgg 1.00 root funcs:sum(Column#28)->Column#27 +StreamAgg 1.00 root funcs:sum(Column#28)->Column#27 └─Projection 733887.82 root mul(tpch.lineitem.l_extendedprice, minus(1, tpch.lineitem.l_discount))->Column#28 └─HashJoin 733887.82 root inner join, equal:[eq(tpch.part.p_partkey, tpch.lineitem.l_partkey)], other cond:or(and(and(eq(tpch.part.p_brand, "Brand#52"), in(tpch.part.p_container, "SM CASE", "SM BOX", "SM PACK", "SM PKG")), and(ge(tpch.lineitem.l_quantity, 4), and(le(tpch.lineitem.l_quantity, 14), le(tpch.part.p_size, 5)))), or(and(and(eq(tpch.part.p_brand, "Brand#11"), in(tpch.part.p_container, "MED BAG", "MED BOX", "MED PKG", "MED PACK")), and(ge(tpch.lineitem.l_quantity, 18), and(le(tpch.lineitem.l_quantity, 28), le(tpch.part.p_size, 10)))), and(and(eq(tpch.part.p_brand, "Brand#51"), in(tpch.part.p_container, "LG CASE", "LG BOX", "LG PACK", "LG PKG")), and(ge(tpch.lineitem.l_quantity, 29), and(le(tpch.lineitem.l_quantity, 39), le(tpch.part.p_size, 15)))))) ├─TableReader(Build) 24323.12 root data:Selection From 550e22eae81a8d1b556fba8062e5b8c107a3469f Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Wed, 29 Dec 2021 16:24:00 +0800 Subject: [PATCH 5/8] fix comment fixSigned-off-by: guo-shaoge --- expression/expr_to_pb_test.go | 23 ++++++++++++++++++++++- expression/expression.go | 4 ++-- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/expression/expr_to_pb_test.go b/expression/expr_to_pb_test.go index 30cdf5599181b..dcfd950a7cd53 100644 --- a/expression/expr_to_pb_test.go +++ b/expression/expr_to_pb_test.go @@ -1065,7 +1065,7 @@ func TestExprPushDownToTiKV(t *testing.T) { exprs := make([]Expression, 0) //jsonColumn := genColumn(mysql.TypeJSON, 1) - //intColumn := genColumn(mysql.TypeLonglong, 2) + intColumn := genColumn(mysql.TypeLonglong, 2) //realColumn := genColumn(mysql.TypeDouble, 3) //decimalColumn := genColumn(mysql.TypeNewDecimal, 4) stringColumn := genColumn(mysql.TypeString, 5) @@ -1073,6 +1073,7 @@ func TestExprPushDownToTiKV(t *testing.T) { binaryStringColumn := genColumn(mysql.TypeString, 7) binaryStringColumn.RetType.Collate = charset.CollationBin + // Test exprs that cannot be pushed. function, err := NewFunction(mock.NewContext(), ast.InetAton, types.NewFieldType(mysql.TypeString), stringColumn) require.NoError(t, err) exprs = append(exprs, function) @@ -1108,6 +1109,26 @@ func TestExprPushDownToTiKV(t *testing.T) { pushed, remained := PushDownExprs(sc, exprs, client, kv.TiKV) require.Len(t, pushed, 0) require.Len(t, remained, len(exprs)) + + // Test exprs that can be pushed. + exprs = exprs[:0] + pushed = pushed[:0] + remained = remained[:0] + + substringRelated := []string{ast.Substr, ast.Substring, ast.Mid} + for _, exprName := range substringRelated { + function, err = NewFunction(mock.NewContext(), exprName, types.NewFieldType(mysql.TypeString), stringColumn, intColumn, intColumn) + require.NoError(t, err) + exprs = append(exprs, function) + } + + function, err = NewFunction(mock.NewContext(), ast.CharLength, types.NewFieldType(mysql.TypeString), stringColumn) + require.NoError(t, err) + exprs = append(exprs, function) + + pushed, remained = PushDownExprs(sc, exprs, client, kv.TiKV) + require.Len(t, pushed, len(exprs)) + require.Len(t, remained, 0) } func TestExprOnlyPushDownToTiKV(t *testing.T) { diff --git a/expression/expression.go b/expression/expression.go index 54edff33da609..b1c5d59fb1409 100644 --- a/expression/expression.go +++ b/expression/expression.go @@ -966,8 +966,8 @@ func scalarExprSupportedByTiKV(sf *ScalarFunction) bool { // string functions. ast.Length, ast.BitLength, ast.Concat, ast.ConcatWS /*ast.Locate,*/, ast.Replace, ast.ASCII, ast.Hex, - ast.Reverse, ast.LTrim, ast.RTrim /*ast.Left,*/, ast.Strcmp, ast.Space, ast.Elt, ast.Field, ast.Substring, - InternalFuncFromBinary, InternalFuncToBinary, + ast.Reverse, ast.LTrim, ast.RTrim /*ast.Left,*/, ast.Strcmp, ast.Space, ast.Elt, ast.Field, + InternalFuncFromBinary, InternalFuncToBinary, ast.Mid, ast.Substring, ast.Substr, ast.CharLength, // json functions. ast.JSONType, ast.JSONExtract, ast.JSONObject, ast.JSONArray, ast.JSONMerge, ast.JSONSet, From 0933c569117c7ae34d4e90e943a8f207875a7432 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Wed, 29 Dec 2021 17:49:36 +0800 Subject: [PATCH 6/8] fix case Signed-off-by: guo-shaoge --- expression/integration_test.go | 32 +++++++++---------- .../core/testdata/integration_suite_out.json | 30 ++++++++--------- 2 files changed, 30 insertions(+), 32 deletions(-) diff --git a/expression/integration_test.go b/expression/integration_test.go index 95dc3157507ad..3191a837443c3 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -3472,36 +3472,36 @@ func TestExprPushdown(t *testing.T) { "(4,'511111','611',7,8,9),(5,'611111','711',8,9,10)") // case 1, index scan without double read, some filters can not be pushed to cop task - rows := tk.MustQuery("explain format = 'brief' select col2, col1 from t use index(key1) where col2 like '5%' and substr(col1, 1, 1) = '4'").Rows() + rows := tk.MustQuery("explain format = 'brief' select col2, col1 from t use index(key1) where col2 like '5%' and from_base64(to_base64(substr(col1, 1, 1))) = '4'").Rows() require.Equal(t, "root", fmt.Sprintf("%v", rows[1][2])) - require.Equal(t, "eq(substr(test.t.col1, 1, 1), \"4\")", fmt.Sprintf("%v", rows[1][4])) + require.Equal(t, "eq(from_base64(to_base64(substr(test.t.col1, 1, 1))), \"4\")", fmt.Sprintf("%v", rows[1][4])) require.Equal(t, "cop[tikv]", fmt.Sprintf("%v", rows[3][2])) require.Equal(t, "like(test.t.col2, \"5%\", 92)", fmt.Sprintf("%v", rows[3][4])) - tk.MustQuery("select col2, col1 from t use index(key1) where col2 like '5%' and substr(col1, 1, 1) = '4'").Check(testkit.Rows("511 411111")) - tk.MustQuery("select count(col2) from t use index(key1) where col2 like '5%' and substr(col1, 1, 1) = '4'").Check(testkit.Rows("1")) + tk.MustQuery("select col2, col1 from t use index(key1) where col2 like '5%' and from_base64(to_base64(substr(col1, 1, 1))) = '4'").Check(testkit.Rows("511 411111")) + tk.MustQuery("select count(col2) from t use index(key1) where col2 like '5%' and from_base64(to_base64(substr(col1, 1, 1))) = '4'").Check(testkit.Rows("1")) // case 2, index scan without double read, none of the filters can be pushed to cop task - rows = tk.MustQuery("explain format = 'brief' select col1, col2 from t use index(key2) where substr(col2, 1, 1) = '5' and substr(col1, 1, 1) = '4'").Rows() + rows = tk.MustQuery("explain format = 'brief' select col1, col2 from t use index(key2) where from_base64(to_base64(substr(col2, 1, 1))) = '5' and from_base64(to_base64(substr(col1, 1, 1))) = '4'").Rows() require.Equal(t, "root", fmt.Sprintf("%v", rows[0][2])) - require.Equal(t, "eq(substr(test.t.col1, 1, 1), \"4\"), eq(substr(test.t.col2, 1, 1), \"5\")", fmt.Sprintf("%v", rows[0][4])) - tk.MustQuery("select col1, col2 from t use index(key2) where substr(col2, 1, 1) = '5' and substr(col1, 1, 1) = '4'").Check(testkit.Rows("411111 511")) - tk.MustQuery("select count(col1) from t use index(key2) where substr(col2, 1, 1) = '5' and substr(col1, 1, 1) = '4'").Check(testkit.Rows("1")) + require.Equal(t, "eq(from_base64(to_base64(substr(test.t.col1, 1, 1))), \"4\"), eq(from_base64(to_base64(substr(test.t.col2, 1, 1))), \"5\")", fmt.Sprintf("%v", rows[0][4])) + tk.MustQuery("select col1, col2 from t use index(key2) where from_base64(to_base64(substr(col2, 1, 1))) = '5' and from_base64(to_base64(substr(col1, 1, 1))) = '4'").Check(testkit.Rows("411111 511")) + tk.MustQuery("select count(col1) from t use index(key2) where from_base64(to_base64(substr(col2, 1, 1))) = '5' and from_base64(to_base64(substr(col1, 1, 1))) = '4'").Check(testkit.Rows("1")) // case 3, index scan with double read, some filters can not be pushed to cop task - rows = tk.MustQuery("explain format = 'brief' select id from t use index(key1) where col2 like '5%' and substr(col1, 1, 1) = '4'").Rows() + rows = tk.MustQuery("explain format = 'brief' select id from t use index(key1) where col2 like '5%' and from_base64(to_base64(substr(col1, 1, 1))) = '4'").Rows() require.Equal(t, "root", fmt.Sprintf("%v", rows[1][2])) - require.Equal(t, "eq(substr(test.t.col1, 1, 1), \"4\")", fmt.Sprintf("%v", rows[1][4])) + require.Equal(t, "eq(from_base64(to_base64(substr(test.t.col1, 1, 1))), \"4\")", fmt.Sprintf("%v", rows[1][4])) require.Equal(t, "cop[tikv]", fmt.Sprintf("%v", rows[3][2])) require.Equal(t, "like(test.t.col2, \"5%\", 92)", fmt.Sprintf("%v", rows[3][4])) - tk.MustQuery("select id from t use index(key1) where col2 like '5%' and substr(col1, 1, 1) = '4'").Check(testkit.Rows("3")) - tk.MustQuery("select count(id) from t use index(key1) where col2 like '5%' and substr(col1, 1, 1) = '4'").Check(testkit.Rows("1")) + tk.MustQuery("select id from t use index(key1) where col2 like '5%' and from_base64(to_base64(substr(col1, 1, 1))) = '4'").Check(testkit.Rows("3")) + tk.MustQuery("select count(id) from t use index(key1) where col2 like '5%' and from_base64(to_base64(substr(col1, 1, 1))) = '4'").Check(testkit.Rows("1")) // case 4, index scan with double read, none of the filters can be pushed to cop task - rows = tk.MustQuery("explain format = 'brief' select id from t use index(key2) where substr(col2, 1, 1) = '5' and substr(col1, 1, 1) = '4'").Rows() + rows = tk.MustQuery("explain format = 'brief' select id from t use index(key2) where from_base64(to_base64(substr(col2, 1, 1))) = '5' and from_base64(to_base64(substr(col1, 1, 1))) = '4'").Rows() require.Equal(t, "root", fmt.Sprintf("%v", rows[1][2])) - require.Equal(t, "eq(substr(test.t.col1, 1, 1), \"4\"), eq(substr(test.t.col2, 1, 1), \"5\")", fmt.Sprintf("%v", rows[1][4])) - tk.MustQuery("select id from t use index(key2) where substr(col2, 1, 1) = '5' and substr(col1, 1, 1) = '4'").Check(testkit.Rows("3")) - tk.MustQuery("select count(id) from t use index(key2) where substr(col2, 1, 1) = '5' and substr(col1, 1, 1) = '4'").Check(testkit.Rows("1")) + require.Equal(t, "eq(from_base64(to_base64(substr(test.t.col1, 1, 1))), \"4\"), eq(from_base64(to_base64(substr(test.t.col2, 1, 1))), \"5\")", fmt.Sprintf("%v", rows[1][4])) + tk.MustQuery("select id from t use index(key2) where from_base64(to_base64(substr(col2, 1, 1))) = '5' and from_base64(to_base64(substr(col1, 1, 1))) = '4'").Check(testkit.Rows("3")) + tk.MustQuery("select count(id) from t use index(key2) where from_base64(to_base64(substr(col2, 1, 1))) = '5' and from_base64(to_base64(substr(col1, 1, 1))) = '4'").Check(testkit.Rows("1")) } func TestIssue16973(t *testing.T) { store, clean := testkit.CreateMockStore(t) diff --git a/planner/core/testdata/integration_suite_out.json b/planner/core/testdata/integration_suite_out.json index f4726f876c70d..4dd412b71c9d3 100644 --- a/planner/core/testdata/integration_suite_out.json +++ b/planner/core/testdata/integration_suite_out.json @@ -2068,14 +2068,13 @@ " ├─TableReader(Build) 10000.00 root data:TableFullScan", " │ └─TableFullScan 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo", " └─StreamAgg(Probe) 1.00 root funcs:min(test.t1.c1)->Column#8, funcs:sum(0)->Column#9, funcs:count(1)->Column#10", - " └─Selection 0.01 root substring(cast(test.t1.c3, var_string(20)), 10)", - " └─IndexMerge 0.01 root ", - " ├─Selection(Build) 1.00 cop[tikv] eq(10, test.t2.c3)", - " │ └─TableRangeScan 1.00 cop[tikv] table:t1 range:[10,10], keep order:false, stats:pseudo", - " ├─Selection(Build) 8.00 cop[tikv] eq(1, test.t2.c3)", - " │ └─IndexRangeScan 10.00 cop[tikv] table:t1, index:c2(c2) range:[1,1], keep order:false, stats:pseudo", - " └─Selection(Probe) 0.01 cop[tikv] or(and(eq(test.t1.c1, 10), eq(10, test.t2.c3)), and(eq(test.t1.c2, 1), eq(1, test.t2.c3)))", - " └─TableRowIDScan 9.00 cop[tikv] table:t1 keep order:false, stats:pseudo" + " └─IndexMerge 0.01 root ", + " ├─Selection(Build) 1.00 cop[tikv] eq(10, test.t2.c3)", + " │ └─TableRangeScan 1.00 cop[tikv] table:t1 range:[10,10], keep order:false, stats:pseudo", + " ├─Selection(Build) 8.00 cop[tikv] eq(1, test.t2.c3)", + " │ └─IndexRangeScan 10.00 cop[tikv] table:t1, index:c2(c2) range:[1,1], keep order:false, stats:pseudo", + " └─Selection(Probe) 0.01 cop[tikv] or(and(eq(test.t1.c1, 10), eq(10, test.t2.c3)), and(eq(test.t1.c2, 1), eq(1, test.t2.c3))), substring(cast(test.t1.c3, var_string(20)), 10)", + " └─TableRowIDScan 9.00 cop[tikv] table:t1 keep order:false, stats:pseudo" ], "Res": [ "1 1 1", @@ -2112,14 +2111,13 @@ " ├─TableReader(Build) 10000.00 root data:TableFullScan", " │ └─TableFullScan 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo", " └─StreamAgg(Probe) 1.00 root funcs:min(test.t1.c1)->Column#8, funcs:sum(0)->Column#9, funcs:count(1)->Column#10", - " └─Selection 3.03 root substring(cast(test.t1.c3, var_string(20)), 10)", - " └─IndexMerge 3.78 root ", - " ├─Selection(Build) 3.33 cop[tikv] eq(test.t1.c1, test.t2.c3)", - " │ └─TableRangeScan 3333.33 cop[tikv] table:t1 range:[10,+inf], keep order:false, stats:pseudo", - " ├─Selection(Build) 8.00 cop[tikv] eq(1, test.t2.c3)", - " │ └─IndexRangeScan 10.00 cop[tikv] table:t1, index:c2(c2) range:[1,1], keep order:false, stats:pseudo", - " └─Selection(Probe) 3.78 cop[tikv] or(and(ge(test.t1.c1, 10), eq(test.t1.c1, test.t2.c3)), and(eq(test.t1.c2, 1), eq(1, test.t2.c3)))", - " └─TableRowIDScan 3338.67 cop[tikv] table:t1 keep order:false, stats:pseudo" + " └─IndexMerge 3.03 root ", + " ├─Selection(Build) 3.33 cop[tikv] eq(test.t1.c1, test.t2.c3)", + " │ └─TableRangeScan 3333.33 cop[tikv] table:t1 range:[10,+inf], keep order:false, stats:pseudo", + " ├─Selection(Build) 8.00 cop[tikv] eq(1, test.t2.c3)", + " │ └─IndexRangeScan 10.00 cop[tikv] table:t1, index:c2(c2) range:[1,1], keep order:false, stats:pseudo", + " └─Selection(Probe) 3.03 cop[tikv] or(and(ge(test.t1.c1, 10), eq(test.t1.c1, test.t2.c3)), and(eq(test.t1.c2, 1), eq(1, test.t2.c3))), substring(cast(test.t1.c3, var_string(20)), 10)", + " └─TableRowIDScan 3338.67 cop[tikv] table:t1 keep order:false, stats:pseudo" ], "Res": [ "1 1 1", From 0a51af1b07e67c1db93c02c892fe920574e6d871 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Wed, 29 Dec 2021 17:56:44 +0800 Subject: [PATCH 7/8] fix fmt Signed-off-by: guo-shaoge --- expression/expr_to_pb_test.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/expression/expr_to_pb_test.go b/expression/expr_to_pb_test.go index 17bf9a4419bd4..0256d696c8cc6 100644 --- a/expression/expr_to_pb_test.go +++ b/expression/expr_to_pb_test.go @@ -1076,7 +1076,7 @@ func TestExprPushDownToTiKV(t *testing.T) { binaryStringColumn := genColumn(mysql.TypeString, 7) binaryStringColumn.RetType.Collate = charset.CollationBin - // Test exprs that cannot be pushed. + // Test exprs that cannot be pushed. function, err := NewFunction(mock.NewContext(), ast.InetAton, types.NewFieldType(mysql.TypeString), stringColumn) require.NoError(t, err) exprs = append(exprs, function) @@ -1113,23 +1113,23 @@ func TestExprPushDownToTiKV(t *testing.T) { require.Len(t, pushed, 0) require.Len(t, remained, len(exprs)) - // Test exprs that can be pushed. - exprs = exprs[:0] - pushed = pushed[:0] - remained = remained[:0] + // Test exprs that can be pushed. + exprs = exprs[:0] + pushed = pushed[:0] + remained = remained[:0] - substringRelated := []string{ast.Substr, ast.Substring, ast.Mid} - for _, exprName := range substringRelated { - function, err = NewFunction(mock.NewContext(), exprName, types.NewFieldType(mysql.TypeString), stringColumn, intColumn, intColumn) - require.NoError(t, err) - exprs = append(exprs, function) - } + substringRelated := []string{ast.Substr, ast.Substring, ast.Mid} + for _, exprName := range substringRelated { + function, err = NewFunction(mock.NewContext(), exprName, types.NewFieldType(mysql.TypeString), stringColumn, intColumn, intColumn) + require.NoError(t, err) + exprs = append(exprs, function) + } - function, err = NewFunction(mock.NewContext(), ast.CharLength, types.NewFieldType(mysql.TypeString), stringColumn) - require.NoError(t, err) - exprs = append(exprs, function) + function, err = NewFunction(mock.NewContext(), ast.CharLength, types.NewFieldType(mysql.TypeString), stringColumn) + require.NoError(t, err) + exprs = append(exprs, function) - pushed, remained = PushDownExprs(sc, exprs, client, kv.TiKV) + pushed, remained = PushDownExprs(sc, exprs, client, kv.TiKV) require.Len(t, pushed, len(exprs)) require.Len(t, remained, 0) } From 82b1c7ad8c5d1d1c9f61a99c2743ace351dce893 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Wed, 29 Dec 2021 19:43:02 +0800 Subject: [PATCH 8/8] fix case Signed-off-by: guo-shaoge --- cmd/explaintest/r/index_merge.result | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/explaintest/r/index_merge.result b/cmd/explaintest/r/index_merge.result index f4d6571fd8fe3..ddce511db13c8 100644 --- a/cmd/explaintest/r/index_merge.result +++ b/cmd/explaintest/r/index_merge.result @@ -722,11 +722,11 @@ c1 c2 c3 c4 c5 explain select /*+ use_index_merge(t1) */ * from t1 where (c1 < 10 or c2 < 10) and substring(c3, 1, 1) = '1' order by 1; id estRows task access object operator info Sort_5 4433.77 root test.t1.c1 -└─Selection_12 4433.77 root eq(substring(cast(test.t1.c3, var_string(20)), 1, 1), "1") - └─IndexMerge_11 5542.21 root - ├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo - ├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo - └─TableRowIDScan_10(Probe) 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo +└─IndexMerge_12 4433.77 root + ├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo + ├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo + └─Selection_11(Probe) 4433.77 cop[tikv] eq(substring(cast(test.t1.c3, var_string(20)), 1, 1), "1") + └─TableRowIDScan_10 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo select /*+ use_index_merge(t1) */ * from t1 where (c1 < 10 or c2 < 10) and substring(c3, 1, 1) = '1' order by 1; c1 c2 c3 c4 c5 1 1 1 1 1