planner: update some UTs to use cost model2

executor/executor_test.go

@@ -3972,12 +3972,13 @@ func TestApplyCache(t *testing.T) {
 	tk := testkit.NewTestKit(t, store)
 
 	tk.MustExec("use test;")
+	tk.MustExec("set tidb_cost_model_version=2")
 	tk.MustExec("drop table if exists t;")
 	tk.MustExec("create table t(a int);")
 	tk.MustExec("insert into t values (1),(1),(1),(1),(1),(1),(1),(1),(1);")
 	tk.MustExec("analyze table t;")
 	result := tk.MustQuery("explain analyze SELECT count(1) FROM (SELECT (SELECT min(a) FROM t as t2 WHERE t2.a > t1.a) AS a from t as t1) t;")
-	require.Equal(t, "└─Apply_39", result.Rows()[1][0])
+	require.Contains(t, result.Rows()[1][0], "Apply")
 	var (
 		ind  int
 		flag bool
@@ -3997,7 +3998,7 @@ func TestApplyCache(t *testing.T) {
 	tk.MustExec("insert into t values (1),(2),(3),(4),(5),(6),(7),(8),(9);")
 	tk.MustExec("analyze table t;")
 	result = tk.MustQuery("explain analyze SELECT count(1) FROM (SELECT (SELECT min(a) FROM t as t2 WHERE t2.a > t1.a) AS a from t as t1) t;")
-	require.Equal(t, "└─Apply_39", result.Rows()[1][0])
+	require.Contains(t, result.Rows()[1][0], "Apply")
 	flag = false
 	value = (result.Rows()[1][5]).(string)
 	for ind = 0; ind < len(value)-5; ind++ {

executor/join_test.go

@@ -198,9 +198,9 @@ func TestJoin2(t *testing.T) {
 	tk.MustQuery("select /*+ INL_JOIN(t, t1) */ t1.b from t1 join t on t.b=t1.b").Check(testkit.Rows("2", "3"))
 	tk.MustQuery("select /*+ INL_HASH_JOIN(t, t1) */ t1.b from t1 join t on t.b=t1.b").Sort().Check(testkit.Rows("2", "3"))
 	tk.MustQuery("select /*+ INL_MERGE_JOIN(t, t1) */ t1.b from t1 join t on t.b=t1.b").Check(testkit.Rows("2", "3"))
-	tk.MustQuery("select /*+ INL_JOIN(t1) */ * from t right outer join t1 on t.a=t1.a").Check(testkit.Rows("1 1 1 2", "1 1 1 3", "1 1 1 4", "3 3 3 4", "<nil> <nil> 4 5"))
-	tk.MustQuery("select /*+ INL_HASH_JOIN(t1) */ * from t right outer join t1 on t.a=t1.a").Check(testkit.Rows("1 1 1 2", "1 1 1 3", "1 1 1 4", "3 3 3 4", "<nil> <nil> 4 5"))
-	tk.MustQuery("select /*+ INL_MERGE_JOIN(t1) */ * from t right outer join t1 on t.a=t1.a").Check(testkit.Rows("1 1 1 2", "1 1 1 3", "1 1 1 4", "3 3 3 4", "<nil> <nil> 4 5"))
+	tk.MustQuery("select /*+ INL_JOIN(t1) */ * from t right outer join t1 on t.a=t1.a").Sort().Check(testkit.Rows("1 1 1 2", "1 1 1 3", "1 1 1 4", "3 3 3 4", "<nil> <nil> 4 5"))
+	tk.MustQuery("select /*+ INL_HASH_JOIN(t1) */ * from t right outer join t1 on t.a=t1.a").Sort().Check(testkit.Rows("1 1 1 2", "1 1 1 3", "1 1 1 4", "3 3 3 4", "<nil> <nil> 4 5"))
+	tk.MustQuery("select /*+ INL_MERGE_JOIN(t1) */ * from t right outer join t1 on t.a=t1.a").Sort().Check(testkit.Rows("1 1 1 2", "1 1 1 3", "1 1 1 4", "3 3 3 4", "<nil> <nil> 4 5"))
 	tk.MustQuery("select /*+ INL_JOIN(t) */ avg(t.b) from t right outer join t1 on t.a=t1.a").Check(testkit.Rows("1.5000"))
 	tk.MustQuery("select /*+ INL_HASH_JOIN(t) */ avg(t.b) from t right outer join t1 on t.a=t1.a").Check(testkit.Rows("1.5000"))
 	tk.MustQuery("select /*+ INL_MERGE_JOIN(t) */ avg(t.b) from t right outer join t1 on t.a=t1.a").Check(testkit.Rows("1.5000"))
@@ -1222,6 +1222,7 @@ func TestIndexLookupJoin(t *testing.T) {
 	store := testkit.CreateMockStore(t)
 	tk := testkit.NewTestKit(t, store)
 	tk.MustExec("use test")
+	tk.MustExec("set tidb_cost_model_version=2")
 	tk.MustExec("set @@tidb_init_chunk_size=2")
 	tk.MustExec("DROP TABLE IF EXISTS t")
 	tk.MustExec("CREATE TABLE `t` (`a` int, pk integer auto_increment,`b` char (20),primary key (pk))")
@@ -1356,7 +1357,7 @@ func TestIndexLookupJoin(t *testing.T) {
 	tk.MustExec("analyze table s;")
 
 	tk.MustQuery("desc format = 'brief' select /*+ TIDB_INLJ(s) */ count(*) from t join s use index(idx) on s.a = t.a and s.b < t.b").Check(testkit.Rows(
-		"HashAgg 1.00 root  funcs:count(1)->Column#6",
+		"StreamAgg 1.00 root  funcs:count(1)->Column#6",
 		"└─IndexJoin 64.00 root  inner join, inner:IndexReader, outer key:test.t.a, inner key:test.s.a, equal cond:eq(test.t.a, test.s.a), other cond:lt(test.s.b, test.t.b)",
 		"  ├─TableReader(Build) 64.00 root  data:Selection",
 		"  │ └─Selection 64.00 cop[tikv]  not(isnull(test.t.b))",
@@ -1369,7 +1370,7 @@ func TestIndexLookupJoin(t *testing.T) {
 	tk.MustQuery("select /*+ TIDB_INLJ(s) */ count(*) from t join s use index(idx) on s.a = t.a and s.b < t.b").Check(testkit.Rows("64"))
 
 	tk.MustQuery("desc format = 'brief' select /*+ INL_MERGE_JOIN(s) */ count(*) from t join s use index(idx) on s.a = t.a and s.b < t.b").Check(testkit.Rows(
-		"HashAgg 1.00 root  funcs:count(1)->Column#6",
+		"StreamAgg 1.00 root  funcs:count(1)->Column#6",
 		"└─IndexMergeJoin 64.00 root  inner join, inner:IndexReader, outer key:test.t.a, inner key:test.s.a, other cond:lt(test.s.b, test.t.b)",
 		"  ├─TableReader(Build) 64.00 root  data:Selection",
 		"  │ └─Selection 64.00 cop[tikv]  not(isnull(test.t.b))",
@@ -1383,7 +1384,7 @@ func TestIndexLookupJoin(t *testing.T) {
 	tk.MustQuery("select /*+ INL_MERGE_JOIN(s) */ count(*) from t join s use index(idx) on s.a = t.a and s.b < t.b").Check(testkit.Rows("64"))
 
 	tk.MustQuery("desc format = 'brief' select /*+ INL_HASH_JOIN(s) */ count(*) from t join s use index(idx) on s.a = t.a and s.b < t.b").Check(testkit.Rows(
-		"HashAgg 1.00 root  funcs:count(1)->Column#6",
+		"StreamAgg 1.00 root  funcs:count(1)->Column#6",
 		"└─IndexHashJoin 64.00 root  inner join, inner:IndexReader, outer key:test.t.a, inner key:test.s.a, equal cond:eq(test.t.a, test.s.a), other cond:lt(test.s.b, test.t.b)",
 		"  ├─TableReader(Build) 64.00 root  data:Selection",
 		"  │ └─Selection 64.00 cop[tikv]  not(isnull(test.t.b))",

expression/constant_propagation_test.go

@@ -27,6 +27,7 @@ func TestOuterJoinPropConst(t *testing.T) {
 
 	tk := testkit.NewTestKit(t, store)
 	tk.MustExec("use test")
+	tk.MustExec("set tidb_cost_model_version=2")
 	tk.MustExec("drop table if exists t1, t2;")
 	tk.MustExec("create table t1(id bigint primary key, a int, b int);")
 	tk.MustExec("create table t2(id bigint primary key, a int, b int);")

expression/testdata/expression_suite_out.json

@@ -65,11 +65,11 @@
         "Result": [
           "HashJoin 4166.67 root  right outer join, equal:[eq(test.t1.a, test.t2.a)]",
           "├─TableReader(Build) 3333.33 root  data:Selection",
-          "│ └─Selection 3333.33 cop[tikv]  gt(test.t1.a, 1), not(isnull(test.t1.a))",
-          "│   └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo",
+          "│ └─Selection 3333.33 cop[tikv]  gt(test.t2.a, 1)",
+          "│   └─TableFullScan 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo",
           "└─TableReader(Probe) 3333.33 root  data:Selection",
-          "  └─Selection 3333.33 cop[tikv]  gt(test.t2.a, 1)",
-          "    └─TableFullScan 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo"
+          "  └─Selection 3333.33 cop[tikv]  gt(test.t1.a, 1), not(isnull(test.t1.a))",
+          "    └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo"
         ]
       },
       {

planner/core/cbo_test.go

@@ -167,6 +167,7 @@ func TestEstimation(t *testing.T) {
 	statistics.RatioOfPseudoEstimate.Store(10.0)
 	defer statistics.RatioOfPseudoEstimate.Store(0.7)
 	testKit.MustExec("use test")
+	testKit.MustExec("set tidb_cost_model_version=2")
 	testKit.MustExec("create table t (a int)")
 	testKit.MustExec("insert into t values (1), (2), (3), (4), (5), (6), (7), (8), (9), (10)")
 	testKit.MustExec("insert into t select * from t")
@@ -211,6 +212,7 @@ func constructInsertSQL(i, n int) string {
 func TestIndexRead(t *testing.T) {
 	store, dom := testkit.CreateMockStoreAndDomain(t)
 	testKit := testkit.NewTestKit(t, store)
+	testKit.MustExec("set tidb_cost_model_version=2")
 	testKit.MustExec("set @@session.tidb_executor_concurrency = 4;")
 	testKit.MustExec("set @@session.tidb_hash_join_concurrency = 5;")
 	testKit.MustExec("set @@session.tidb_distsql_scan_concurrency = 15;")
@@ -662,6 +664,7 @@ func TestLimitCrossEstimation(t *testing.T) {
 	store := testkit.CreateMockStore(t)
 	tk := testkit.NewTestKit(t, store)
 
+	tk.MustExec("set tidb_cost_model_version=2")
 	tk.MustExec("set @@session.tidb_executor_concurrency = 4;")
 	tk.MustExec("set @@session.tidb_hash_join_concurrency = 5;")
 	tk.MustExec("set @@session.tidb_distsql_scan_concurrency = 15;")
@@ -796,6 +799,7 @@ func TestLimitIndexEstimation(t *testing.T) {
 	tk := testkit.NewTestKit(t, store)
 
 	tk.MustExec("use test")
+	tk.MustExec("set tidb_cost_model_version=2")
 	tk.MustExec("drop table if exists t")
 	tk.MustExec("create table t(a int, b int, key idx_a(a), key idx_b(b))")
 	tk.MustExec("set session tidb_enable_extended_stats = on")

planner/core/integration_test.go

@@ -68,7 +68,7 @@ func TestShowSubquery(t *testing.T) {
 	))
 	tk.MustQuery("show columns from t where field in (select 'b') and false").Check(testkit.Rows())
 	tk.MustExec("insert into t values('c', 0, 0)")
-	tk.MustQuery("show columns from t where field < all (select a from t)").Check(testkit.Rows(
+	tk.MustQuery("show columns from t where field < all (select a from t)").Sort().Check(testkit.Rows(
 		"a varchar(10) YES  <nil> ",
 		"b int(11) YES  <nil> ",
 	))
@@ -471,6 +471,7 @@ func TestVerboseExplain(t *testing.T) {
 	store := testkit.CreateMockStore(t)
 	tk := testkit.NewTestKit(t, store)
 	tk.MustExec("use test")
+	tk.MustExec("set tidb_cost_model_version=2")
 	tk.MustExec(`set tidb_opt_limit_push_down_threshold=0`)
 	tk.MustExec("drop table if exists t1, t2, t3")
 	tk.MustExec("create table t1(a int, b int)")

planner/core/plan_cost_ver2.go

@@ -425,6 +425,10 @@ func (p *PhysicalHashAgg) getPlanCostVer2(taskType property.TaskType, option *Pl
 	memFactor := getTaskMemFactorVer2(p, taskType)
 	concurrency := float64(p.ctx.GetSessionVars().HashAggFinalConcurrency())
 
+	if inputRows < 2000 { // prefer to use StreamAgg if no much data to process
+		inputRows = 2000
+	}
+
 	aggCost := aggCostVer2(option, inputRows, p.AggFuncs, cpuFactor)
 	groupCost := groupCostVer2(option, inputRows, p.GroupByItems, cpuFactor)
 	hashBuildCost := hashBuildCostVer2(option, outputRows, outputRowSize, p.GroupByItems, cpuFactor, memFactor)

planner/core/testdata/analyze_suite_out.json

@@ -45,10 +45,9 @@
       {
         "SQL": "explain format = 'brief' select count(*) from t group by a",
         "Plan": [
-          "HashAgg 2.00 root  group by:test.t.a, funcs:count(Column#4)->Column#3",
-          "└─TableReader 2.00 root  data:HashAgg",
-          "  └─HashAgg 2.00 cop[tikv]  group by:test.t.a, funcs:count(1)->Column#4",
-          "    └─TableFullScan 8.00 cop[tikv] table:t keep order:false"
+          "HashAgg 2.00 root  group by:test.t.a, funcs:count(1)->Column#3",
+          "└─TableReader 8.00 root  data:TableFullScan",
+          "  └─TableFullScan 8.00 cop[tikv] table:t keep order:false"
         ]
       }
     ]
@@ -160,13 +159,13 @@
         ],
         "Plan": [
           "Limit 1.00 root  offset:0, count:1",
-          "└─IndexJoin 1.00 root  left outer semi join, inner:IndexReader, outer key:test.t.a, inner key:test.t.b, equal cond:eq(test.t.a, test.t.b)",
-          "  ├─TopN(Build) 1.00 root  test.t.a, offset:0, count:1",
-          "  │ └─IndexReader 1.00 root  index:TopN",
-          "  │   └─TopN 1.00 cop[tikv]  test.t.a, offset:0, count:1",
-          "  │     └─IndexRangeScan 6.00 cop[tikv] table:t1, index:idx_bc(b, c) range:[-inf,6], keep order:false",
-          "  └─IndexReader(Probe) 1.04 root  index:IndexRangeScan",
-          "    └─IndexRangeScan 1.04 cop[tikv] table:t2, index:idx_bc(b, c) range: decided by [eq(test.t.b, test.t.a)], keep order:false"
+          "└─MergeJoin 1.00 root  left outer semi join, left key:test.t.a, right key:test.t.b",
+          "  ├─IndexReader(Build) 25.00 root  index:IndexFullScan",
+          "  │ └─IndexFullScan 25.00 cop[tikv] table:t2, index:idx_bc(b, c) keep order:true",
+          "  └─TopN(Probe) 1.00 root  test.t.a, offset:0, count:1",
+          "    └─IndexReader 1.00 root  index:TopN",
+          "      └─TopN 1.00 cop[tikv]  test.t.a, offset:0, count:1",
+          "        └─IndexRangeScan 6.00 cop[tikv] table:t1, index:idx_bc(b, c) range:[-inf,6], keep order:false"
         ]
       },
       {
@@ -433,13 +432,13 @@
   {
     "Name": "TestIndexRead",
     "Cases": [
-      "IndexReader(Index(t.e)[[NULL,+inf]])->HashAgg",
+      "IndexReader(Index(t.e)[[NULL,+inf]]->StreamAgg)->StreamAgg",
       "IndexReader(Index(t.e)[[-inf,10]]->StreamAgg)->StreamAgg",
       "IndexReader(Index(t.e)[[-inf,50]]->StreamAgg)->StreamAgg",
-      "IndexReader(Index(t.b_c)[[NULL,+inf]]->Sel([gt(test.t.c, 1)])->HashAgg)->HashAgg",
+      "IndexReader(Index(t.b_c)[[NULL,+inf]]->Sel([gt(test.t.c, 1)])->StreamAgg)->StreamAgg",
       "IndexLookUp(Index(t.e)[[1,1]], Table(t))->HashAgg",
       "TableReader(Table(t)->Sel([gt(test.t.e, 1)])->HashAgg)->HashAgg",
-      "IndexLookUp(Index(t.b)[[-inf,20]], Table(t)->HashAgg)->HashAgg",
+      "TableReader(Table(t)->Sel([le(test.t.b, 20)])->StreamAgg)->StreamAgg",
       "TableReader(Table(t)->Sel([le(test.t.b, 30)])->StreamAgg)->StreamAgg",
       "TableReader(Table(t)->Sel([le(test.t.b, 40)])->StreamAgg)->StreamAgg",
       "TableReader(Table(t)->Sel([le(test.t.b, 50)])->StreamAgg)->StreamAgg",
@@ -449,7 +448,7 @@
       "TableReader(Table(t)->Sel([le(test.t.b, 10000000000)]))",
       "TableReader(Table(t)->Sel([le(test.t.b, 50)]))",
       "TableReader(Table(t)->Sel([le(test.t.b, 100)])->Limit)->Limit",
-      "IndexLookUp(Index(t.b)[[-inf,1]]->TopN([test.t.a],0,10), Table(t))->TopN([test.t.a],0,10)",
+      "TableReader(Table(t)->Sel([le(test.t.b, 1)])->Limit)->Limit",
       "IndexLookUp(Index(t.b)[[1,1]], Table(t))",
       "IndexLookUp(Index(t.d)[[-inf,1991-09-05 00:00:00)], Table(t))",
       "IndexLookUp(Index(t.ts)[[-inf,1991-09-05 00:00:00)], Table(t))",
@@ -501,10 +500,10 @@
         "SQL": "explain format = 'brief' select * from t where a <= 10000 order by b limit 1",
         "Plan": [
           "TopN 1.00 root  test.t.b, offset:0, count:1",
-          "└─IndexLookUp 1.00 root  ",
-          "  ├─IndexRangeScan(Build) 10000.00 cop[tikv] table:t, index:idx_a(a) range:[-inf,10000], keep order:false",
-          "  └─TopN(Probe) 1.00 cop[tikv]  test.t.b, offset:0, count:1",
-          "    └─TableRowIDScan 10000.00 cop[tikv] table:t keep order:false"
+          "└─TableReader 1.00 root  data:TopN",
+          "  └─TopN 1.00 cop[tikv]  test.t.b, offset:0, count:1",
+          "    └─Selection 10000.00 cop[tikv]  le(test.t.a, 10000)",
+          "      └─TableFullScan 1000000.00 cop[tikv] table:t keep order:false"
         ]
       },
       {

planner/core/testdata/integration_suite_in.json

@@ -468,10 +468,10 @@
       "explain format = 'verbose' select /*+ use_index(t3, c) */ count(a) from t3 where b = 0",
       "explain format = 'verbose' select count(*) from t2 where a = 0",
       "explain format = 'verbose' select count(*) from t3 t join t3 on t.a = t3.b",
-      "explain format = 'verbose' select count(*) from t1 join t2 on t1.a = t2.a",
-      "explain format = 'verbose' select count(*) from t1 join t2 on t1.a = t2.a join t3 on t1.b = t3.b",
-      "explain format = 'verbose' select (2) in (select count(*) from t1) from (select t.b < (select t.b from t2 limit 1 )  from t3 t) t",
-      "explain format = 'verbose' select /*+ merge_join(t1) */ count(*) from t1 join t2 on t1.a = t2.a"
+      "explain format = 'verbose' select /*+ read_from_storage(tiflash[t1, t2]) */ count(*) from t1 join t2 on t1.a = t2.a",
+      "explain format = 'verbose' select /*+ read_from_storage(tiflash[t1, t2]) */ count(*) from t1 join t2 on t1.a = t2.a join t3 on t1.b = t3.b",
+      "explain format = 'verbose' select (2) in (select /*+ read_from_storage(tiflash[t1]) */ count(*) from t1) from (select t.b < (select /*+ read_from_storage(tiflash[t2]) */ t.b from t2 limit 1 )  from t3 t) t",
+      "explain format = 'verbose' select /*+ merge_join(t1), read_from_storage(tiflash[t1, t2]) */ count(*) from t1 join t2 on t1.a = t2.a"
     ]
 
   },