planner: refine prefer-range-scan behavior

cmd/explaintest/r/explain_complex_stats.result

@@ -131,18 +131,18 @@ Projection	424.00	root		test.st.id, test.dd.id, test.st.aid, test.st.cm, test.dd
     └─HashJoin	424.00	root		inner join, equal:[eq(test.st.aid, test.dd.aid) eq(test.st.ip, test.dd.ip)], other cond:gt(test.dd.t, test.st.t)
       ├─TableReader(Build)	424.00	root		data:Selection
       │ └─Selection	424.00	cop[tikv]		eq(test.st.bm, 0), eq(test.st.pt, "android"), gt(test.st.t, 1478143908), not(isnull(test.st.ip))
-      │   └─TableRangeScan	1999.00	cop[tikv]	table:gad	range:[0,+inf], keep order:false
+      │   └─TableFullScan	1999.00	cop[tikv]	table:gad	keep order:false
       └─TableReader(Probe)	450.56	root		data:Selection
         └─Selection	450.56	cop[tikv]		eq(test.dd.bm, 0), eq(test.dd.pt, "android"), gt(test.dd.t, 1478143908), not(isnull(test.dd.ip)), not(isnull(test.dd.t))
-          └─TableRangeScan	2000.00	cop[tikv]	table:dd	range:[0,+inf], keep order:false
+          └─TableFullScan	2000.00	cop[tikv]	table:dd	keep order:false
 explain format = 'brief' select gad.id as gid,sdk.id as sid,gad.aid as aid,gad.cm as cm,sdk.dic as dic,sdk.ip as ip, sdk.t as t, gad.p1 as p1, gad.p2 as p2, gad.p3 as p3, gad.p4 as p4, gad.p5 as p5, gad.p6_md5 as p6, gad.p7_md5 as p7, gad.ext as ext from st gad join dd sdk on gad.aid = sdk.aid and gad.dic = sdk.mac and gad.t < sdk.t where gad.t > 1477971479 and gad.bm = 0 and gad.pt = 'ios' and gad.dit = 'mac' and sdk.t > 1477971479 and sdk.bm = 0 and sdk.pt = 'ios' limit 3000;
 id	estRows	task	access object	operator info
 Projection	170.34	root		test.st.id, test.dd.id, test.st.aid, test.st.cm, test.dd.dic, test.dd.ip, test.dd.t, test.st.p1, test.st.p2, test.st.p3, test.st.p4, test.st.p5, test.st.p6_md5, test.st.p7_md5, test.st.ext
 └─Limit	170.34	root		offset:0, count:3000
   └─IndexJoin	170.34	root		inner join, inner:IndexLookUp, outer key:test.st.aid, inner key:test.dd.aid, equal cond:eq(test.st.aid, test.dd.aid), eq(test.st.dic, test.dd.mac), other cond:lt(test.st.t, test.dd.t)
     ├─TableReader(Build)	170.34	root		data:Selection
     │ └─Selection	170.34	cop[tikv]		eq(test.st.bm, 0), eq(test.st.dit, "mac"), eq(test.st.pt, "ios"), gt(test.st.t, 1477971479), not(isnull(test.st.dic))
-    │   └─TableRangeScan	1999.00	cop[tikv]	table:gad	range:[0,+inf], keep order:false
+    │   └─TableFullScan	1999.00	cop[tikv]	table:gad	keep order:false
     └─IndexLookUp(Probe)	1.00	root		
       ├─IndexRangeScan(Build)	3.93	cop[tikv]	table:sdk, index:aid(aid, dic)	range: decided by [eq(test.dd.aid, test.st.aid)], keep order:false
       └─Selection(Probe)	1.00	cop[tikv]		eq(test.dd.bm, 0), eq(test.dd.pt, "ios"), gt(test.dd.t, 1477971479), not(isnull(test.dd.mac)), not(isnull(test.dd.t))
@@ -162,7 +162,7 @@ Projection	428.32	root		test.dt.id, test.dt.aid, test.dt.pt, test.dt.dic, test.d
   └─IndexJoin	428.32	root		inner join, inner:IndexLookUp, outer key:test.dt.aid, test.dt.dic, inner key:test.rr.aid, test.rr.dic, equal cond:eq(test.dt.aid, test.rr.aid), eq(test.dt.dic, test.rr.dic)
     ├─TableReader(Build)	428.32	root		data:Selection
     │ └─Selection	428.32	cop[tikv]		eq(test.dt.bm, 0), eq(test.dt.pt, "ios"), gt(test.dt.t, 1478185592), not(isnull(test.dt.dic))
-    │   └─TableRangeScan	2000.00	cop[tikv]	table:dt	range:[0,+inf], keep order:false
+    │   └─TableFullScan	2000.00	cop[tikv]	table:dt	keep order:false
     └─IndexLookUp(Probe)	1.00	root		
       ├─IndexRangeScan(Build)	1.00	cop[tikv]	table:rr, index:PRIMARY(aid, dic)	range: decided by [eq(test.rr.aid, test.dt.aid) eq(test.rr.dic, test.dt.dic)], keep order:false
       └─Selection(Probe)	1.00	cop[tikv]		eq(test.rr.pt, "ios"), gt(test.rr.t, 1478185592)

cmd/explaintest/r/select.result

@@ -478,8 +478,8 @@ id	estRows	task	access object	operator info
 Projection	10000.00	root		minus(Column#5, test.t.x)->Column#7
 └─Window	10000.00	root		row_number()->Column#5 over(partition by test.t.i rows between current row and current row)
   └─Sort	10000.00	root		test.t.i
-    └─TableReader	10000.00	root		data:TableRangeScan
-      └─TableRangeScan	10000.00	cop[tikv]	table:t	range:[0,+inf], keep order:false, stats:pseudo
+    └─TableReader	10000.00	root		data:TableFullScan
+      └─TableFullScan	10000.00	cop[tikv]	table:t	keep order:false, stats:pseudo
 create table precise_types (
 a BIGINT UNSIGNED NOT NULL,
 b BIGINT NOT NULL,

executor/executor_test.go

@@ -5623,7 +5623,7 @@ func (s *testSerialSuite2) TestUnsignedFeedback(c *C) {
 	tk.MustQuery("select count(distinct b) from t").Check(testkit.Rows("2"))
 	result := tk.MustQuery("explain analyze select count(distinct b) from t")
 	c.Assert(result.Rows()[2][4], Equals, "table:t")
-	c.Assert(result.Rows()[2][6], Equals, "range:[0,+inf], keep order:false")
+	c.Assert(result.Rows()[2][6], Equals, "keep order:false")
 }
 
 func (s *testSerialSuite2) TestIssue23567(c *C) {

executor/set_test.go

@@ -540,6 +540,16 @@ func (s *testSerialSuite1) TestSetVar(c *C) {
 	tk.MustExec(`set tidb_opt_limit_push_down_threshold = 20`)
 	tk.MustQuery(`select @@global.tidb_opt_limit_push_down_threshold`).Check(testkit.Rows("100"))
 	tk.MustQuery(`select @@tidb_opt_limit_push_down_threshold`).Check(testkit.Rows("20"))
+
+	tk.MustQuery("select @@tidb_opt_prefer_range_scan").Check(testkit.Rows("0"))
+	tk.MustExec("set global tidb_opt_prefer_range_scan = 1")
+	tk.MustQuery("select @@global.tidb_opt_prefer_range_scan").Check(testkit.Rows("1"))
+	tk.MustExec("set global tidb_opt_prefer_range_scan = 0")
+	tk.MustQuery("select @@global.tidb_opt_prefer_range_scan").Check(testkit.Rows("0"))
+	tk.MustExec("set session tidb_opt_prefer_range_scan = 1")
+	tk.MustQuery("select @@session.tidb_opt_prefer_range_scan").Check(testkit.Rows("1"))
+	tk.MustExec("set session tidb_opt_prefer_range_scan = 0")
+	tk.MustQuery("select @@session.tidb_opt_prefer_range_scan").Check(testkit.Rows("0"))
 }
 
 func (s *testSuite5) TestTruncateIncorrectIntSessionVar(c *C) {

planner/core/explain.go

@@ -185,7 +185,7 @@ func (p *PhysicalIndexScan) isFullScan() bool {
 		return false
 	}
 	for _, ran := range p.Ranges {
-		if !ran.IsFullRange() {
+		if !ran.IsFullRange(false) {
 			return false
 		}
 	}
@@ -322,8 +322,14 @@ func (p *PhysicalTableScan) isFullScan() bool {
 	if len(p.rangeDecidedBy) > 0 || p.haveCorCol() {
 		return false
 	}
+	var unsignedIntHandle bool
+	if p.Table.PKIsHandle {
+		if pkColInfo := p.Table.GetPkColInfo(); pkColInfo != nil {
+			unsignedIntHandle = mysql.HasUnsignedFlag(pkColInfo.Flag)
+		}
+	}
 	for _, ran := range p.Ranges {
-		if !ran.IsFullRange() {
+		if !ran.IsFullRange(unsignedIntHandle) {
 			return false
 		}
 	}

planner/core/find_best_task.go

@@ -625,14 +625,28 @@ func (ds *DataSource) skylinePruning(prop *property.PhysicalProperty) []*candida
 	}
 
 	if ds.ctx.GetSessionVars().GetAllowPreferRangeScan() && len(candidates) > 1 {
-		// remove the table/index full scan path
-		for i, c := range candidates {
-			for _, ran := range c.path.Ranges {
-				if ran.IsFullRange() {
-					candidates = append(candidates[:i], candidates[i+1:]...)
-					return candidates
+		// If a candidate path is TiFlash-path or forced-path, we just keep them. For other candidate paths, if there exists
+		// any range scan path, we remove full scan paths and keep range scan paths.
+		preferredPaths := make([]*candidatePath, 0, len(candidates))
+		var hasRangeScanPath bool
+		for _, c := range candidates {
+			if c.path.Forced || c.path.StoreType == kv.TiFlash {
+				preferredPaths = append(preferredPaths, c)
+				continue
+			}
+			var unsignedIntHandle bool
+			if c.path.IsIntHandlePath && ds.tableInfo.PKIsHandle {
+				if pkColInfo := ds.tableInfo.GetPkColInfo(); pkColInfo != nil {
+					unsignedIntHandle = mysql.HasUnsignedFlag(pkColInfo.Flag)
 				}
 			}
+			if !ranger.HasFullRange(c.path.Ranges, unsignedIntHandle) {
+				preferredPaths = append(preferredPaths, c)
+				hasRangeScanPath = true
+			}
+		}
+		if hasRangeScanPath {
+			return preferredPaths
 		}
 	}
 

planner/core/integration_test.go

@@ -34,6 +34,7 @@ import (
 	"github.com/pingcap/tidb/session"
 	"github.com/pingcap/tidb/sessionctx/stmtctx"
 	"github.com/pingcap/tidb/sessionctx/variable"
+	"github.com/pingcap/tidb/statistics/handle"
 	"github.com/pingcap/tidb/table"
 	"github.com/pingcap/tidb/util/collate"
 	"github.com/pingcap/tidb/util/testkit"
@@ -4217,6 +4218,41 @@ func (s *testIntegrationSuite) TestOutputSkylinePruningInfo(c *C) {
 	}
 }
 
+func (s *testIntegrationSuite) TestPreferRangeScanForUnsignedIntHandle(c *C) {
+	tk := testkit.NewTestKit(c, s.store)
+	tk.MustExec("use test")
+	tk.MustExec("drop table if exists t")
+	tk.MustExec("create table t(a int unsigned primary key, b int, c int, index idx_b(b))")
+	tk.MustExec("insert into t values (1,2,3), (4,5,6), (7,8,9), (10,11,12), (13,14,15)")
+	do, _ := session.GetDomain(s.store)
+	c.Assert(do.StatsHandle().DumpStatsDeltaToKV(handle.DumpAll), IsNil)
+	tk.MustExec("analyze table t")
+
+	var input []string
+	var output []struct {
+		SQL      string
+		Plan     []string
+		Warnings []string
+	}
+	s.testData.GetTestCases(c, &input, &output)
+	for i, tt := range input {
+		s.testData.OnRecord(func() {
+			output[i].SQL = tt
+		})
+		if strings.HasPrefix(tt, "set") {
+			tk.MustExec(tt)
+			continue
+		}
+		s.testData.OnRecord(func() {
+			output[i].SQL = tt
+			output[i].Plan = s.testData.ConvertRowsToStrings(tk.MustQuery(tt).Rows())
+			output[i].Warnings = s.testData.ConvertRowsToStrings(tk.MustQuery("show warnings").Rows())
+		})
+		tk.MustQuery(tt).Check(testkit.Rows(output[i].Plan...))
+		tk.MustQuery("show warnings").Check(testkit.Rows(output[i].Warnings...))
+	}
+}
+
 func (s *testIntegrationSuite) TestIssues27130(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("use test")

planner/core/stats.go

@@ -623,8 +623,14 @@ func (ds *DataSource) accessPathsForConds(conditions []expression.Expression, us
 				logutil.BgLogger().Debug("can not derive statistics of a path", zap.Error(err))
 				continue
 			}
+			var unsignedIntHandle bool
+			if path.IsIntHandlePath && ds.tableInfo.PKIsHandle {
+				if pkColInfo := ds.tableInfo.GetPkColInfo(); pkColInfo != nil {
+					unsignedIntHandle = mysql.HasUnsignedFlag(pkColInfo.Flag)
+				}
+			}
 			// If the path contains a full range, ignore it.
-			if ranger.HasFullRange(path.Ranges) {
+			if ranger.HasFullRange(path.Ranges, unsignedIntHandle) {
 				continue
 			}
 			// If we have point or empty range, just remove other possible paths.
@@ -650,7 +656,7 @@ func (ds *DataSource) accessPathsForConds(conditions []expression.Expression, us
 			}
 			ds.deriveIndexPathStats(path, conditions, true)
 			// If the path contains a full range, ignore it.
-			if ranger.HasFullRange(path.Ranges) {
+			if ranger.HasFullRange(path.Ranges, false) {
 				continue
 			}
 			// If we have empty range, or point range on unique index, just remove other possible paths.

planner/core/testdata/integration_suite_in.json

@@ -349,5 +349,18 @@
       "select * from t where g = 5 order by f",
       "select * from t where d = 3 order by c, e"
     ]
+  },
+  {
+    "name": "TestPreferRangeScanForUnsignedIntHandle",
+    "cases": [
+      "set tidb_opt_prefer_range_scan = 0",
+      "explain format = 'verbose' select * from t where b > 5",
+      "explain format = 'verbose' select * from t where b = 6 order by a limit 1",
+      "explain format = 'verbose' select * from t where b = 6 limit 1",
+      "set tidb_opt_prefer_range_scan = 1",
+      "explain format = 'verbose' select * from t where b > 5",
+      "explain format = 'verbose' select * from t where b = 6 order by a limit 1",
+      "explain format = 'verbose' select * from t where b = 6 limit 1"
+    ]
   }
 ]

planner/core/testdata/integration_suite_out.json

@@ -1883,5 +1883,87 @@
         ]
       }
     ]
+  },
+  {
+    "Name": "TestPreferRangeScanForUnsignedIntHandle",
+    "Cases": [
+      {
+        "SQL": "set tidb_opt_prefer_range_scan = 0",
+        "Plan": null,
+        "Warnings": null
+      },
+      {
+        "SQL": "explain format = 'verbose' select * from t where b > 5",
+        "Plan": [
+          "TableReader_7 3.00 19.21 root  data:Selection_6",
+          "└─Selection_6 3.00 215.00 cop[tikv]  gt(test.t.b, 5)",
+          "  └─TableFullScan_5 5.00 200.00 cop[tikv] table:t keep order:false"
+        ],
+        "Warnings": null
+      },
+      {
+        "SQL": "explain format = 'verbose' select * from t where b = 6 order by a limit 1",
+        "Plan": [
+          "Limit_11 0.00 14.33 root  offset:0, count:1",
+          "└─TableReader_24 0.00 14.33 root  data:Limit_23",
+          "  └─Limit_23 0.00 215.00 cop[tikv]  offset:0, count:1",
+          "    └─Selection_22 0.00 215.00 cop[tikv]  eq(test.t.b, 6)",
+          "      └─TableFullScan_21 5.00 200.00 cop[tikv] table:t keep order:true"
+        ],
+        "Warnings": null
+      },
+      {
+        "SQL": "explain format = 'verbose' select * from t where b = 6 limit 1",
+        "Plan": [
+          "Limit_8 0.00 14.33 root  offset:0, count:1",
+          "└─TableReader_13 0.00 14.33 root  data:Limit_12",
+          "  └─Limit_12 0.00 215.00 cop[tikv]  offset:0, count:1",
+          "    └─Selection_11 0.00 215.00 cop[tikv]  eq(test.t.b, 6)",
+          "      └─TableFullScan_10 5.00 200.00 cop[tikv] table:t keep order:false"
+        ],
+        "Warnings": null
+      },
+      {
+        "SQL": "set tidb_opt_prefer_range_scan = 1",
+        "Plan": null,
+        "Warnings": null
+      },
+      {
+        "SQL": "explain format = 'verbose' select * from t where b > 5",
+        "Plan": [
+          "IndexLookUp_7 3.00 64.81 root  ",
+          "├─IndexRangeScan_5(Build) 3.00 191.00 cop[tikv] table:t, index:idx_b(b) range:(5,+inf], keep order:false",
+          "└─TableRowIDScan_6(Probe) 3.00 191.00 cop[tikv] table:t keep order:false"
+        ],
+        "Warnings": [
+          "Note 1105 [idx_b] remain after pruning paths for t given Prop{SortItems: [], TaskTp: rootTask}"
+        ]
+      },
+      {
+        "SQL": "explain format = 'verbose' select * from t where b = 6 order by a limit 1",
+        "Plan": [
+          "TopN_9 0.00 19.34 root  test.t.a, offset:0, count:1",
+          "└─IndexLookUp_16 0.00 19.33 root  ",
+          "  ├─TopN_15(Build) 0.00 0.00 cop[tikv]  test.t.a, offset:0, count:1",
+          "  │ └─IndexRangeScan_13 0.00 20.00 cop[tikv] table:t, index:idx_b(b) range:[6,6], keep order:false",
+          "  └─TableRowIDScan_14(Probe) 0.00 20.00 cop[tikv] table:t keep order:false, stats:pseudo"
+        ],
+        "Warnings": [
+          "Note 1105 [idx_b] remain after pruning paths for t given Prop{SortItems: [], TaskTp: copDoubleReadTask}"
+        ]
+      },
+      {
+        "SQL": "explain format = 'verbose' select * from t where b = 6 limit 1",
+        "Plan": [
+          "IndexLookUp_13 0.00 19.33 root  limit embedded(offset:0, count:1)",
+          "├─Limit_12(Build) 0.00 20.00 cop[tikv]  offset:0, count:1",
+          "│ └─IndexRangeScan_10 0.00 20.00 cop[tikv] table:t, index:idx_b(b) range:[6,6], keep order:false",
+          "└─TableRowIDScan_11(Probe) 0.00 20.00 cop[tikv] table:t keep order:false, stats:pseudo"
+        ],
+        "Warnings": [
+          "Note 1105 [idx_b] remain after pruning paths for t given Prop{SortItems: [], TaskTp: copDoubleReadTask}"
+        ]
+      }
+    ]
   }
 ]

sessionctx/variable/sysvar.go

@@ -922,7 +922,7 @@ var defaultSysVars = []*SysVar{
 		s.SetAllowInSubqToJoinAndAgg(TiDBOptOn(val))
 		return nil
 	}},
-	{Scope: ScopeSession, Name: TiDBOptPreferRangeScan, Value: BoolToOnOff(DefOptPreferRangeScan), Type: TypeBool, IsHintUpdatable: true, SetSession: func(s *SessionVars, val string) error {
+	{Scope: ScopeGlobal | ScopeSession, Name: TiDBOptPreferRangeScan, Value: BoolToOnOff(DefOptPreferRangeScan), Type: TypeBool, IsHintUpdatable: true, SetSession: func(s *SessionVars, val string) error {
 		s.SetAllowPreferRangeScan(TiDBOptOn(val))
 		return nil
 	}},

util/ranger/testdata/ranger_suite_out.json

@@ -366,7 +366,7 @@
         "Plan": [
           "TableReader_7 1.00 root  data:Selection_6",
           "└─Selection_6 1.00 cop[tikv]  eq(test.t2.t, \"aaaa\")",
-          "  └─TableRangeScan_5 2.00 cop[tikv] table:t2 range:[0,+inf], keep order:false"
+          "  └─TableFullScan_5 2.00 cop[tikv] table:t2 keep order:false"
         ],
         "Result": [
           "1 aaaa"
@@ -377,7 +377,7 @@
         "Plan": [
           "TableReader_7 1.60 root  data:Selection_6",
           "└─Selection_6 1.60 cop[tikv]  or(eq(test.t2.t, \"aaaa\"), eq(test.t2.t, \"a\"))",
-          "  └─TableRangeScan_5 2.00 cop[tikv] table:t2 range:[0,+inf], keep order:false"
+          "  └─TableFullScan_5 2.00 cop[tikv] table:t2 keep order:false"
         ],
         "Result": [
           "1 aaaa",

util/ranger/types.go

@@ -107,7 +107,15 @@ func (ran *Range) IsPointNullable(sc *stmtctx.StatementContext) bool {
 }
 
 // IsFullRange check if the range is full scan range
-func (ran *Range) IsFullRange() bool {
+func (ran *Range) IsFullRange(unsignedIntHandle bool) bool {
+	if unsignedIntHandle {
+		if len(ran.LowVal) != 1 || len(ran.HighVal) != 1 {
+			return false
+		}
+		lowValRawString := formatDatum(ran.LowVal[0], true)
+		highValRawString := formatDatum(ran.HighVal[0], false)
+		return lowValRawString == "0" && highValRawString == "+inf"
+	}
 	if len(ran.LowVal) != len(ran.HighVal) {
 		return false
 	}
@@ -124,9 +132,9 @@ func (ran *Range) IsFullRange() bool {
 }
 
 // HasFullRange checks if any range in the slice is a full range.
-func HasFullRange(ranges []*Range) bool {
+func HasFullRange(ranges []*Range, unsignedIntHandle bool) bool {
 	for _, ran := range ranges {
-		if ran.IsFullRange() {
+		if ran.IsFullRange(unsignedIntHandle) {
 			return true
 		}
 	}