planner: refine prefer-range-scan behavior

executor/set_test.go

@@ -539,6 +539,16 @@ func (s *testSerialSuite1) TestSetVar(c *C) {
 	tk.MustExec(`set tidb_opt_limit_push_down_threshold = 20`)
 	tk.MustQuery(`select @@global.tidb_opt_limit_push_down_threshold`).Check(testkit.Rows("100"))
 	tk.MustQuery(`select @@tidb_opt_limit_push_down_threshold`).Check(testkit.Rows("20"))
+
+	tk.MustQuery("select @@tidb_opt_prefer_range_scan").Check(testkit.Rows("0"))
+	tk.MustExec("set global tidb_opt_prefer_range_scan = 1")
+	tk.MustQuery("select @@global.tidb_opt_prefer_range_scan").Check(testkit.Rows("1"))
+	tk.MustExec("set global tidb_opt_prefer_range_scan = 0")
+	tk.MustQuery("select @@global.tidb_opt_prefer_range_scan").Check(testkit.Rows("0"))
+	tk.MustExec("set session tidb_opt_prefer_range_scan = 1")
+	tk.MustQuery("select @@session.tidb_opt_prefer_range_scan").Check(testkit.Rows("1"))
+	tk.MustExec("set session tidb_opt_prefer_range_scan = 0")
+	tk.MustQuery("select @@session.tidb_opt_prefer_range_scan").Check(testkit.Rows("0"))
 }
 
 func (s *testSuite5) TestTruncateIncorrectIntSessionVar(c *C) {

planner/core/explain.go

@@ -184,7 +184,7 @@ func (p *PhysicalIndexScan) isFullScan() bool {
 		return false
 	}
 	for _, ran := range p.Ranges {
-		if !ran.IsFullRange() {
+		if !ran.IsFullRange(false) {
 			return false
 		}
 	}
@@ -322,7 +322,13 @@ func (p *PhysicalTableScan) isFullScan() bool {
 		return false
 	}
 	for _, ran := range p.Ranges {
-		if !ran.IsFullRange() {
+		var unsignedIntHandle bool
+		if p.Table.PKIsHandle {
+			if pkColInfo := p.Table.GetPkColInfo(); pkColInfo != nil {
+				unsignedIntHandle = mysql.HasUnsignedFlag(pkColInfo.Flag)
+			}
+		}
+		if !ran.IsFullRange(unsignedIntHandle) {
 			return false
 		}
 	}

planner/core/find_best_task.go

@@ -624,14 +624,27 @@ func (ds *DataSource) skylinePruning(prop *property.PhysicalProperty) []*candida
 	}
 
 	if ds.ctx.GetSessionVars().GetAllowPreferRangeScan() && len(candidates) > 1 {
-		// remove the table/index full scan path
-		for i, c := range candidates {
-			for _, ran := range c.path.Ranges {
-				if ran.IsFullRange() {
-					candidates = append(candidates[:i], candidates[i+1:]...)
-					return candidates
+		// remove table/index full scan paths if there exists some range scan path
+		preferredPaths := make([]*candidatePath, 0, len(candidates))
+		var hasRangeScanPath bool
+		for _, c := range candidates {
+			if c.path.Forced || c.path.StoreType == kv.TiFlash {
+				preferredPaths = append(preferredPaths, c)
+				continue
+			}
+			var unsignedIntHandle bool
+			if c.path.IsIntHandlePath && ds.tableInfo.PKIsHandle {
+				if pkColInfo := ds.tableInfo.GetPkColInfo(); pkColInfo != nil {
+					unsignedIntHandle = mysql.HasUnsignedFlag(pkColInfo.Flag)
 				}
 			}
+			if !ranger.HasFullRange(c.path.Ranges, unsignedIntHandle) {
+				preferredPaths = append(preferredPaths, c)
+				hasRangeScanPath = true
+			}
+		}
+		if hasRangeScanPath {
+			return preferredPaths
 		}
 	}
 

planner/core/integration_test.go

@@ -33,6 +33,7 @@ import (
 	"github.com/pingcap/tidb/session"
 	"github.com/pingcap/tidb/sessionctx/stmtctx"
 	"github.com/pingcap/tidb/sessionctx/variable"
+	"github.com/pingcap/tidb/statistics/handle"
 	"github.com/pingcap/tidb/table"
 	"github.com/pingcap/tidb/util/collate"
 	"github.com/pingcap/tidb/util/testkit"
@@ -4201,3 +4202,38 @@ func (s *testIntegrationSuite) TestOutputSkylinePruningInfo(c *C) {
 		tk.MustQuery("show warnings").Check(testkit.Rows(output[i].Warnings...))
 	}
 }
+
+func (s *testIntegrationSuite) TestPreferRangeScanForUnsignedIntHandle(c *C) {
+	tk := testkit.NewTestKit(c, s.store)
+	tk.MustExec("use test")
+	tk.MustExec("drop table if exists t")
+	tk.MustExec("create table t(a int unsigned primary key, b int, c int, index idx_b(b))")
+	tk.MustExec("insert into t values (1,2,3), (4,5,6), (7,8,9), (10,11,12), (13,14,15)")
+	do, _ := session.GetDomain(s.store)
+	c.Assert(do.StatsHandle().DumpStatsDeltaToKV(handle.DumpAll), IsNil)
+	tk.MustExec("analyze table t")
+
+	var input []string
+	var output []struct {
+		SQL      string
+		Plan     []string
+		Warnings []string
+	}
+	s.testData.GetTestCases(c, &input, &output)
+	for i, tt := range input {
+		s.testData.OnRecord(func() {
+			output[i].SQL = tt
+		})
+		if strings.HasPrefix(tt, "set") {
+			tk.MustExec(tt)
+			continue
+		}
+		s.testData.OnRecord(func() {
+			output[i].SQL = tt
+			output[i].Plan = s.testData.ConvertRowsToStrings(tk.MustQuery(tt).Rows())
+			output[i].Warnings = s.testData.ConvertRowsToStrings(tk.MustQuery("show warnings").Rows())
+		})
+		tk.MustQuery(tt).Check(testkit.Rows(output[i].Plan...))
+		tk.MustQuery("show warnings").Check(testkit.Rows(output[i].Warnings...))
+	}
+}

planner/core/stats.go

@@ -622,8 +622,14 @@ func (ds *DataSource) accessPathsForConds(conditions []expression.Expression, us
 				logutil.BgLogger().Debug("can not derive statistics of a path", zap.Error(err))
 				continue
 			}
+			var unsignedIntHandle bool
+			if path.IsIntHandlePath && ds.tableInfo.PKIsHandle {
+				if pkColInfo := ds.tableInfo.GetPkColInfo(); pkColInfo != nil {
+					unsignedIntHandle = mysql.HasUnsignedFlag(pkColInfo.Flag)
+				}
+			}
 			// If the path contains a full range, ignore it.
-			if ranger.HasFullRange(path.Ranges) {
+			if ranger.HasFullRange(path.Ranges, unsignedIntHandle) {
 				continue
 			}
 			// If we have point or empty range, just remove other possible paths.
@@ -649,7 +655,7 @@ func (ds *DataSource) accessPathsForConds(conditions []expression.Expression, us
 			}
 			ds.deriveIndexPathStats(path, conditions, true)
 			// If the path contains a full range, ignore it.
-			if ranger.HasFullRange(path.Ranges) {
+			if ranger.HasFullRange(path.Ranges, false) {
 				continue
 			}
 			// If we have empty range, or point range on unique index, just remove other possible paths.

planner/core/testdata/integration_partition_suite_out.json

@@ -1021,11 +1021,11 @@
         ]
       },
       {
-        "SQL": "create table tto_seconds (a date, b datetime) partition by list (TO_SECONDS(a)) (partition p0 values in (0, 1, 2, 3, 63745012800), partition p1 values in (4, 5, 6, 7, 63744969600))",
+        "SQL": "create table tto_seconds (a date, b datetime) partition by list (TO_SECONDS(a)) (partition p0 values in (0, 1, 2, 3, 63740649600), partition p1 values in (4, 5, 6, 7, 63744969600))",
         "Results": null
       },
       {
-        "SQL": "insert into tto_seconds values ('2019-12-31 12:00:00', '2019-12-31 23:59:59'), ('2019-12-31 23:06:59', '2019-12-31 12:00:00')",
+        "SQL": "insert into tto_seconds values ('2019-12-31 12:00:00', '2019-12-31 23:59:59'), ('2019-11-11 23:06:59', '2019-12-31 12:00:00')",
         "Results": null
       },
       {

planner/core/testdata/integration_suite_in.json

@@ -341,5 +341,18 @@
       "select * from t where g = 5 order by f",
       "select * from t where d = 3 order by c, e"
     ]
+  },
+  {
+    "name": "TestPreferRangeScanForUnsignedIntHandle",
+    "cases": [
+      "set tidb_opt_prefer_range_scan = 0",
+      "explain format = 'verbose' select * from t where b > 5",
+      "explain format = 'verbose' select * from t where b = 6 order by a limit 1",
+      "explain format = 'verbose' select * from t where b = 6 limit 1",
+      "set tidb_opt_prefer_range_scan = 1",
+      "explain format = 'verbose' select * from t where b > 5",
+      "explain format = 'verbose' select * from t where b = 6 order by a limit 1",
+      "explain format = 'verbose' select * from t where b = 6 limit 1"
+    ]
   }
 ]

planner/core/testdata/integration_suite_out.json

@@ -1805,5 +1805,86 @@
         ]
       }
     ]
+  },
+  {
+    "Name": "TestPreferRangeScanForUnsignedIntHandle",
+    "Cases": [
+      {
+        "SQL": "set tidb_opt_prefer_range_scan = 0",
+        "Plan": null,
+        "Warnings": null
+      },
+      {
+        "SQL": "explain format = 'verbose' select * from t where b > 5",
+        "Plan": [
+          "TableReader_7 3333.33 45418.00 root  data:Selection_6",
+          "└─Selection_6 3333.33 600020.00 cop[tikv]  gt(test.t.b, 5)",
+          "  └─TableFullScan_5 10000.00 570020.00 cop[tikv] table:t keep order:false, stats:pseudo"
+        ],
+        "Warnings": null
+      },
+      {
+        "SQL": "explain format = 'verbose' select * from t where b = 6 order by a limit 1",
+        "Plan": [
+          "TopN_9 1.00 72.74 root  test.t.a, offset:0, count:1",
+          "└─IndexLookUp_20 1.00 69.74 root  ",
+          "  ├─TopN_19(Build) 1.00 0.00 cop[tikv]  test.t.a, offset:0, count:1",
+          "  │ └─IndexRangeScan_17 10.00 590.00 cop[tikv] table:t, index:idx_b(b) range:[6,6], keep order:false, stats:pseudo",
+          "  └─TableRowIDScan_18(Probe) 1.00 590.00 cop[tikv] table:t keep order:false, stats:pseudo"
+        ],
+        "Warnings": null
+      },
+      {
+        "SQL": "explain format = 'verbose' select * from t where b = 6 limit 1",
+        "Plan": [
+          "IndexLookUp_17 1.00 33.54 root  limit embedded(offset:0, count:1)",
+          "├─Limit_16(Build) 1.00 77.00 cop[tikv]  offset:0, count:1",
+          "│ └─IndexRangeScan_14 1.00 77.00 cop[tikv] table:t, index:idx_b(b) range:[6,6], keep order:false, stats:pseudo",
+          "└─TableRowIDScan_15(Probe) 1.00 77.00 cop[tikv] table:t keep order:false, stats:pseudo"
+        ],
+        "Warnings": null
+      },
+      {
+        "SQL": "set tidb_opt_prefer_range_scan = 1",
+        "Plan": null,
+        "Warnings": null
+      },
+      {
+        "SQL": "explain format = 'verbose' select * from t where b > 5",
+        "Plan": [
+          "IndexLookUp_7 3333.33 70785.94 root  ",
+          "├─IndexRangeScan_5(Build) 3333.33 190020.00 cop[tikv] table:t, index:idx_b(b) range:(5,+inf], keep order:false, stats:pseudo",
+          "└─TableRowIDScan_6(Probe) 3333.33 190020.00 cop[tikv] table:t keep order:false, stats:pseudo"
+        ],
+        "Warnings": [
+          "Note 1105 [idx_b] remain after pruning paths for t given Prop{SortItems: [], TaskTp: rootTask}"
+        ]
+      },
+      {
+        "SQL": "explain format = 'verbose' select * from t where b = 6 order by a limit 1",
+        "Plan": [
+          "TopN_9 1.00 72.74 root  test.t.a, offset:0, count:1",
+          "└─IndexLookUp_16 1.00 69.74 root  ",
+          "  ├─TopN_15(Build) 1.00 0.00 cop[tikv]  test.t.a, offset:0, count:1",
+          "  │ └─IndexRangeScan_13 10.00 590.00 cop[tikv] table:t, index:idx_b(b) range:[6,6], keep order:false, stats:pseudo",
+          "  └─TableRowIDScan_14(Probe) 1.00 590.00 cop[tikv] table:t keep order:false, stats:pseudo"
+        ],
+        "Warnings": [
+          "Note 1105 [idx_b] remain after pruning paths for t given Prop{SortItems: [], TaskTp: copDoubleReadTask}"
+        ]
+      },
+      {
+        "SQL": "explain format = 'verbose' select * from t where b = 6 limit 1",
+        "Plan": [
+          "IndexLookUp_13 1.00 33.54 root  limit embedded(offset:0, count:1)",
+          "├─Limit_12(Build) 1.00 77.00 cop[tikv]  offset:0, count:1",
+          "│ └─IndexRangeScan_10 1.00 77.00 cop[tikv] table:t, index:idx_b(b) range:[6,6], keep order:false, stats:pseudo",
+          "└─TableRowIDScan_11(Probe) 1.00 77.00 cop[tikv] table:t keep order:false, stats:pseudo"
+        ],
+        "Warnings": [
+          "Note 1105 [idx_b] remain after pruning paths for t given Prop{SortItems: [], TaskTp: copDoubleReadTask}"
+        ]
+      }
+    ]
   }
 ]

sessionctx/variable/sysvar.go

@@ -921,7 +921,7 @@ var defaultSysVars = []*SysVar{
 		s.SetAllowInSubqToJoinAndAgg(TiDBOptOn(val))
 		return nil
 	}},
-	{Scope: ScopeSession, Name: TiDBOptPreferRangeScan, Value: BoolToOnOff(DefOptPreferRangeScan), Type: TypeBool, IsHintUpdatable: true, SetSession: func(s *SessionVars, val string) error {
+	{Scope: ScopeGlobal | ScopeSession, Name: TiDBOptPreferRangeScan, Value: BoolToOnOff(DefOptPreferRangeScan), Type: TypeBool, IsHintUpdatable: true, SetSession: func(s *SessionVars, val string) error {
 		s.SetAllowPreferRangeScan(TiDBOptOn(val))
 		return nil
 	}},

util/ranger/types.go

@@ -106,7 +106,15 @@ func (ran *Range) IsPointNullable(sc *stmtctx.StatementContext) bool {
 }
 
 // IsFullRange check if the range is full scan range
-func (ran *Range) IsFullRange() bool {
+func (ran *Range) IsFullRange(unsignedIntHandle bool) bool {
+	if unsignedIntHandle {
+		if len(ran.LowVal) != 1 || len(ran.HighVal) != 1 {
+			return false
+		}
+		lowValRawString := formatDatum(ran.LowVal[0], true)
+		highValRawString := formatDatum(ran.HighVal[0], false)
+		return lowValRawString == "0" && highValRawString == "+inf"
+	}
 	if len(ran.LowVal) != len(ran.HighVal) {
 		return false
 	}
@@ -123,9 +131,9 @@ func (ran *Range) IsFullRange() bool {
 }
 
 // HasFullRange checks if any range in the slice is a full range.
-func HasFullRange(ranges []*Range) bool {
+func HasFullRange(ranges []*Range, unsignedIntHandle bool) bool {
 	for _, ran := range ranges {
-		if ran.IsFullRange() {
+		if ran.IsFullRange(unsignedIntHandle) {
 			return true
 		}
 	}

util/ranger/types_test.go

@@ -134,53 +134,68 @@ func TestIsFullRange(t *testing.T) {
 	nullDatum := types.MinNotNullDatum()
 	nullDatum.SetNull()
 	isFullRangeTests := []struct {
-		ran         ranger.Range
-		isFullRange bool
+		ran               ranger.Range
+		unsignedIntHandle bool
+		isFullRange       bool
 	}{
 		{
 			ran: ranger.Range{
 				LowVal:  []types.Datum{types.NewIntDatum(math.MinInt64)},
 				HighVal: []types.Datum{types.NewIntDatum(math.MaxInt64)},
 			},
+			unsignedIntHandle: false,
 			isFullRange: true,
 		},
 		{
 			ran: ranger.Range{
 				LowVal:  []types.Datum{types.NewIntDatum(math.MaxInt64)},
 				HighVal: []types.Datum{types.NewIntDatum(math.MinInt64)},
 			},
+			unsignedIntHandle: false,
 			isFullRange: false,
 		},
 		{
 			ran: ranger.Range{
 				LowVal:  []types.Datum{types.NewIntDatum(1)},
 				HighVal: []types.Datum{types.NewUintDatum(math.MaxUint64)},
 			},
+			unsignedIntHandle: false,
 			isFullRange: false,
 		},
 		{
 			ran: ranger.Range{
 				LowVal:  []types.Datum{*nullDatum.Clone()},
 				HighVal: []types.Datum{types.NewUintDatum(math.MaxUint64)},
 			},
+			unsignedIntHandle: false,
 			isFullRange: true,
 		},
 		{
 			ran: ranger.Range{
 				LowVal:  []types.Datum{*nullDatum.Clone()},
 				HighVal: []types.Datum{*nullDatum.Clone()},
 			},
+			unsignedIntHandle: false,
 			isFullRange: false,
 		},
 		{
 			ran: ranger.Range{
 				LowVal:  []types.Datum{types.MinNotNullDatum()},
 				HighVal: []types.Datum{types.MaxValueDatum()},
 			},
+			unsignedIntHandle: false,
+			isFullRange: true,
+		},
+		{
+			ran: ranger.Range{
+				LowVal:  []types.Datum{types.NewUintDatum(0)},
+				HighVal: []types.Datum{types.NewUintDatum(math.MaxUint64)},
+			},
+			unsignedIntHandle: true,
 			isFullRange: true,
 		},
 	}
 	for _, v := range isFullRangeTests {
-		require.Equal(t, v.isFullRange, v.ran.IsFullRange())
+		require.Equal(t, v.isFullRange, v.ran.IsFullRange(t.unsignedIntHandle))
 	}
 }