planner: support keep_order and no_keep_order hint (pingcap#40089)

ref pingcap#39964

parser/ast/dml.go

@@ -356,6 +356,8 @@ const (
 	HintUse IndexHintType = iota + 1
 	HintIgnore
 	HintForce
+	HintKeepOrder
+	HintNoKeepOrder
 )
 
 // IndexHintScope is the type for index hint for join, order by or group by.
@@ -386,6 +388,10 @@ func (n *IndexHint) Restore(ctx *format.RestoreCtx) error {
 		indexHintType = "IGNORE INDEX"
 	case HintForce:
 		indexHintType = "FORCE INDEX"
+	case HintKeepOrder:
+		indexHintType = "KEEP ORDER"
+	case HintNoKeepOrder:
+		indexHintType = "NO KEEP ORDER"
 	default: // Prevent accidents
 		return errors.New("IndexHintType has an error while matching")
 	}

planner/core/find_best_task.go

@@ -1454,6 +1454,14 @@ func (ds *DataSource) convertToIndexScan(prop *property.PhysicalProperty,
 	if !prop.IsSortItemEmpty() && !candidate.isMatchProp {
 		return invalidTask, nil
 	}
+	// If we need to keep order for the index scan, we should forbid the non-keep-order index scan when we try to generate the path.
+	if prop.IsSortItemEmpty() && candidate.path.ForceKeepOrder {
+		return invalidTask, nil
+	}
+	// If we don't need to keep order for the index scan, we should forbid the non-keep-order index scan when we try to generate the path.
+	if !prop.IsSortItemEmpty() && candidate.path.ForceNoKeepOrder {
+		return invalidTask, nil
+	}
 	path := candidate.path
 	is := ds.getOriginalPhysicalIndexScan(prop, path, candidate.isMatchProp, candidate.path.IsSingleScan)
 	cop := &copTask{
@@ -1993,6 +2001,14 @@ func (ds *DataSource) convertToTableScan(prop *property.PhysicalProperty, candid
 	if !prop.IsSortItemEmpty() && !candidate.isMatchProp {
 		return invalidTask, nil
 	}
+	// If we need to keep order for the index scan, we should forbid the non-keep-order index scan when we try to generate the path.
+	if prop.IsSortItemEmpty() && candidate.path.ForceKeepOrder {
+		return invalidTask, nil
+	}
+	// If we don't need to keep order for the index scan, we should forbid the non-keep-order index scan when we try to generate the path.
+	if !prop.IsSortItemEmpty() && candidate.path.ForceNoKeepOrder {
+		return invalidTask, nil
+	}
 	ts, _ := ds.getOriginalPhysicalTableScan(prop, candidate.path, candidate.isMatchProp)
 	if ts.KeepOrder && ts.StoreType == kv.TiFlash && (ts.Desc || ds.SCtx().GetSessionVars().TiFlashFastScan) {
 		// TiFlash fast mode(https://github.com/pingcap/tidb/pull/35851) does not keep order in TableScan

planner/core/integration_test.go

@@ -1324,6 +1324,43 @@ func TestReadFromStorageHint(t *testing.T) {
 	}
 }
 
+func TestKeepOrderHint(t *testing.T) {
+	store := testkit.CreateMockStore(t)
+	tk := testkit.NewTestKit(t, store)
+
+	tk.MustExec("use test")
+	tk.MustExec("set tidb_cost_model_version=2")
+	tk.MustExec("drop table if exists t, t1, t2")
+	tk.MustExec("create table t(a int, b int, primary key(a));")
+	tk.MustExec("create table t1(a int, b int, index idx_a(a), index idx_b(b));")
+
+	// If the optimizer can not generate the keep order plan, it will report error
+	err := tk.ExecToErr("explain select /*+ keep_order(t1, idx_a) */ * from t1 where a<10 limit 1;")
+	require.EqualError(t, err, "[planner:1815]Internal : Can't find a proper physical plan for this query")
+
+	err = tk.ExecToErr("explain select /*+ keep_order(t, primary) */ * from t where a<10 limit 1;")
+	require.EqualError(t, err, "[planner:1815]Internal : Can't find a proper physical plan for this query")
+
+	var input []string
+	var output []struct {
+		SQL  string
+		Plan []string
+		Warn []string
+	}
+	integrationSuiteData := core.GetIntegrationSuiteData()
+	integrationSuiteData.LoadTestCases(t, &input, &output)
+	for i, tt := range input {
+		testdata.OnRecord(func() {
+			output[i].SQL = tt
+			output[i].Plan = testdata.ConvertRowsToStrings(tk.MustQuery(tt).Rows())
+			output[i].Warn = testdata.ConvertSQLWarnToStrings(tk.Session().GetSessionVars().StmtCtx.GetWarnings())
+		})
+		res := tk.MustQuery(tt)
+		res.Check(testkit.Rows(output[i].Plan...))
+		require.Equal(t, output[i].Warn, testdata.ConvertSQLWarnToStrings(tk.Session().GetSessionVars().StmtCtx.GetWarnings()))
+	}
+}
+
 func TestViewHint(t *testing.T) {
 	store := testkit.CreateMockStore(t)
 	tk := testkit.NewTestKit(t, store)

planner/core/logical_plan_builder.go

@@ -115,6 +115,10 @@ const (
 	HintIgnoreIndex = "ignore_index"
 	// HintForceIndex make optimizer to use this index even if it thinks a table scan is more efficient.
 	HintForceIndex = "force_index"
+	// HintKeepOrder is hint enforce using some indexes and keep the index's order.
+	HintKeepOrder = "keep_order"
+	// HintNoKeepOrder is hint enforce using some indexes and not keep the index's order.
+	HintNoKeepOrder = "no_keep_order"
 	// HintAggToCop is hint enforce pushing aggregation to coprocessor.
 	HintAggToCop = "agg_to_cop"
 	// HintReadFromStorage is hint enforce some tables read from specific type of storage.
@@ -3589,7 +3593,7 @@ func (b *PlanBuilder) pushTableHints(hints []*ast.TableOptimizerHint, currentLev
 		// Set warning for the hint that requires the table name.
 		switch hint.HintName.L {
 		case TiDBMergeJoin, HintSMJ, TiDBIndexNestedLoopJoin, HintINLJ, HintINLHJ, HintINLMJ,
-			TiDBHashJoin, HintHJ, HintUseIndex, HintIgnoreIndex, HintForceIndex, HintIndexMerge, HintLeading:
+			TiDBHashJoin, HintHJ, HintUseIndex, HintIgnoreIndex, HintForceIndex, HintKeepOrder, HintNoKeepOrder, HintIndexMerge, HintLeading:
 			if len(hint.Tables) == 0 {
 				b.pushHintWithoutTableWarning(hint)
 				continue
@@ -3625,48 +3629,31 @@ func (b *PlanBuilder) pushTableHints(hints []*ast.TableOptimizerHint, currentLev
 			aggHints.preferAggType |= preferStreamAgg
 		case HintAggToCop:
 			aggHints.preferAggToCop = true
-		case HintUseIndex:
+		case HintUseIndex, HintIgnoreIndex, HintForceIndex, HintKeepOrder, HintNoKeepOrder:
 			dbName := hint.Tables[0].DBName
 			if dbName.L == "" {
 				dbName = model.NewCIStr(b.ctx.GetSessionVars().CurrentDB)
 			}
-			indexHintList = append(indexHintList, indexHintInfo{
-				dbName:     dbName,
-				tblName:    hint.Tables[0].TableName,
-				partitions: hint.Tables[0].PartitionList,
-				indexHint: &ast.IndexHint{
-					IndexNames: hint.Indexes,
-					HintType:   ast.HintUse,
-					HintScope:  ast.HintForScan,
-				},
-			})
-		case HintIgnoreIndex:
-			dbName := hint.Tables[0].DBName
-			if dbName.L == "" {
-				dbName = model.NewCIStr(b.ctx.GetSessionVars().CurrentDB)
-			}
-			indexHintList = append(indexHintList, indexHintInfo{
-				dbName:     dbName,
-				tblName:    hint.Tables[0].TableName,
-				partitions: hint.Tables[0].PartitionList,
-				indexHint: &ast.IndexHint{
-					IndexNames: hint.Indexes,
-					HintType:   ast.HintIgnore,
-					HintScope:  ast.HintForScan,
-				},
-			})
-		case HintForceIndex:
-			dbName := hint.Tables[0].DBName
-			if dbName.L == "" {
-				dbName = model.NewCIStr(b.ctx.GetSessionVars().CurrentDB)
+			var hintType ast.IndexHintType
+			switch hint.HintName.L {
+			case HintUseIndex:
+				hintType = ast.HintUse
+			case HintIgnoreIndex:
+				hintType = ast.HintIgnore
+			case HintForceIndex:
+				hintType = ast.HintForce
+			case HintKeepOrder:
+				hintType = ast.HintKeepOrder
+			case HintNoKeepOrder:
+				hintType = ast.HintNoKeepOrder
 			}
 			indexHintList = append(indexHintList, indexHintInfo{
 				dbName:     dbName,
 				tblName:    hint.Tables[0].TableName,
 				partitions: hint.Tables[0].PartitionList,
 				indexHint: &ast.IndexHint{
 					IndexNames: hint.Indexes,
-					HintType:   ast.HintForce,
+					HintType:   hintType,
 					HintScope:  ast.HintForScan,
 				},
 			})

planner/core/planbuilder.go

@@ -1360,6 +1360,12 @@ func getPossibleAccessPaths(ctx sessionctx.Context, tableHints *tableHintInfo, i
 			// our cost estimation is not reliable.
 			hasUseOrForce = true
 			path.Forced = true
+			if hint.HintType == ast.HintKeepOrder {
+				path.ForceKeepOrder = true
+			}
+			if hint.HintType == ast.HintNoKeepOrder {
+				path.ForceNoKeepOrder = true
+			}
 			available = append(available, path)
 		}
 	}

planner/core/testdata/integration_suite_in.json

@@ -653,6 +653,17 @@
       "desc format = 'brief' select /*+ read_from_storage(tiflash[t, ttt], tikv[tt]) */ * from ttt"
     ]
   },
+  {
+    "name": "TestKeepOrderHint",
+    "cases": [
+      "explain select /*+ keep_order(t1, idx_a) */ * from t1 where a<10 order by a limit 1;",
+      "explain select /*+ keep_order(t, primary) */ * from t where a<10 order by a limit 1;",
+      "explain select /*+ no_keep_order(t1, idx_a) */ * from t1 where a<10 order by a limit 1;",
+      "explain select /*+ no_keep_order(t, primary) */ * from t where a<10 order by a limit 1;",
+      "explain select /*+ no_keep_order(t1, idx_a) */ * from t1 where a<10 limit 1;",
+      "explain select /*+ no_keep_order(t, primary) */ * from t where a<10 limit 1;"
+    ]
+  },
   {
     "name": "TestViewHint",
     "cases": [

planner/core/testdata/integration_suite_out.json

@@ -4318,6 +4318,74 @@
       }
     ]
   },
+  {
+    "Name": "TestKeepOrderHint",
+    "Cases": [
+      {
+        "SQL": "explain select /*+ keep_order(t1, idx_a) */ * from t1 where a<10 order by a limit 1;",
+        "Plan": [
+          "Limit_12 1.00 root  offset:0, count:1",
+          "└─Projection_17 1.00 root  test.t1.a, test.t1.b",
+          "  └─IndexLookUp_16 1.00 root  ",
+          "    ├─Limit_15(Build) 1.00 cop[tikv]  offset:0, count:1",
+          "    │ └─IndexRangeScan_13 1.00 cop[tikv] table:t1, index:idx_a(a) range:[-inf,10), keep order:true, stats:pseudo",
+          "    └─TableRowIDScan_14(Probe) 1.00 cop[tikv] table:t1 keep order:false, stats:pseudo"
+        ],
+        "Warn": null
+      },
+      {
+        "SQL": "explain select /*+ keep_order(t, primary) */ * from t where a<10 order by a limit 1;",
+        "Plan": [
+          "Limit_11 1.00 root  offset:0, count:1",
+          "└─TableReader_15 1.00 root  data:Limit_14",
+          "  └─Limit_14 1.00 cop[tikv]  offset:0, count:1",
+          "    └─TableRangeScan_13 333.33 cop[tikv] table:t range:[-inf,10), keep order:true, stats:pseudo"
+        ],
+        "Warn": null
+      },
+      {
+        "SQL": "explain select /*+ no_keep_order(t1, idx_a) */ * from t1 where a<10 order by a limit 1;",
+        "Plan": [
+          "TopN_9 1.00 root  test.t1.a, offset:0, count:1",
+          "└─IndexLookUp_16 1.00 root  ",
+          "  ├─TopN_15(Build) 1.00 cop[tikv]  test.t1.a, offset:0, count:1",
+          "  │ └─IndexRangeScan_13 3323.33 cop[tikv] table:t1, index:idx_a(a) range:[-inf,10), keep order:false, stats:pseudo",
+          "  └─TableRowIDScan_14(Probe) 1.00 cop[tikv] table:t1 keep order:false, stats:pseudo"
+        ],
+        "Warn": null
+      },
+      {
+        "SQL": "explain select /*+ no_keep_order(t, primary) */ * from t where a<10 order by a limit 1;",
+        "Plan": [
+          "TopN_8 1.00 root  test.t.a, offset:0, count:1",
+          "└─TableReader_15 1.00 root  data:TopN_14",
+          "  └─TopN_14 1.00 cop[tikv]  test.t.a, offset:0, count:1",
+          "    └─TableRangeScan_13 3333.33 cop[tikv] table:t range:[-inf,10), keep order:false, stats:pseudo"
+        ],
+        "Warn": null
+      },
+      {
+        "SQL": "explain select /*+ no_keep_order(t1, idx_a) */ * from t1 where a<10 limit 1;",
+        "Plan": [
+          "IndexLookUp_13 1.00 root  limit embedded(offset:0, count:1)",
+          "├─Limit_12(Build) 1.00 cop[tikv]  offset:0, count:1",
+          "│ └─IndexRangeScan_10 1.00 cop[tikv] table:t1, index:idx_a(a) range:[-inf,10), keep order:false, stats:pseudo",
+          "└─TableRowIDScan_11(Probe) 1.00 cop[tikv] table:t1 keep order:false, stats:pseudo"
+        ],
+        "Warn": null
+      },
+      {
+        "SQL": "explain select /*+ no_keep_order(t, primary) */ * from t where a<10 limit 1;",
+        "Plan": [
+          "Limit_8 1.00 root  offset:0, count:1",
+          "└─TableReader_12 1.00 root  data:Limit_11",
+          "  └─Limit_11 1.00 cop[tikv]  offset:0, count:1",
+          "    └─TableRangeScan_10 333.33 cop[tikv] table:t range:[-inf,10), keep order:false, stats:pseudo"
+        ],
+        "Warn": null
+      }
+    ]
+  },
   {
     "Name": "TestViewHint",
     "Cases": [

planner/util/path.go

@@ -63,7 +63,9 @@ type AccessPath struct {
 	IsIntHandlePath    bool
 	IsCommonHandlePath bool
 	// Forced means this path is generated by `use/force index()`.
-	Forced bool
+	Forced           bool
+	ForceKeepOrder   bool
+	ForceNoKeepOrder bool
 	// IsSingleScan indicates whether the path is a single index/table scan or table access after index scan.
 	IsSingleScan bool
 
@@ -97,6 +99,8 @@ func (path *AccessPath) Clone() *AccessPath {
 		IsIntHandlePath:          path.IsIntHandlePath,
 		IsCommonHandlePath:       path.IsCommonHandlePath,
 		Forced:                   path.Forced,
+		ForceKeepOrder:           path.ForceKeepOrder,
+		ForceNoKeepOrder:         path.ForceNoKeepOrder,
 		IsSingleScan:             path.IsSingleScan,
 		IsUkShardIndexPath:       path.IsUkShardIndexPath,
 	}