planner: output skyline pruning information when executing EXPLAIN (#…

…26748)

executor/executor.go

@@ -1704,8 +1704,9 @@ func ResetContextOfStmt(ctx sessionctx.Context, s ast.StmtNode) (err error) {
 		sc.InVerboseExplain = strings.ToLower(explainStmt.Format) == types.ExplainFormatVerbose
 		s = explainStmt.Stmt
 	}
-	if _, ok := s.(*ast.ExplainForStmt); ok {
+	if explainForStmt, ok := s.(*ast.ExplainForStmt); ok {
 		sc.InExplainStmt = true
+		sc.InVerboseExplain = strings.ToLower(explainForStmt.Format) == types.ExplainFormatVerbose
 	}
 	// TODO: Many same bool variables here.
 	// We should set only two variables (

planner/core/common_plans.go

@@ -1115,6 +1115,11 @@ func (e *Explain) explainPlanInRowFormatCTE() (err error) {
 func (e *Explain) explainPlanInRowFormat(p Plan, taskType, driverSide, indent string, isLastChild bool) (err error) {
 	e.prepareOperatorInfo(p, taskType, driverSide, indent, isLastChild)
 	e.explainedPlans[p.ID()] = true
+	if e.ctx != nil && e.ctx.GetSessionVars() != nil && e.ctx.GetSessionVars().StmtCtx != nil {
+		if optimInfo, ok := e.ctx.GetSessionVars().StmtCtx.OptimInfo[p.ID()]; ok {
+			e.ctx.GetSessionVars().StmtCtx.AppendNote(errors.New(optimInfo))
+		}
+	}
 
 	// For every child we create a new sub-tree rooted by it.
 	childIndent := texttree.Indent4Child(indent, isLastChild)

planner/core/enforce_mpp_test.go

@@ -20,6 +20,7 @@ import (
 	"github.com/pingcap/parser/model"
 	"github.com/pingcap/tidb/domain"
 	"github.com/pingcap/tidb/kv"
+	"github.com/pingcap/tidb/sessionctx/stmtctx"
 	"github.com/pingcap/tidb/util/collate"
 	"github.com/pingcap/tidb/util/testkit"
 	"github.com/pingcap/tidb/util/testutil"
@@ -105,6 +106,16 @@ func (s *testEnforceMPPSuite) TestEnforceMPP(c *C) {
 		Warn []string
 	}
 	s.testData.GetTestCases(c, &input, &output)
+	filterWarnings := func(originalWarnings []stmtctx.SQLWarn) []stmtctx.SQLWarn {
+		warnings := make([]stmtctx.SQLWarn, 0, 4)
+		for _, warning := range originalWarnings {
+			// filter out warning about skyline pruning
+			if !strings.Contains(warning.Err.Error(), "remain after pruning paths for") {
+				warnings = append(warnings, warning)
+			}
+		}
+		return warnings
+	}
 	for i, tt := range input {
 		s.testData.OnRecord(func() {
 			output[i].SQL = tt
@@ -116,11 +127,11 @@ func (s *testEnforceMPPSuite) TestEnforceMPP(c *C) {
 		s.testData.OnRecord(func() {
 			output[i].SQL = tt
 			output[i].Plan = s.testData.ConvertRowsToStrings(tk.MustQuery(tt).Rows())
-			output[i].Warn = s.testData.ConvertSQLWarnToStrings(tk.Se.GetSessionVars().StmtCtx.GetWarnings())
+			output[i].Warn = s.testData.ConvertSQLWarnToStrings(filterWarnings(tk.Se.GetSessionVars().StmtCtx.GetWarnings()))
 		})
 		res := tk.MustQuery(tt)
 		res.Check(testkit.Rows(output[i].Plan...))
-		c.Assert(s.testData.ConvertSQLWarnToStrings(tk.Se.GetSessionVars().StmtCtx.GetWarnings()), DeepEquals, output[i].Warn)
+		c.Assert(s.testData.ConvertSQLWarnToStrings(filterWarnings(tk.Se.GetSessionVars().StmtCtx.GetWarnings())), DeepEquals, output[i].Warn)
 	}
 }
 

planner/core/find_best_task.go

@@ -14,7 +14,9 @@
 package core
 
 import (
+	"fmt"
 	"math"
+	"strings"
 
 	"github.com/pingcap/errors"
 	"github.com/pingcap/parser/ast"
@@ -546,19 +548,6 @@ func (ds *DataSource) getTableCandidate(path *util.AccessPath, prop *property.Ph
 
 func (ds *DataSource) getIndexCandidate(path *util.AccessPath, prop *property.PhysicalProperty) *candidatePath {
 	candidate := &candidatePath{path: path}
-	all, _ := prop.AllSameOrder()
-	// When the prop is empty or `all` is false, `isMatchProp` is better to be `false` because
-	// it needs not to keep order for index scan.
-	if !prop.IsEmpty() && all {
-		for i, col := range path.IdxCols {
-			if col.EqualByExprAndID(nil, prop.SortItems[0].Col) {
-				candidate.isMatchProp = matchIndicesProp(path.IdxCols[i:], path.IdxColLens[i:], prop.SortItems)
-				break
-			} else if i >= path.EqCondCount {
-				break
-			}
-		}
-	}
 	candidate.isMatchProp = ds.isMatchProp(path, prop)
 	candidate.accessCondsColSet = expression.ExtractColumnSet(path.AccessConds)
 	candidate.indexFiltersColSet = expression.ExtractColumnSet(path.IndexFilters)
@@ -649,6 +638,49 @@ func (ds *DataSource) skylinePruning(prop *property.PhysicalProperty) []*candida
 	return candidates
 }
 
+func (ds *DataSource) getPruningInfo(candidates []*candidatePath, prop *property.PhysicalProperty) string {
+	if !ds.ctx.GetSessionVars().StmtCtx.InVerboseExplain || len(candidates) == len(ds.possibleAccessPaths) {
+		return ""
+	}
+	if len(candidates) == 1 && len(candidates[0].path.Ranges) == 0 {
+		// For TableDual, we don't need to output pruning info.
+		return ""
+	}
+	names := make([]string, 0, len(candidates))
+	var tableName string
+	if ds.TableAsName.O == "" {
+		tableName = ds.tableInfo.Name.O
+	} else {
+		tableName = ds.TableAsName.O
+	}
+	getSimplePathName := func(path *util.AccessPath) string {
+		if path.IsTablePath() {
+			if path.StoreType == kv.TiFlash {
+				return tableName + "(tiflash)"
+			}
+			return tableName
+		}
+		return path.Index.Name.O
+	}
+	for _, cand := range candidates {
+		if cand.path.PartialIndexPaths != nil {
+			partialNames := make([]string, 0, len(cand.path.PartialIndexPaths))
+			for _, partialPath := range cand.path.PartialIndexPaths {
+				partialNames = append(partialNames, getSimplePathName(partialPath))
+			}
+			names = append(names, fmt.Sprintf("IndexMerge{%s}", strings.Join(partialNames, ",")))
+		} else {
+			names = append(names, getSimplePathName(cand.path))
+		}
+	}
+	items := make([]string, 0, len(prop.SortItems))
+	for _, item := range prop.SortItems {
+		items = append(items, item.String())
+	}
+	return fmt.Sprintf("[%s] remain after pruning paths for %s given Prop{SortItems: [%s], TaskTp: %s}",
+		strings.Join(names, ","), tableName, strings.Join(items, " "), prop.TaskTp)
+}
+
 func (ds *DataSource) isPointGetConvertableSchema() bool {
 	for _, col := range ds.Columns {
 		// Only handle tables that all columns are public.
@@ -724,6 +756,15 @@ func (ds *DataSource) findBestTask(prop *property.PhysicalProperty, planCounter
 
 	t = invalidTask
 	candidates := ds.skylinePruning(prop)
+	pruningInfo := ds.getPruningInfo(candidates, prop)
+	defer func() {
+		if err == nil && t != nil && !t.invalid() && pruningInfo != "" {
+			if ds.ctx.GetSessionVars().StmtCtx.OptimInfo == nil {
+				ds.ctx.GetSessionVars().StmtCtx.OptimInfo = make(map[int]string)
+			}
+			ds.ctx.GetSessionVars().StmtCtx.OptimInfo[t.plan().ID()] = pruningInfo
+		}
+	}()
 
 	cntPlan = 0
 	for _, candidate := range candidates {

planner/core/integration_test.go

@@ -4170,10 +4170,34 @@ func (s *testIntegrationSuite) TestHeuristicIndexSelection(c *C) {
 	for i, tt := range input {
 		s.testData.OnRecord(func() {
 			output[i].SQL = tt
-			output[i].Plan = s.testData.ConvertRowsToStrings(tk.MustQuery("explain format = 'brief' " + tt).Rows())
+			output[i].Plan = s.testData.ConvertRowsToStrings(tk.MustQuery("explain format = 'verbose' " + tt).Rows())
 			output[i].Warnings = s.testData.ConvertRowsToStrings(tk.MustQuery("show warnings").Rows())
 		})
-		tk.MustQuery("explain format = 'brief' " + tt).Check(testkit.Rows(output[i].Plan...))
+		tk.MustQuery("explain format = 'verbose' " + tt).Check(testkit.Rows(output[i].Plan...))
+		tk.MustQuery("show warnings").Check(testkit.Rows(output[i].Warnings...))
+	}
+}
+
+func (s *testIntegrationSuite) TestOutputSkylinePruningInfo(c *C) {
+	tk := testkit.NewTestKit(c, s.store)
+	tk.MustExec("use test")
+	tk.MustExec("drop table if exists t")
+	tk.MustExec("create table t(a int, b int, c int, d int, e int, f int, g int, primary key (a), unique key c_d_e (c, d, e), unique key f (f), unique key f_g (f, g), key g (g))")
+
+	var input []string
+	var output []struct {
+		SQL      string
+		Plan     []string
+		Warnings []string
+	}
+	s.testData.GetTestCases(c, &input, &output)
+	for i, tt := range input {
+		s.testData.OnRecord(func() {
+			output[i].SQL = tt
+			output[i].Plan = s.testData.ConvertRowsToStrings(tk.MustQuery("explain format = 'verbose' " + tt).Rows())
+			output[i].Warnings = s.testData.ConvertRowsToStrings(tk.MustQuery("show warnings").Rows())
+		})
+		tk.MustQuery("explain format = 'verbose' " + tt).Check(testkit.Rows(output[i].Plan...))
 		tk.MustQuery("show warnings").Check(testkit.Rows(output[i].Warnings...))
 	}
 }

planner/core/stats.go

@@ -344,7 +344,7 @@ func (ds *DataSource) derivePathStatsAndTryHeuristics() error {
 		ds.possibleAccessPaths = ds.possibleAccessPaths[:1]
 		// TODO: Can we make a more careful check on whether the optimization depends on mutable constants?
 		ds.ctx.GetSessionVars().StmtCtx.OptimDependOnMutableConst = true
-		if ds.ctx.GetSessionVars().StmtCtx.InExplainStmt {
+		if ds.ctx.GetSessionVars().StmtCtx.InVerboseExplain {
 			var tableName string
 			if ds.TableAsName.O == "" {
 				tableName = ds.tableInfo.Name.O

planner/core/testdata/integration_suite_in.json

@@ -328,7 +328,18 @@
       "select f, g from t1 where f = 2 and g in (3, 4, 5)",
       "select * from t1 where c = 1 and (d = 2 or d = 3) and e in (4, 5)",
       "select f, g from t1 where f = 2 and g > 3",
-      "select a, b, c from t2 where a = 1 and b = 2 and c in (1, 2, 3, 4, 5);"
+      "select a, b, c from t2 where a = 1 and b = 2 and c in (1, 2, 3, 4, 5)"
+    ]
+  },
+  {
+    "name": "TestOutputSkylinePruningInfo",
+    "cases": [
+      "select * from t where a > 1 order by f",
+      "select * from t where f > 1",
+      "select f from t where f > 1",
+      "select * from t where f > 3 and g = 5",
+      "select * from t where g = 5 order by f",
+      "select * from t where d = 3 order by c, e"
     ]
   }
 ]

planner/core/testdata/integration_suite_out.json

@@ -1686,7 +1686,7 @@
       {
         "SQL": "select * from t1 where a = 3 or a = 5",
         "Plan": [
-          "Batch_Point_Get 2.00 root table:t1 handle:[3 5], keep order:false, desc:false"
+          "Batch_Point_Get_5 2.00 12.53 root table:t1 handle:[3 5], keep order:false, desc:false"
         ],
         "Warnings": [
           "Note 1105 handle of t1 is selected since the path only has point ranges"
@@ -1695,7 +1695,7 @@
       {
         "SQL": "select f, g from t1 where f = 2 and g in (3, 4, 5)",
         "Plan": [
-          "Batch_Point_Get 3.00 root table:t1, index:f_g(f, g) keep order:false, desc:false"
+          "Batch_Point_Get_5 3.00 11.40 root table:t1, index:f_g(f, g) keep order:false, desc:false"
         ],
         "Warnings": [
           "Note 1105 unique index f_g of t1 is selected since the path only has point ranges with single scan"
@@ -1704,7 +1704,7 @@
       {
         "SQL": "select * from t1 where c = 1 and (d = 2 or d = 3) and e in (4, 5)",
         "Plan": [
-          "Batch_Point_Get 4.00 root table:t1, index:c_d_e(c, d, e) keep order:false, desc:false"
+          "Batch_Point_Get_5 4.00 27.20 root table:t1, index:c_d_e(c, d, e) keep order:false, desc:false"
         ],
         "Warnings": [
           "Note 1105 unique index c_d_e of t1 is selected since the path only has point ranges with double scan"
@@ -1713,8 +1713,8 @@
       {
         "SQL": "select f, g from t1 where f = 2 and g > 3",
         "Plan": [
-          "IndexReader 33.33 root  index:IndexRangeScan",
-          "└─IndexRangeScan 33.33 cop[tikv] table:t1, index:f_g(f, g) range:(2 3,2 +inf], keep order:false, stats:pseudo"
+          "IndexReader_6 33.33 160.78 root  index:IndexRangeScan_5",
+          "└─IndexRangeScan_5 33.33 1870.00 cop[tikv] table:t1, index:f_g(f, g) range:(2 3,2 +inf], keep order:false, stats:pseudo"
         ],
         "Warnings": [
           "Note 1105 unique index f_g of t1 is selected since the path only fetches limited number of rows with single scan"
@@ -1723,13 +1723,87 @@
       {
         "SQL": "select a, b, c from t2 where a = 1 and b = 2 and c in (1, 2, 3, 4, 5)",
         "Plan": [
-          "Selection 0.01 root  eq(test.t2.b, 2), in(test.t2.c, 1, 2, 3, 4, 5)",
-          "└─Point_Get 1.00 root table:t2, index:idx_a(a) "
+          "Selection_6 0.01 0.00 root  eq(test.t2.b, 2), in(test.t2.c, 1, 2, 3, 4, 5)",
+          "└─Point_Get_5 1.00 8.60 root table:t2, index:idx_a(a) "
         ],
         "Warnings": [
           "Note 1105 unique index idx_a of t2 is selected since the path only has point ranges with double scan"
         ]
       }
     ]
+  },
+  {
+    "Name": "TestOutputSkylinePruningInfo",
+    "Cases": [
+      {
+        "SQL": "select * from t where a > 1 order by f",
+        "Plan": [
+          "IndexLookUp_14 3333.33 139413.67 root  ",
+          "├─Selection_13(Build) 3333.33 0.00 cop[tikv]  gt(test.t.a, 1)",
+          "│ └─IndexFullScan_11 10000.00 555020.00 cop[tikv] table:t, index:f(f) keep order:true, stats:pseudo",
+          "└─TableRowIDScan_12(Probe) 3333.33 555020.00 cop[tikv] table:t keep order:false, stats:pseudo"
+        ],
+        "Warnings": [
+          "Note 1105 [t,f,f_g] remain after pruning paths for t given Prop{SortItems: [{test.t.f asc}], TaskTp: rootTask}"
+        ]
+      },
+      {
+        "SQL": "select * from t where f > 1",
+        "Plan": [
+          "TableReader_7 3333.33 88640.22 root  data:Selection_6",
+          "└─Selection_6 3333.33 1140020.00 cop[tikv]  gt(test.t.f, 1)",
+          "  └─TableFullScan_5 10000.00 1110020.00 cop[tikv] table:t keep order:false, stats:pseudo"
+        ],
+        "Warnings": [
+          "Note 1105 [t,f,f_g] remain after pruning paths for t given Prop{SortItems: [], TaskTp: rootTask}"
+        ]
+      },
+      {
+        "SQL": "select f from t where f > 1",
+        "Plan": [
+          "IndexReader_6 3333.33 11140.22 root  index:IndexRangeScan_5",
+          "└─IndexRangeScan_5 3333.33 140020.00 cop[tikv] table:t, index:f(f) range:(1,+inf], keep order:false, stats:pseudo"
+        ],
+        "Warnings": [
+          "Note 1105 [f,f_g] remain after pruning paths for t given Prop{SortItems: [], TaskTp: rootTask}"
+        ]
+      },
+      {
+        "SQL": "select * from t where f > 3 and g = 5",
+        "Plan": [
+          "IndexLookUp_15 3.33 215.74 root  ",
+          "├─IndexRangeScan_12(Build) 10.00 590.00 cop[tikv] table:t, index:g(g) range:[5,5], keep order:false, stats:pseudo",
+          "└─Selection_14(Probe) 3.33 0.00 cop[tikv]  gt(test.t.f, 3)",
+          "  └─TableRowIDScan_13 10.00 590.00 cop[tikv] table:t keep order:false, stats:pseudo"
+        ],
+        "Warnings": [
+          "Note 1105 [t,f_g,g] remain after pruning paths for t given Prop{SortItems: [], TaskTp: rootTask}"
+        ]
+      },
+      {
+        "SQL": "select * from t where g = 5 order by f",
+        "Plan": [
+          "Sort_5 10.00 362.68 root  test.t.f",
+          "└─IndexLookUp_13 10.00 239.01 root  ",
+          "  ├─IndexRangeScan_11(Build) 10.00 590.00 cop[tikv] table:t, index:g(g) range:[5,5], keep order:false, stats:pseudo",
+          "  └─TableRowIDScan_12(Probe) 10.00 590.00 cop[tikv] table:t keep order:false, stats:pseudo"
+        ],
+        "Warnings": [
+          "Note 1105 [t,g] remain after pruning paths for t given Prop{SortItems: [], TaskTp: rootTask}"
+        ]
+      },
+      {
+        "SQL": "select * from t where d = 3 order by c, e",
+        "Plan": [
+          "IndexLookUp_15 10.00 57230.78 root  ",
+          "├─Selection_14(Build) 10.00 0.00 cop[tikv]  eq(test.t.d, 3)",
+          "│ └─IndexFullScan_12 10000.00 825020.00 cop[tikv] table:t, index:c_d_e(c, d, e) keep order:true, stats:pseudo",
+          "└─TableRowIDScan_13(Probe) 10.00 825020.00 cop[tikv] table:t keep order:false, stats:pseudo"
+        ],
+        "Warnings": [
+          "Note 1105 [t,c_d_e] remain after pruning paths for t given Prop{SortItems: [{test.t.c asc} {test.t.e asc}], TaskTp: rootTask}"
+        ]
+      }
+    ]
   }
 ]

planner/property/physical_property.go

@@ -33,6 +33,13 @@ type SortItem struct {
 	Desc bool
 }
 
+func (s *SortItem) String() string {
+	if s.Desc {
+		return fmt.Sprintf("{%s desc}", s.Col)
+	}
+	return fmt.Sprintf("{%s asc}", s.Col)
+}
+
 // MPPPartitionType is the way to partition during mpp data exchanging.
 type MPPPartitionType int
 

sessionctx/stmtctx/stmtctx.go

@@ -181,6 +181,8 @@ type StatementContext struct {
 		LogOnExceed [2]memory.LogOnExceed
 	}
 
+	// OptimInfo maps Plan.ID() to optimization information when generating Plan.
+	OptimInfo map[int]string
 	// InVerboseExplain indicates the statement is "explain format='verbose' ...".
 	InVerboseExplain bool
 }