Planner: Do not allow cardinality to go below 1 | tidb-test=pr/2376

pkg/planner/cardinality/BUILD.bazel

@@ -26,6 +26,7 @@ go_library(
         "//pkg/planner/property",
         "//pkg/planner/util",
         "//pkg/planner/util/debugtrace",
+        "//pkg/planner/util/fixcontrol",
         "//pkg/sessionctx/stmtctx",
         "//pkg/statistics",
         "//pkg/tablecodec",

pkg/planner/cardinality/row_count_column.go

@@ -18,6 +18,7 @@ import (
 	"github.com/pingcap/errors"
 	"github.com/pingcap/tidb/pkg/planner/context"
 	"github.com/pingcap/tidb/pkg/planner/util/debugtrace"
+	"github.com/pingcap/tidb/pkg/planner/util/fixcontrol"
 	"github.com/pingcap/tidb/pkg/statistics"
 	"github.com/pingcap/tidb/pkg/types"
 	"github.com/pingcap/tidb/pkg/util/codec"
@@ -312,7 +313,17 @@ func GetColumnRowCount(sctx context.PlanContext, c *statistics.Column, ranges []
 		}
 		rowCount += cnt
 	}
-	rowCount = mathutil.Clamp(rowCount, 0, float64(realtimeRowCount))
+	allowZeroEst := fixcontrol.GetBoolWithDefault(
+		sctx.GetSessionVars().GetOptimizerFixControlMap(),
+		fixcontrol.Fix47400,
+		false,
+	)
+	if allowZeroEst {
+		rowCount = mathutil.Clamp(rowCount, 0, float64(realtimeRowCount))
+	} else {
+		// Don't allow the final result to go below 1 row
+		rowCount = mathutil.Clamp(rowCount, 1, float64(realtimeRowCount))
+	}
 	return rowCount, nil
 }
 

pkg/planner/cardinality/row_count_index.go

@@ -26,6 +26,7 @@ import (
 	"github.com/pingcap/tidb/pkg/kv"
 	"github.com/pingcap/tidb/pkg/planner/context"
 	"github.com/pingcap/tidb/pkg/planner/util/debugtrace"
+	"github.com/pingcap/tidb/pkg/planner/util/fixcontrol"
 	"github.com/pingcap/tidb/pkg/sessionctx/stmtctx"
 	"github.com/pingcap/tidb/pkg/statistics"
 	"github.com/pingcap/tidb/pkg/types"
@@ -350,7 +351,17 @@ func getIndexRowCountForStatsV2(sctx context.PlanContext, idx *statistics.Index,
 		}
 		totalCount += count
 	}
-	totalCount = mathutil.Clamp(totalCount, 0, float64(realtimeRowCount))
+	allowZeroEst := fixcontrol.GetBoolWithDefault(
+		sctx.GetSessionVars().GetOptimizerFixControlMap(),
+		fixcontrol.Fix47400,
+		false,
+	)
+	if allowZeroEst {
+		totalCount = mathutil.Clamp(totalCount, 0, float64(realtimeRowCount))
+	} else {
+		// Don't allow the final result to go below 1 row
+		totalCount = mathutil.Clamp(totalCount, 1, float64(realtimeRowCount))
+	}
 	return totalCount, nil
 }
 

pkg/planner/cardinality/selectivity_test.go

@@ -236,7 +236,7 @@ func TestEstimationForUnknownValues(t *testing.T) {
 	colID := table.Meta().Columns[0].ID
 	count, err := cardinality.GetRowCountByColumnRanges(sctx, &statsTbl.HistColl, colID, getRange(30, 30))
 	require.NoError(t, err)
-	require.Equal(t, 0.2, count)
+	require.Equal(t, 1.0, count)
 
 	count, err = cardinality.GetRowCountByColumnRanges(sctx, &statsTbl.HistColl, colID, getRange(9, 30))
 	require.NoError(t, err)
@@ -265,7 +265,7 @@ func TestEstimationForUnknownValues(t *testing.T) {
 	colID = table.Meta().Columns[0].ID
 	count, err = cardinality.GetRowCountByColumnRanges(sctx, &statsTbl.HistColl, colID, getRange(1, 30))
 	require.NoError(t, err)
-	require.Equal(t, 0.0, count)
+	require.Equal(t, 1.0, count)
 
 	testKit.MustExec("drop table t")
 	testKit.MustExec("create table t(a int, b int, index idx(b))")
@@ -278,7 +278,7 @@ func TestEstimationForUnknownValues(t *testing.T) {
 	colID = table.Meta().Columns[0].ID
 	count, err = cardinality.GetRowCountByColumnRanges(sctx, &statsTbl.HistColl, colID, getRange(2, 2))
 	require.NoError(t, err)
-	require.Equal(t, 0.0, count)
+	require.Equal(t, 1.0, count)
 
 	idxID = table.Meta().Indices[0].ID
 	count, err = cardinality.GetRowCountByIndexRanges(sctx, &statsTbl.HistColl, idxID, getRange(2, 2))
@@ -402,8 +402,8 @@ func TestSelectivity(t *testing.T) {
 		},
 		{
 			exprs:                    "a >= 1 and b > 1 and a < 2",
-			selectivity:              0.01783264746,
-			selectivityAfterIncrease: 0.01851851852,
+			selectivity:              0.017832647462277088,
+			selectivityAfterIncrease: 0.018518518518518517,
 		},
 		{
 			exprs:                    "a >= 1 and c > 1 and a < 2",
@@ -422,13 +422,13 @@ func TestSelectivity(t *testing.T) {
 		},
 		{
 			exprs:                    "b > 1",
-			selectivity:              0.96296296296,
+			selectivity:              0.9629629629629629,
 			selectivityAfterIncrease: 1,
 		},
 		{
 			exprs:                    "a > 1 and b < 2 and c > 3 and d < 4 and e > 5",
-			selectivity:              0,
-			selectivityAfterIncrease: 0,
+			selectivity:              5.870830440255832e-05,
+			selectivityAfterIncrease: 1.51329827770157e-05,
 		},
 		{
 			exprs:                    longExpr,
@@ -1190,8 +1190,8 @@ func TestCrossValidationSelectivity(t *testing.T) {
 	require.NoError(t, h.DumpStatsDeltaToKV(true))
 	tk.MustExec("analyze table t")
 	tk.MustQuery("explain format = 'brief' select * from t where a = 1 and b > 0 and b < 1000 and c > 1000").Check(testkit.Rows(
-		"TableReader 0.00 root  data:Selection",
-		"└─Selection 0.00 cop[tikv]  gt(test.t.c, 1000)",
+		"TableReader 1.00 root  data:Selection",
+		"└─Selection 1.00 cop[tikv]  gt(test.t.c, 1000)",
 		"  └─TableRangeScan 2.00 cop[tikv] table:t range:(1 0,1 1000), keep order:false"))
 }
 

pkg/planner/cardinality/testdata/cardinality_suite_out.json

@@ -24,7 +24,7 @@
       {
         "Start": 800,
         "End": 900,
-        "Count": 755.754166655054
+        "Count": 791.004166655054
       },
       {
         "Start": 900,
@@ -79,7 +79,7 @@
       {
         "Start": 800,
         "End": 1000,
-        "Count": 1213.946869573942
+        "Count": 1249.196869573942
       },
       {
         "Start": 900,
@@ -104,7 +104,7 @@
       {
         "Start": 200,
         "End": 400,
-        "Count": 1215.0288209899081
+        "Count": 1188.7788209899081
       },
       {
         "Start": 200,
@@ -2589,7 +2589,7 @@
                   },
                   {
                     "Name": "a",
-                    "Result": 0
+                    "Result": 1
                   }
                 ]
               },
@@ -2887,15 +2887,15 @@
                       },
                       {
                         "End estimate range": {
-                          "RowCount": 0,
+                          "RowCount": 1,
                           "Type": "Range"
                         }
                       }
                     ]
                   },
                   {
                     "Name": "iab",
-                    "Result": 0
+                    "Result": 1
                   }
                 ]
               },
@@ -3447,11 +3447,11 @@
                 "Expressions": [
                   "lt(test.t.a, -1500)"
                 ],
-                "Selectivity": 0,
+                "Selectivity": 0.0003246753246753247,
                 "partial cover": false
               },
               {
-                "Result": 0
+                "Result": 2.1082813290605499e-7
               }
             ]
           }
@@ -3940,7 +3940,7 @@
                   },
                   {
                     "Name": "iab",
-                    "Result": 0
+                    "Result": 1
                   }
                 ]
               },
@@ -4093,11 +4093,11 @@
                 "Expressions": [
                   "lt(test.t.a, -1500)"
                 ],
-                "Selectivity": 0,
+                "Selectivity": 0.0003246753246753247,
                 "partial cover": false
               },
               {
-                "Result": 0
+                "Result": 1.9066503965832828e-7
               }
             ]
           }

pkg/planner/core/casetest/cbotest/testdata/analyze_suite_out.json

@@ -364,13 +364,13 @@
         "└─TableRowIDScan(Probe) 2.00 cop[tikv] table:t keep order:false"
       ],
       [
-        "TableReader 0.00 root  data:Selection",
-        "└─Selection 0.00 cop[tikv]  eq(test.t.b, 1)",
+        "TableReader 1.00 root  data:Selection",
+        "└─Selection 1.00 cop[tikv]  eq(test.t.b, 1)",
         "  └─TableFullScan 2.00 cop[tikv] table:t keep order:false"
       ],
       [
-        "TableReader 0.00 root  data:Selection",
-        "└─Selection 0.00 cop[tikv]  lt(test.t.b, 1)",
+        "TableReader 1.00 root  data:Selection",
+        "└─Selection 1.00 cop[tikv]  lt(test.t.b, 1)",
         "  └─TableFullScan 2.00 cop[tikv] table:t keep order:false"
       ]
     ]

pkg/planner/core/casetest/partition/testdata/integration_partition_suite_out.json

@@ -709,7 +709,7 @@
           "└─IndexRangeScan 1.00 cop[tikv] table:t, index:b(b) range:[1,1], keep order:false"
         ],
         "StaticPlan": [
-          "PartitionUnion 1.00 root  ",
+          "PartitionUnion 3.00 root  ",
           "├─IndexReader 1.00 root  index:IndexRangeScan",
           "│ └─IndexRangeScan 1.00 cop[tikv] table:t, partition:P0, index:b(b) range:[1,1], keep order:false",
           "├─IndexReader 1.00 root  index:IndexRangeScan",
@@ -725,7 +725,7 @@
           "└─IndexRangeScan 1.00 cop[tikv] table:t, index:b(b) range:[2,2], keep order:false"
         ],
         "StaticPlan": [
-          "PartitionUnion 1.00 root  ",
+          "PartitionUnion 3.00 root  ",
           "├─IndexReader 1.00 root  index:IndexRangeScan",
           "│ └─IndexRangeScan 1.00 cop[tikv] table:t, partition:P0, index:b(b) range:[2,2], keep order:false",
           "├─IndexReader 1.00 root  index:IndexRangeScan",
@@ -741,7 +741,7 @@
           "└─IndexRangeScan 2.00 cop[tikv] table:t, index:b(b) range:[1,2], keep order:false"
         ],
         "StaticPlan": [
-          "PartitionUnion 2.00 root  ",
+          "PartitionUnion 3.00 root  ",
           "├─IndexReader 1.00 root  index:IndexRangeScan",
           "│ └─IndexRangeScan 1.00 cop[tikv] table:t, partition:P0, index:b(b) range:[1,2], keep order:false",
           "├─IndexReader 1.00 root  index:IndexRangeScan",
@@ -757,7 +757,7 @@
           "└─IndexRangeScan 2.00 cop[tikv] table:t, index:b(b) range:[2,2], [3,3], [4,4], keep order:false"
         ],
         "StaticPlan": [
-          "PartitionUnion 2.00 root  ",
+          "PartitionUnion 3.00 root  ",
           "├─IndexReader 1.00 root  index:IndexRangeScan",
           "│ └─IndexRangeScan 1.00 cop[tikv] table:t, partition:P0, index:b(b) range:[2,2], [3,3], [4,4], keep order:false",
           "├─IndexReader 1.00 root  index:IndexRangeScan",
@@ -773,7 +773,7 @@
           "└─IndexRangeScan 2.00 cop[tikv] table:t, index:b(b) range:[2,2], [3,3], keep order:false"
         ],
         "StaticPlan": [
-          "PartitionUnion 2.00 root  ",
+          "PartitionUnion 3.00 root  ",
           "├─IndexReader 1.00 root  index:IndexRangeScan",
           "│ └─IndexRangeScan 1.00 cop[tikv] table:t, partition:P0, index:b(b) range:[2,2], [3,3], keep order:false",
           "├─IndexReader 1.00 root  index:IndexRangeScan",
@@ -854,7 +854,7 @@
           "└─IndexRangeScan 1.00 cop[tikv] table:t, index:b(b) range:[1,1], keep order:false"
         ],
         "StaticPlan": [
-          "PartitionUnion 1.00 root  ",
+          "PartitionUnion 2.00 root  ",
           "├─IndexReader 1.00 root  index:IndexRangeScan",
           "│ └─IndexRangeScan 1.00 cop[tikv] table:t, partition:P0, index:b(b) range:[1,1], keep order:false",
           "└─IndexReader 1.00 root  index:IndexRangeScan",

pkg/planner/core/casetest/planstats/testdata/plan_stats_suite_out.json

@@ -116,10 +116,10 @@
       {
         "Query": "explain format = brief select * from t join tp where tp.a = 10 and t.b = tp.c",
         "Result": [
-          "Projection 0.00 root  test.t.a, test.t.b, test.t.c, test.tp.a, test.tp.b, test.tp.c",
-          "└─HashJoin 0.00 root  inner join, equal:[eq(test.tp.c, test.t.b)]",
-          "  ├─TableReader(Build) 0.00 root partition:p1 data:Selection",
-          "  │ └─Selection 0.00 cop[tikv]  eq(test.tp.a, 10), not(isnull(test.tp.c))",
+          "Projection 1.00 root  test.t.a, test.t.b, test.t.c, test.tp.a, test.tp.b, test.tp.c",
+          "└─HashJoin 1.00 root  inner join, equal:[eq(test.tp.c, test.t.b)]",
+          "  ├─TableReader(Build) 1.00 root partition:p1 data:Selection",
+          "  │ └─Selection 1.00 cop[tikv]  eq(test.tp.a, 10), not(isnull(test.tp.c))",
           "  │   └─TableFullScan 6.00 cop[tikv] table:tp keep order:false, stats:partial[c:allEvicted]",
           "  └─TableReader(Probe) 3.00 root  data:Selection",
           "    └─Selection 3.00 cop[tikv]  not(isnull(test.t.b))",

pkg/planner/core/casetest/testdata/integration_suite_out.json

@@ -93,25 +93,25 @@
         "SQL": "explain format = 'verbose' select count(*) from t3 where b = 0",
         "Plan": [
           "StreamAgg_10 1.00 64.98 root  funcs:count(1)->Column#4",
-          "└─IndexReader_15 0.00 15.08 root  index:IndexRangeScan_14",
-          "  └─IndexRangeScan_14 0.00 162.80 cop[tikv] table:t3, index:c(b) range:[0,0], keep order:false"
+          "└─IndexReader_15 1.00 15.08 root  index:IndexRangeScan_14",
+          "  └─IndexRangeScan_14 1.00 162.80 cop[tikv] table:t3, index:c(b) range:[0,0], keep order:false"
         ]
       },
       {
         "SQL": "explain format = 'verbose' select /*+ use_index(t3, c) */ count(a) from t3 where b = 0",
         "Plan": [
           "StreamAgg_10 1.00 2001.63 root  funcs:count(test.t3.a)->Column#4",
-          "└─IndexLookUp_17 0.00 1951.73 root  ",
-          "  ├─IndexRangeScan_15(Build) 0.00 203.50 cop[tikv] table:t3, index:c(b) range:[0,0], keep order:false",
-          "  └─TableRowIDScan_16(Probe) 0.00 227.31 cop[tikv] table:t3 keep order:false"
+          "└─IndexLookUp_17 1.00 1951.73 root  ",
+          "  ├─IndexRangeScan_15(Build) 1.00 203.50 cop[tikv] table:t3, index:c(b) range:[0,0], keep order:false",
+          "  └─TableRowIDScan_16(Probe) 1.00 227.31 cop[tikv] table:t3 keep order:false"
         ]
       },
       {
         "SQL": "explain format = 'verbose' select count(*) from t2 where a = 0",
         "Plan": [
           "StreamAgg_12 1.00 109.57 root  funcs:count(1)->Column#4",
-          "└─TableReader_20 0.00 59.67 root  data:Selection_19",
-          "  └─Selection_19 0.00 831.62 cop[tikv]  eq(test.t2.a, 0)",
+          "└─TableReader_20 1.00 59.67 root  data:Selection_19",
+          "  └─Selection_19 1.00 831.62 cop[tikv]  eq(test.t2.a, 0)",
           "    └─TableFullScan_18 3.00 681.92 cop[tikv] table:t2 keep order:false"
         ]
       },

pkg/planner/core/casetest/testdata/plan_normalized_suite_out.json

@@ -461,8 +461,8 @@
         "Plan": [
           " TableReader         root         ",
           " └─ExchangeSender    cop[tiflash] ",
-          "   └─Selection       cop[tiflash] gt(test.t1.b, ?), gt(test.t1.c, ?), or(gt(test.t1.a, ?), lt(test.t1.b, ?))",
-          "     └─TableFullScan cop[tiflash] table:t1, range:[?,?], pushed down filter:gt(test.t1.a, ?), keep order:false"
+          "   └─Selection       cop[tiflash] gt(test.t1.a, ?), gt(test.t1.c, ?), or(gt(test.t1.a, ?), lt(test.t1.b, ?))",
+          "     └─TableFullScan cop[tiflash] table:t1, range:[?,?], pushed down filter:gt(test.t1.b, ?), keep order:false"
         ]
       },
       {

pkg/planner/core/testdata/index_merge_suite_out.json

@@ -252,7 +252,7 @@
       {
         "SQL": "select * from vh",
         "Plan": [
-          "PartitionUnion 0.50 root  ",
+          "PartitionUnion 1.50 root  ",
           "├─IndexMerge 0.50 root  type: intersection",
           "│ ├─IndexRangeScan(Build) 2.00 cop[tikv] table:t1, partition:p0, index:ia(a) range:[10,10], keep order:false",
           "│ ├─IndexRangeScan(Build) 1.00 cop[tikv] table:t1, partition:p0, index:ibc(b, c) range:[20 -inf,20 30), keep order:false",
@@ -276,7 +276,7 @@
       {
         "SQL": "select /*+ qb_name(v, v), use_index_merge(@v t1, ia, ibc, id) */ * from v",
         "Plan": [
-          "PartitionUnion 0.50 root  ",
+          "PartitionUnion 1.50 root  ",
           "├─IndexMerge 0.50 root  type: intersection",
           "│ ├─IndexRangeScan(Build) 2.00 cop[tikv] table:t1, partition:p0, index:ia(a) range:[10,10], keep order:false",
           "│ ├─IndexRangeScan(Build) 1.00 cop[tikv] table:t1, partition:p0, index:ibc(b, c) range:[20 -inf,20 30), keep order:false",
@@ -300,7 +300,7 @@
       {
         "SQL": "select /*+ qb_name(v, v@sel_1), use_index_merge(@v t1, ia, ibc, id) */ * from v",
         "Plan": [
-          "PartitionUnion 0.50 root  ",
+          "PartitionUnion 1.50 root  ",
           "├─IndexMerge 0.50 root  type: intersection",
           "│ ├─IndexRangeScan(Build) 2.00 cop[tikv] table:t1, partition:p0, index:ia(a) range:[10,10], keep order:false",
           "│ ├─IndexRangeScan(Build) 1.00 cop[tikv] table:t1, partition:p0, index:ibc(b, c) range:[20 -inf,20 30), keep order:false",
@@ -324,7 +324,7 @@
       {
         "SQL": "select /*+ qb_name(v, v@sel_1 .@sel_1), use_index_merge(@v t1, ia, ibc, id) */ * from v",
         "Plan": [
-          "PartitionUnion 0.50 root  ",
+          "PartitionUnion 1.50 root  ",
           "├─IndexMerge 0.50 root  type: intersection",
           "│ ├─IndexRangeScan(Build) 2.00 cop[tikv] table:t1, partition:p0, index:ia(a) range:[10,10], keep order:false",
           "│ ├─IndexRangeScan(Build) 1.00 cop[tikv] table:t1, partition:p0, index:ibc(b, c) range:[20 -inf,20 30), keep order:false",
@@ -348,7 +348,7 @@
       {
         "SQL": "select /*+ qb_name(v, v@sel_1 .@sel_1), use_index_merge(@v t1, ia, ibc, id) */ * from v",
         "Plan": [
-          "PartitionUnion 0.50 root  ",
+          "PartitionUnion 1.50 root  ",
           "├─IndexMerge 0.50 root  type: intersection",
           "│ ├─IndexRangeScan(Build) 2.00 cop[tikv] table:t1, partition:p0, index:ia(a) range:[10,10], keep order:false",
           "│ ├─IndexRangeScan(Build) 1.00 cop[tikv] table:t1, partition:p0, index:ibc(b, c) range:[20 -inf,20 30), keep order:false",