Merge branch 'hash-agg-final-inpu-parallel' of https://github.com/win…

…oros/tidb into hash-agg-final-inpu-parallel

.github/CODEOWNERS

@@ -1 +1,5 @@
 /expression @pingcap/co-expression
+/planner @pingcap/co-planner
+/statistics @pingcap/co-planner
+/util/ranger  @pingcap/co-planner
+/bindinfo @pingcap/co-planner

CONTRIBUTORS.md

@@ -1 +1 @@
-Contributor list is moved to [Contributors](https://github.com/pingcap/community/blob/master/contributors#tidb-contributors)
+Contributor list is moved to [Contributors](https://github.com/pingcap/community/blob/master/architecture/contributor-list.md#tidb-contributors)

Makefile

@@ -1,5 +1,6 @@
 PROJECT=tidb
 GOPATH ?= $(shell go env GOPATH)
+P=8
 
 # Ensure GOPATH is set before running build process.
 ifeq "$(GOPATH)" ""
@@ -14,7 +15,7 @@ export PATH := $(path_to_add):$(PATH)
 GO              := GO111MODULE=on go
 GOBUILD         := $(GO) build $(BUILD_FLAG) -tags codes
 GOBUILDCOVERAGE := GOPATH=$(GOPATH) cd tidb-server; $(GO) test -coverpkg="../..." -c .
-GOTEST          := $(GO) test -p 8
+GOTEST          := $(GO) test -p $(P)
 OVERALLS        := GO111MODULE=on overalls
 
 ARCH      := "`uname -s`"

bindinfo/bind_test.go

@@ -528,3 +528,25 @@ func (s *testSuite) TestAddEvolveTasks(c *C) {
 	status := rows[1][3].(string)
 	c.Assert(status == "using" || status == "rejected", IsTrue)
 }
+
+func (s *testSuite) TestBindingCache(c *C) {
+	tk := testkit.NewTestKit(c, s.store)
+	s.cleanBindingEnv(tk)
+	tk.MustExec("use test")
+	tk.MustExec("drop table if exists t")
+	tk.MustExec("create table t(a int, b int, index idx(a))")
+	tk.MustExec("create global binding for select * from t using select * from t use index(idx)")
+	tk.MustExec("create database tmp")
+	tk.MustExec("use tmp")
+	tk.MustExec("create table t(a int, b int, index idx(a))")
+	tk.MustExec("create global binding for select * from t using select * from t use index(idx)")
+
+	c.Assert(s.domain.BindHandle().Update(false), IsNil)
+	c.Assert(s.domain.BindHandle().Update(false), IsNil)
+	res := tk.MustQuery("show global bindings")
+	c.Assert(len(res.Rows()), Equals, 2)
+
+	tk.MustExec("drop global binding for select * from t")
+	c.Assert(s.domain.BindHandle().Update(false), IsNil)
+	c.Assert(len(s.domain.BindHandle().GetAllBindRecord()), Equals, 1)
+}

bindinfo/handle.go

@@ -126,7 +126,7 @@ func (h *BindHandle) Update(fullLoad bool) (err error) {
 
 	sql := "select original_sql, bind_sql, default_db, status, create_time, update_time, charset, collation from mysql.bind_info"
 	if !fullLoad {
-		sql += " where update_time >= \"" + lastUpdateTime.String() + "\""
+		sql += " where update_time > \"" + lastUpdateTime.String() + "\""
 	}
 	// We need to apply the updates by order, wrong apply order of same original sql may cause inconsistent state.
 	sql += " order by update_time"
@@ -154,7 +154,7 @@ func (h *BindHandle) Update(fullLoad bool) (err error) {
 			lastUpdateTime = meta.Bindings[0].UpdateTime
 		}
 		if err != nil {
-			logutil.BgLogger().Error("update bindinfo failed", zap.Error(err))
+			logutil.BgLogger().Info("update bindinfo failed", zap.Error(err))
 			continue
 		}
 
@@ -163,7 +163,7 @@ func (h *BindHandle) Update(fullLoad bool) (err error) {
 		if len(newRecord.Bindings) > 0 {
 			newCache.setBindRecord(hash, newRecord)
 		} else {
-			newCache.removeDeletedBindRecord(hash, oldRecord)
+			newCache.removeDeletedBindRecord(hash, newRecord)
 		}
 		updateMetrics(metrics.ScopeGlobal, oldRecord, newCache.getBindRecord(hash, meta.OriginalSQL, meta.Db), true)
 	}
@@ -459,6 +459,7 @@ func (c cache) removeDeletedBindRecord(hash string, meta *BindRecord) {
 			}
 		}
 	}
+	c[hash] = metas
 }
 
 func (c cache) setBindRecord(hash string, meta *BindRecord) {

checkout-pr-branch.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # This script is used to checkout a TiDB PR branch in a forked repo.
 if test -z $1; then

cmd/explaintest/r/access_tiflash.result

@@ -40,3 +40,18 @@ id	count	task	operator info
 TableReader_7	44.00	root	data:Selection_6
 └─Selection_6	44.00	cop[tiflash]	or(and(gt(test.tt.a, 1), lt(test.tt.a, 20)), and(ge(test.tt.a, 30), lt(test.tt.a, 55)))
   └─TableScan_5	44.00	cop[tiflash]	table:tt, range:[-inf,+inf], keep order:false, stats:pseudo
+drop table if exists ttt;
+create table ttt (a int, primary key (a desc));
+desc select * from ttt order by ttt.a desc;
+id	count	task	operator info
+TableReader_11	10000.00	root	data:TableScan_10
+└─TableScan_10	10000.00	cop[tikv]	table:ttt, range:[-inf,+inf], keep order:true, desc, stats:pseudo
+desc select /*+ read_from_storage(tiflash[ttt]) */ * from ttt order by ttt.a desc;
+id	count	task	operator info
+Sort_4	10000.00	root	test.ttt.a:desc
+└─TableReader_8	10000.00	root	data:TableScan_7
+  └─TableScan_7	10000.00	cop[tiflash]	table:ttt, range:[-inf,+inf], keep order:false, stats:pseudo
+desc select /*+ read_from_storage(tiflash[ttt]) */ * from ttt order by ttt.a;
+id	count	task	operator info
+TableReader_11	10000.00	root	data:TableScan_10
+└─TableScan_10	10000.00	cop[tiflash]	table:ttt, range:[-inf,+inf], keep order:true, stats:pseudo

cmd/explaintest/r/explain_easy.result

@@ -79,9 +79,9 @@ TopN_7	1.00	root	test.t2.c2:asc, offset:0, count:1
     └─TableScan_13	10000.00	cop[tikv]	table:t2, range:[-inf,+inf], keep order:false, stats:pseudo
 explain select * from t1 where c1 > 1 and c2 = 1 and c3 < 1;
 id	count	task	operator info
-IndexLookUp_11	1.11	root	
+IndexLookUp_11	11.08	root	
 ├─IndexScan_8	33.33	cop[tikv]	table:t1, index:c2, range:(1 1,1 +inf], keep order:false, stats:pseudo
-└─Selection_10	1.11	cop[tikv]	lt(test.t1.c3, 1)
+└─Selection_10	11.08	cop[tikv]	lt(test.t1.c3, 1)
   └─TableScan_9	33.33	cop[tikv]	table:t1, keep order:false, stats:pseudo
 explain select * from t1 where c1 = 1 and c2 > 1;
 id	count	task	operator info

cmd/explaintest/run-tests.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 TIDB_TEST_STORE_NAME=$TIDB_TEST_STORE_NAME
 TIKV_PATH=$TIKV_PATH

cmd/explaintest/t/access_tiflash.test

@@ -15,4 +15,11 @@ desc select /*+ read_from_storage(tiflash[t]) */ sum(isnull(a)) from t;
 create table tt(a int, b int, primary key(a));
 
 desc select * from tt where (tt.a > 1 and tt.a < 20) or (tt.a >= 30 and tt.a < 55);
-desc select /*+ read_from_storage(tiflash[tt]) */ * from tt where (tt.a > 1 and tt.a < 20) or (tt.a >= 30 and tt.a < 55);
+desc select /*+ read_from_storage(tiflash[tt]) */ * from tt where (tt.a > 1 and tt.a < 20) or (tt.a >= 30 and tt.a < 55);
+
+drop table if exists ttt;
+create table ttt (a int, primary key (a desc));
+
+desc select * from ttt order by ttt.a desc;
+desc select /*+ read_from_storage(tiflash[ttt]) */ * from ttt order by ttt.a desc;
+desc select /*+ read_from_storage(tiflash[ttt]) */ * from ttt order by ttt.a;

config/config.go

@@ -29,6 +29,7 @@ import (
 	"github.com/BurntSushi/toml"
 	"github.com/pingcap/errors"
 	zaplog "github.com/pingcap/log"
+	"github.com/pingcap/parser/mysql"
 	"github.com/pingcap/tidb/util/logutil"
 	tracing "github.com/uber/jaeger-client-go/config"
 	"go.uber.org/atomic"
@@ -38,7 +39,7 @@ import (
 const (
 	MaxLogFileSize = 4096 // MB
 	// DefTxnTotalSizeLimit is the default value of TxnTxnTotalSizeLimit.
-	DefTxnTotalSizeLimit = 100 * 1024 * 1024
+	DefTxnTotalSizeLimit = 1024 * 1024 * 1024
 )
 
 // Valid config maps
@@ -74,8 +75,8 @@ type Config struct {
 	TxnLocalLatches  TxnLocalLatches `toml:"txn-local-latches" json:"txn-local-latches"`
 	// Set sys variable lower-case-table-names, ref: https://dev.mysql.com/doc/refman/5.7/en/identifier-case-sensitivity.html.
 	// TODO: We actually only support mode 2, which keeps the original case, but the comparison is case-insensitive.
-	LowerCaseTableNames int `toml:"lower-case-table-names" json:"lower-case-table-names"`
-
+	LowerCaseTableNames int               `toml:"lower-case-table-names" json:"lower-case-table-names"`
+	ServerVersion       string            `toml:"server-version" json:"server-version"`
 	Log                 Log               `toml:"log" json:"log"`
 	Security            Security          `toml:"security" json:"security"`
 	Status              Status            `toml:"status" json:"status"`
@@ -100,6 +101,9 @@ type Config struct {
 	DelayCleanTableLock uint64      `toml:"delay-clean-table-lock" json:"delay-clean-table-lock"`
 	SplitRegionMaxNum   uint64      `toml:"split-region-max-num" json:"split-region-max-num"`
 	StmtSummary         StmtSummary `toml:"stmt-summary" json:"stmt-summary"`
+	// RepairMode indicates that the TiDB is in the repair mode for table meta.
+	RepairMode      bool     `toml:"repair-mode" json:"repair-mode"`
+	RepairTableList []string `toml:"repair-table-list" json:"repair-table-list"`
 }
 
 // nullableBool defaults unset bool options to unset instead of false, which enables us to know if the user has set 2
@@ -440,11 +444,14 @@ var defaultConf = Config{
 	EnableTableLock:              false,
 	DelayCleanTableLock:          0,
 	SplitRegionMaxNum:            1000,
+	RepairMode:                   false,
+	RepairTableList:              []string{},
 	TxnLocalLatches: TxnLocalLatches{
 		Enabled:  false,
 		Capacity: 2048000,
 	},
 	LowerCaseTableNames: 2,
+	ServerVersion:       "",
 	Log: Log{
 		Level:               "info",
 		Format:              "text",
@@ -638,6 +645,9 @@ func (c *Config) Load(confFile string) error {
 	if c.TokenLimit == 0 {
 		c.TokenLimit = 1000
 	}
+	if len(c.ServerVersion) > 0 {
+		mysql.ServerVersion = c.ServerVersion
+	}
 	// If any items in confFile file are not mapped into the Config struct, issue
 	// an error and stop the server from starting.
 	undecoded := metaData.Undecoded()
@@ -700,6 +710,10 @@ func (c *Config) Valid() error {
 	if c.TiKVClient.GrpcConnectionCount == 0 {
 		return fmt.Errorf("grpc-connection-count should be greater than 0")
 	}
+
+	if c.Performance.TxnTotalSizeLimit > (10 << 30) {
+		return fmt.Errorf("txn-total-size-limit should be less than %d", 10<<30)
+	}
 	return nil
 }
 

config/config.toml.example

@@ -74,6 +74,19 @@ split-region-max-num = 1000
 # In order to support "drop primary key" operation , this flag must be true and the table does not have the pkIsHandle flag.
 alter-primary-key = false
 
+# server-version is used to change the version string of TiDB in the following scenarios:
+# 1. the server version returned by builtin-function `VERSION()`.
+# 2. the server version filled in handshake packets of MySQL Connection Protocol, see https://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::Handshake for more details.
+# if server-version = "", the default value(original TiDB version string) is used.
+server-version = ""
+
+# repair mode is used to repair the broken table meta in TiKV in extreme cases.
+repair-mode = false
+
+# Repair table list is used to list the tables in repair mode with the format like ["db.table",].
+# In repair mode, repairing table which is not in repair list will get wrong database or wrong table error.
+repair-table-list = []
+
 [log]
 # Log level: debug, info, warn, error, fatal.
 level = "info"
@@ -201,7 +214,7 @@ bind-info-lease = "3s"
 # If using TiKV as the storage, the entry represents a key/value pair.
 # WARNING: Do not set the value too large, otherwise it will make a very large impact on the TiKV cluster.
 # Please adjust this configuration carefully.
-txn-total-size-limit = 104857600
+txn-total-size-limit = 1073741824
 
 [proxy-protocol]
 # PROXY protocol acceptable client networks.

config/config_test.go

@@ -23,6 +23,7 @@ import (
 	"github.com/BurntSushi/toml"
 	. "github.com/pingcap/check"
 	zaplog "github.com/pingcap/log"
+	"github.com/pingcap/parser/mysql"
 	"github.com/pingcap/tidb/util/logutil"
 	tracing "github.com/uber/jaeger-client-go/config"
 )
@@ -179,6 +180,8 @@ alter-primary-key = true
 delay-clean-table-lock = 5
 split-region-max-num=10000
 enable-batch-dml = true
+server-version = "test_version"
+repair-mode = true
 [performance]
 txn-total-size-limit=2000
 [tikv-client]
@@ -196,6 +199,8 @@ max-sql-length=1024
 
 	c.Assert(conf.Load(configFile), IsNil)
 
+	c.Assert(conf.ServerVersion, Equals, "test_version")
+	c.Assert(mysql.ServerVersion, Equals, conf.ServerVersion)
 	// Test that the original value will not be clear by load the config file that does not contain the option.
 	c.Assert(conf.Binlog.Enable, Equals, true)
 	c.Assert(conf.Binlog.Strategy, Equals, "hash")
@@ -215,6 +220,7 @@ max-sql-length=1024
 	c.Assert(conf.StmtSummary.MaxStmtCount, Equals, uint(1000))
 	c.Assert(conf.StmtSummary.MaxSQLLength, Equals, uint(1024))
 	c.Assert(conf.EnableBatchDML, Equals, true)
+	c.Assert(conf.RepairMode, Equals, true)
 	c.Assert(f.Close(), IsNil)
 	c.Assert(os.Remove(configFile), IsNil)
 
@@ -224,7 +230,7 @@ max-sql-length=1024
 	// Make sure the example config is the same as default config.
 	c.Assert(conf, DeepEquals, GetGlobalConfig())
 
-	// Test for lof config.
+	// Test for log config.
 	c.Assert(conf.Log.ToLogConfig(), DeepEquals, logutil.NewLogConfig("info", "text", "tidb-slow.log", conf.Log.File, false, func(config *zaplog.Config) { config.DisableErrorVerbose = conf.Log.getDisableErrorStack() }))
 
 	// Test for tracing config.
@@ -349,3 +355,20 @@ func (s *testConfigSuite) TestOOMActionValid(c *C) {
 		c.Assert(c1.Valid() == nil, Equals, tt.valid)
 	}
 }
+
+func (s *testConfigSuite) TestTxnTotalSizeLimitValid(c *C) {
+	conf := NewConfig()
+	tests := []struct {
+		limit uint64
+		valid bool
+	}{
+		{4 << 10, true},
+		{10 << 30, true},
+		{10<<30 + 1, false},
+	}
+
+	for _, tt := range tests {
+		conf.Performance.TxnTotalSizeLimit = tt.limit
+		c.Assert(conf.Valid() == nil, Equals, tt.valid)
+	}
+}

ddl/column_test.go

@@ -800,7 +800,7 @@ func (s *testColumnSuite) TestAddColumn(c *C) {
 			hookErr = errors.Trace(err1)
 			return
 		}
-		newCol := table.FindCol(t.(*tables.Table).Columns, newColName)
+		newCol := table.FindCol(t.(*tables.TableCommon).Columns, newColName)
 		if newCol == nil {
 			return
 		}
@@ -891,7 +891,7 @@ func (s *testColumnSuite) TestDropColumn(c *C) {
 			hookErr = errors.Trace(err1)
 			return
 		}
-		col := table.FindCol(t.(*tables.Table).Columns, colName)
+		col := table.FindCol(t.(*tables.TableCommon).Columns, colName)
 		if col == nil {
 			checkOK = true
 			return