Merge branch 'master' into fix-charset-upgrade

Makefile

@@ -13,7 +13,7 @@ export PATH := $(path_to_add):$(PATH)
 
 GO        := GO111MODULE=on go
 GOBUILD   := CGO_ENABLED=1 $(GO) build $(BUILD_FLAG)
-GOTEST    := CGO_ENABLED=1 $(GO) test -p 3
+GOTEST    := CGO_ENABLED=1 $(GO) test -p 4
 OVERALLS  := CGO_ENABLED=1 GO111MODULE=on overalls
 
 ARCH      := "`uname -s`"
@@ -128,8 +128,12 @@ ifeq ("$(TRAVIS_COVERAGE)", "1")
 	@echo "Running in TRAVIS_COVERAGE mode."
 	@export log_level=error; \
 	$(GO) get github.com/go-playground/overalls
-	$(OVERALLS) -project=github.com/pingcap/tidb -covermode=count -ignore='.git,vendor,cmd,docs,LICENSES,ddl/failtest,ddl/testutil/,executor/esqtest' \
-			-concurrency=1 || { $(GOFAIL_DISABLE); exit 1; }
+	$(OVERALLS) -project=github.com/pingcap/tidb \
+			-covermode=count \
+			-ignore='.git,vendor,cmd,docs,LICENSES' \
+			-concurrency=2 \
+			-- -coverpkg=./... \
+			|| { $(GOFAIL_DISABLE); exit 1; }
 else
 	@echo "Running in native mode."
 	@export log_level=error; \

cmd/importer/main.go

@@ -18,7 +18,8 @@ import (
 	"os"
 
 	"github.com/pingcap/errors"
-	log "github.com/sirupsen/logrus"
+	"github.com/pingcap/log"
+	"go.uber.org/zap"
 )
 
 func main() {
@@ -29,31 +30,31 @@ func main() {
 	case flag.ErrHelp:
 		os.Exit(0)
 	default:
-		log.Errorf("parse cmd flags err %s\n", err)
+		log.Error("parse cmd flags", zap.Error(err))
 		os.Exit(2)
 	}
 
 	table := newTable()
 	err = parseTableSQL(table, cfg.DDLCfg.TableSQL)
 	if err != nil {
-		log.Fatal(err)
+		log.Fatal(err.Error())
 	}
 
 	err = parseIndexSQL(table, cfg.DDLCfg.IndexSQL)
 	if err != nil {
-		log.Fatal(err)
+		log.Fatal(err.Error())
 	}
 
 	dbs, err := createDBs(cfg.DBCfg, cfg.SysCfg.WorkerCount)
 	if err != nil {
-		log.Fatal(err)
+		log.Fatal(err.Error())
 	}
 	defer closeDBs(dbs)
 
 	if len(cfg.StatsCfg.Path) > 0 {
 		statsInfo, err1 := loadStats(table.tblInfo, cfg.StatsCfg.Path)
 		if err1 != nil {
-			log.Fatal(err1)
+			log.Fatal(err1.Error())
 		}
 		for _, idxInfo := range table.tblInfo.Indices {
 			offset := idxInfo.Columns[0].Offset
@@ -75,12 +76,12 @@ func main() {
 
 	err = execSQL(dbs[0], cfg.DDLCfg.TableSQL)
 	if err != nil {
-		log.Fatal(err)
+		log.Fatal(err.Error())
 	}
 
 	err = execSQL(dbs[0], cfg.DDLCfg.IndexSQL)
 	if err != nil {
-		log.Fatal(err)
+		log.Fatal(err.Error())
 	}
 
 	doProcess(table, dbs, cfg.SysCfg.JobCount, cfg.SysCfg.WorkerCount, cfg.SysCfg.Batch)

cmd/importer/rand.go

@@ -18,7 +18,8 @@ import (
 	"math/rand"
 	"time"
 
-	log "github.com/sirupsen/logrus"
+	"github.com/pingcap/log"
+	"go.uber.org/zap"
 )
 
 const (
@@ -80,7 +81,7 @@ func randDate(col *column) string {
 
 	minTime, err := time.Parse(dateFormat, min)
 	if err != nil {
-		log.Warnf("randDate err %s", err)
+		log.Warn("parse min date failed", zap.Error(err))
 	}
 	if len(max) == 0 {
 		t := minTime.Add(time.Duration(randInt(0, 365)) * 24 * time.Hour)
@@ -89,7 +90,7 @@ func randDate(col *column) string {
 
 	maxTime, err := time.Parse(dateFormat, max)
 	if err != nil {
-		log.Warnf("randDate err %s", err)
+		log.Warn("parse max date failed", zap.Error(err))
 	}
 	days := int(maxTime.Sub(minTime).Hours() / 24)
 	t := minTime.Add(time.Duration(randInt(0, days)) * 24 * time.Hour)
@@ -110,11 +111,11 @@ func randTime(col *column) string {
 
 	minTime, err := time.Parse(timeFormat, min)
 	if err != nil {
-		log.Warnf("randTime err %s", err)
+		log.Warn("parse min time failed", zap.Error(err))
 	}
 	maxTime, err := time.Parse(timeFormat, max)
 	if err != nil {
-		log.Warnf("randTime err %s", err)
+		log.Warn("parse max time failed", zap.Error(err))
 	}
 	seconds := int(maxTime.Sub(minTime).Seconds())
 	t := minTime.Add(time.Duration(randInt(0, seconds)) * time.Second)
@@ -138,7 +139,7 @@ func randTimestamp(col *column) string {
 
 	minTime, err := time.Parse(dateTimeFormat, min)
 	if err != nil {
-		log.Warnf("randTimestamp err %s", err)
+		log.Warn("parse min timestamp failed", zap.Error(err))
 	}
 	if len(max) == 0 {
 		t := minTime.Add(time.Duration(randInt(0, 365)) * 24 * time.Hour)
@@ -147,7 +148,7 @@ func randTimestamp(col *column) string {
 
 	maxTime, err := time.Parse(dateTimeFormat, max)
 	if err != nil {
-		log.Warnf("randTimestamp err %s", err)
+		log.Warn("parse max timestamp failed", zap.Error(err))
 	}
 	seconds := int(maxTime.Sub(minTime).Seconds())
 	t := minTime.Add(time.Duration(randInt(0, seconds)) * time.Second)
@@ -165,11 +166,11 @@ func randYear(col *column) string {
 
 	minTime, err := time.Parse(yearFormat, min)
 	if err != nil {
-		log.Warnf("randYear err %s", err)
+		log.Warn("parse min year failed", zap.Error(err))
 	}
 	maxTime, err := time.Parse(yearFormat, max)
 	if err != nil {
-		log.Warnf("randYear err %s", err)
+		log.Warn("parse max year failed", zap.Error(err))
 	}
 	seconds := int(maxTime.Sub(minTime).Seconds())
 	t := minTime.Add(time.Duration(randInt(0, seconds)) * time.Second)

cmd/pluginpkg/pluginpkg.go

@@ -62,9 +62,18 @@ func PluginManifest() *plugin.Manifest {
 				},
 				{{end}}
 			},
-			Validate:   {{.validate}},
-			OnInit:     {{.onInit}},
-			OnShutdown: {{.onShutdown}},
+			{{if .validate }}
+				Validate:   {{.validate}},
+			{{end}}
+			{{if .onInit }}
+				OnInit:     {{.onInit}},
+			{{end}}
+			{{if .onShutdown }}
+				OnShutdown: {{.onShutdown}},
+			{{end}}
+			{{if .onFlush }}
+				OnFlush:    {{.onFlush}},
+			{{end}}
 		},
 		{{range .export}}
 		{{.extPoint}}: {{.impl}},

config/config.go

@@ -254,13 +254,14 @@ type TiKVClient struct {
 // Binlog is the config for binlog.
 type Binlog struct {
 	Enable       bool   `toml:"enable" json:"enable"`
-	AutoMode     bool   `toml:"auto-mode" json:"auto-mode"`
 	WriteTimeout string `toml:"write-timeout" json:"write-timeout"`
 	// If IgnoreError is true, when writing binlog meets error, TiDB would
 	// ignore the error.
 	IgnoreError bool `toml:"ignore-error" json:"ignore-error"`
 	// Use socket file to write binlog, for compatible with kafka version tidb-binlog.
 	BinlogSocket string `toml:"binlog-socket" json:"binlog-socket"`
+	// The strategy for sending binlog to pump, value can be "range" or "hash" now.
+	Strategy string `toml:"strategy" json:"strategy"`
 }
 
 // Plugin is the config for plugin
@@ -349,6 +350,7 @@ var defaultConf = Config{
 	},
 	Binlog: Binlog{
 		WriteTimeout: "15s",
+		Strategy:     "range",
 	},
 }
 

config/config.toml.example

@@ -264,12 +264,9 @@ enabled = true
 capacity = 2048000
 
 [binlog]
-# Enable to write binlog. This config will be disabled if auto-mode is true.
+# enable to write binlog.
 enable = false
 
-# If auto-mode is true, will enable binlog according to the system variables 'tidb_log_bin'.
-auto-mode = false
-
 # WriteTimeout specifies how long it will wait for writing binlog to pump.
 write-timeout = "15s"
 
@@ -279,3 +276,6 @@ ignore-error = false
 
 # use socket file to write binlog, for compatible with kafka version tidb-binlog.
 binlog-socket = ""
+
+# the strategy for sending binlog to pump, value can be "range" or "hash" now.
+strategy = "range"

config/config_test.go

@@ -34,8 +34,8 @@ func TestT(t *testing.T) {
 func (s *testConfigSuite) TestConfig(c *C) {
 	conf := new(Config)
 	conf.Binlog.Enable = true
-	conf.Binlog.AutoMode = true
 	conf.Binlog.IgnoreError = true
+	conf.Binlog.Strategy = "hash"
 	conf.TiKVClient.CommitTimeout = "10s"
 	conf.CheckMb4ValueInUTF8 = true
 	configFile := "config.toml"
@@ -56,7 +56,7 @@ max-batch-size=128
 
 	// Test that the original value will not be clear by load the config file that does not contain the option.
 	c.Assert(conf.Binlog.Enable, Equals, true)
-	c.Assert(conf.Binlog.AutoMode, Equals, true)
+	c.Assert(conf.Binlog.Strategy, Equals, "hash")
 
 	c.Assert(conf.TiKVClient.CommitTimeout, Equals, "41s")
 	c.Assert(conf.TiKVClient.MaxBatchSize, Equals, uint(128))

ddl/column.go

@@ -591,3 +591,12 @@ func findColumnInIndexCols(c *expression.Column, cols []*ast.IndexColName) bool
 	}
 	return false
 }
+
+func getColumnInfoByName(tbInfo *model.TableInfo, column string) *model.ColumnInfo {
+	for _, colInfo := range tbInfo.Cols() {
+		if colInfo.Name.L == column {
+			return colInfo
+		}
+	}
+	return nil
+}

ddl/db_partition_test.go

@@ -25,6 +25,7 @@ import (
 	"github.com/pingcap/errors"
 	"github.com/pingcap/parser/model"
 	tmysql "github.com/pingcap/parser/mysql"
+	"github.com/pingcap/parser/terror"
 	"github.com/pingcap/tidb/ddl"
 	"github.com/pingcap/tidb/ddl/testutil"
 	"github.com/pingcap/tidb/domain"
@@ -115,9 +116,9 @@ func (s *testIntegrationSuite) TestCreateTableWithPartition(c *C) {
 	assertErrorCode(c, tk, sql4, tmysql.ErrPartitionMaxvalue)
 
 	_, err = tk.Exec(`CREATE TABLE rc (
-    		a INT NOT NULL,
-    		b INT NOT NULL,
-			c INT NOT NULL
+		a INT NOT NULL,
+		b INT NOT NULL,
+		c INT NOT NULL
 	)
 	partition by range columns(a,b,c) (
     	partition p0 values less than (10,5,1),
@@ -217,22 +218,14 @@ func (s *testIntegrationSuite) TestCreateTableWithPartition(c *C) {
 		  partition p4 values less than (18446744073709551000 + 10)
 		);`)
 
-	// Only range type partition is now supported, range columns partition only implements the parser, so it will not be checked.
-	// So the following SQL statements can be executed successfully.
-	tk.MustExec(`create table t29 (
-		a decimal
-	)
-	partition by range columns (a)
-	(partition p0 values less than (0));`)
-
 	tk.MustExec("set @@tidb_enable_table_partition = 1")
 	_, err = tk.Exec(`create table t30 (
 		  a int,
 		  b float,
 		  c varchar(30))
 		  partition by range columns (a, b)
 		  (partition p0 values less than (10, 10.0))`)
-	c.Assert(ddl.ErrUnsupportedPartitionByRangeColumns.Equal(err), IsTrue)
+	c.Assert(ddl.ErrNotAllowedTypeInPartition.Equal(err), IsTrue)
 
 	assertErrorCode(c, tk, `create table t31 (a int not null) partition by range( a );`, tmysql.ErrPartitionsMustBeDefined)
 	assertErrorCode(c, tk, `create table t32 (a int not null) partition by range columns( a );`, tmysql.ErrPartitionsMustBeDefined)
@@ -304,6 +297,70 @@ create table log_message_1 (
 		store_id int
 	)
 	partition by hash( year(hired) ) partitions 4;`)
+
+	tk.MustExec("drop table if exists t")
+
+	type testCase struct {
+		sql string
+		err *terror.Error
+	}
+
+	cases := []testCase{
+		{
+			"create table t (id int) partition by range columns (id);",
+			ddl.ErrPartitionsMustBeDefined,
+		},
+		{
+			"create table t (id int) partition by range columns (id) (partition p0 values less than (1, 2));",
+			ddl.ErrPartitionColumnList,
+		},
+		{
+			"create table t (a int) partition by range columns (b) (partition p0 values less than (1, 2));",
+			ddl.ErrFieldNotFoundPart,
+		},
+		{
+			"create table t (id timestamp) partition by range columns (id) (partition p0 values less than ('2019-01-09 11:23:34'));",
+			ddl.ErrNotAllowedTypeInPartition,
+		},
+		{
+			`create table t29 (
+				a decimal
+			)
+			partition by range columns (a)
+			(partition p0 values less than (0));`,
+			ddl.ErrNotAllowedTypeInPartition,
+		},
+		{
+			"create table t (id text) partition by range columns (id) (partition p0 values less than ('abc'));",
+			ddl.ErrNotAllowedTypeInPartition,
+		},
+		{
+			"create table t (a int, b varchar(64)) partition by range columns (a, b) (" +
+				"partition p0 values less than (1, 'a')," +
+				"partition p1 values less than (1, 'a'))",
+			ddl.ErrRangeNotIncreasing,
+		},
+		{
+			"create table t (a int, b varchar(64)) partition by range columns (a, b) (" +
+				"partition p0 values less than (1, 'b')," +
+				"partition p1 values less than (1, 'a'))",
+			ddl.ErrRangeNotIncreasing,
+		},
+		{
+			"create table t (a int, b varchar(64)) partition by range columns (a, b) (" +
+				"partition p0 values less than (1, maxvalue)," +
+				"partition p1 values less than (1, 'a'))",
+			ddl.ErrRangeNotIncreasing,
+		},
+	}
+	for i, t := range cases {
+		_, err := tk.Exec(t.sql)
+		c.Assert(t.err.Equal(err), IsTrue, Commentf("case %d fail, sql = %s", i, t.sql))
+	}
+
+	tk.MustExec("create table t1 (a int, b char(3)) partition by range columns (a, b) (" +
+		"partition p0 values less than (1, 'a')," +
+		"partition p1 values less than (2, maxvalue))")
 }
 
 func (s *testIntegrationSuite) TestCreateTableWithKeyPartition(c *C) {