Merge branch 'master' into msc-framework

BUILD.bazel

@@ -4,6 +4,7 @@ load("@bazel_gazelle//:def.bzl", "gazelle")
 # gazelle:prefix github.com/pingcap/tidb
 # gazelle:exclude tidb-binlog/proto/proto
 # gazelle:exclude plugin/conn_ip_example
+# gazelle:exclude build/linter/staticcheck
 gazelle(name = "gazelle")
 
 package(default_visibility = ["//visibility:public"])

DEPS.bzl

@@ -5,8 +5,9 @@ def go_deps():
         name = "co_honnef_go_tools",
         build_file_proto_mode = "disable_global",
         importpath = "honnef.co/go/tools",
-        sum = "h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8=",
-        version = "v0.0.1-2020.1.4",
+        replace = "honnef.co/go/tools",
+        sum = "h1:ytYb4rOqyp1TSa2EPvNVwtPQJctSELKaMyLfqNP4+34=",
+        version = "v0.3.2",
     )
     go_repository(
         name = "com_github_ajg_form",
@@ -174,8 +175,8 @@ def go_deps():
         name = "com_github_burntsushi_toml",
         build_file_proto_mode = "disable_global",
         importpath = "github.com/BurntSushi/toml",
-        sum = "h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=",
-        version = "v0.3.1",
+        sum = "h1:GaI7EiDXDRfa8VshkTj7Fym7ha+y8/XxIgD2okUIjLw=",
+        version = "v0.4.1",
     )
     go_repository(
         name = "com_github_burntsushi_xgb",
@@ -219,6 +220,14 @@ def go_deps():
         sum = "h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=",
         version = "v2.1.2",
     )
+    go_repository(
+        name = "com_github_charithe_durationcheck",
+        build_file_proto_mode = "disable",
+        importpath = "github.com/charithe/durationcheck",
+        sum = "h1:mPP4ucLrf/rKZiIG/a9IPXHGlh8p4CzgpyTy6EEutYk=",
+        version = "v0.0.9",
+    )
+
     go_repository(
         name = "com_github_cheggaaa_pb_v3",
         build_file_proto_mode = "disable_global",
@@ -918,6 +927,14 @@ def go_deps():
         sum = "h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=",
         version = "v0.0.4",
     )
+    go_repository(
+        name = "com_github_golangci_prealloc",
+        build_file_proto_mode = "disable",
+        importpath = "github.com/golangci/prealloc",
+        sum = "h1:leSNB7iYzLYSSx3J/s5sVf4Drkc68W2wm4Ixh/mr0us=",
+        version = "v0.0.0-20180630174525-215b22d4de21",
+    )
+
     go_repository(
         name = "com_github_gomodule_redigo",
         build_file_proto_mode = "disable_global",
@@ -2928,6 +2945,14 @@ def go_deps():
         sum = "h1:rxKZ2gOnYxjfmakvUUqh9Gyb6KXfrj7JWTxORTYqb0E=",
         version = "v0.0.0-20220426173459-3bcf042a4bf5",
     )
+    go_repository(
+        name = "org_golang_x_exp_typeparams",
+        build_file_proto_mode = "disable",
+        importpath = "golang.org/x/exp/typeparams",
+        sum = "h1:qyrTQ++p1afMkO4DPEeLGq/3oTsdlvdH4vqZUBWzUKM=",
+        version = "v0.0.0-20220218215828-6cf2b201936e",
+    )
+
     go_repository(
         name = "org_golang_x_image",
         build_file_proto_mode = "disable_global",
@@ -2953,8 +2978,8 @@ def go_deps():
         name = "org_golang_x_mod",
         build_file_proto_mode = "disable_global",
         importpath = "golang.org/x/mod",
-        sum = "h1:LQmS1nU0twXLA96Kt7U9qtHJEbBk3z6Q0V4UXjZkpr4=",
-        version = "v0.6.0-dev.0.20211013180041-c96bc1413d57",
+        sum = "h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s=",
+        version = "v0.6.0-dev.0.20220419223038-86c51ed26bb4",
     )
     go_repository(
         name = "org_golang_x_net",
@@ -3009,8 +3034,8 @@ def go_deps():
         name = "org_golang_x_tools",
         build_file_proto_mode = "disable_global",
         importpath = "golang.org/x/tools",
-        sum = "h1:P1HhGGuLW4aAclzjtmJdf0mJOjVUZUzOTqkAkWL+l6w=",
-        version = "v0.1.8",
+        sum = "h1:OKYpQQVE3DKSc3r3zHVzq46vq5YH7x8xpR3/k9ixmUg=",
+        version = "v0.1.11-0.20220513221640-090b14e8501f",
     )
     go_repository(
         name = "org_golang_x_xerrors",

README.md

@@ -60,7 +60,7 @@ The [community repository](https://github.com/pingcap/community) hosts all infor
 
 [<img src="docs/contribution-map.png" alt="contribution-map" width="180">](https://github.com/pingcap/tidb-map/blob/master/maps/contribution-map.md#tidb-is-an-open-source-distributed-htap-database-compatible-with-the-mysql-protocol)
 
-Contributions are welcomed and greatly appreciated. See [Contribution to TiDB](https://pingcap.github.io/tidb-dev-guide/contribute-to-tidb/introduction.html) for details on typical contribution workflows. For more contributing information, click on the contributor icon above.
+Contributions are welcomed and greatly appreciated. All the contributors are welcomed to claim your reward by filing this [form](https://forms.pingcap.com/f/tidb-contribution-swag). See [Contribution to TiDB](https://pingcap.github.io/tidb-dev-guide/contribute-to-tidb/introduction.html) for details on typical contribution workflows. For more contributing information, click on the contributor icon above.
 
 ## Adopters
 

bindinfo/bind_cache.go

@@ -153,6 +153,7 @@ func (c *bindCache) GetAllBindRecords() []*BindRecord {
 	c.lock.Lock()
 	defer c.lock.Unlock()
 	values := c.cache.Values()
+	//nolint: prealloc
 	var bindRecords []*BindRecord
 	for _, vals := range values {
 		bindRecords = append(bindRecords, vals.([]*BindRecord)...)

br/cmd/tidb-lightning-ctl/main.go

@@ -262,7 +262,7 @@ func checkpointDump(ctx context.Context, cfg *config.Config, dumpFolder string)
 }
 
 func getLocalStoringTables(ctx context.Context, cfg *config.Config) (err2 error) {
-	//nolint:prealloc // This is a placeholder.
+	//nolint: prealloc
 	var tables []string
 	defer func() {
 		if err2 == nil {

br/pkg/lightning/BUILD.bazel

@@ -51,6 +51,7 @@ go_test(
         "//br/pkg/lightning/checkpoints",
         "//br/pkg/lightning/config",
         "//br/pkg/lightning/glue",
+        "//br/pkg/lightning/log",
         "//br/pkg/lightning/mydump",
         "//br/pkg/lightning/web",
         "@com_github_docker_go_units//:go-units",

br/pkg/lightning/backend/backend.go

@@ -69,8 +69,8 @@ func makeTag(tableName string, engineID int32) string {
 	return fmt.Sprintf("%s:%d", tableName, engineID)
 }
 
-func makeLogger(tag string, engineUUID uuid.UUID) log.Logger {
-	return log.With(
+func makeLogger(logger log.Logger, tag string, engineUUID uuid.UUID) log.Logger {
+	return logger.With(
 		zap.String("engineTag", tag),
 		zap.Stringer("engineUUID", engineUUID),
 	)
@@ -143,7 +143,7 @@ type AbstractBackend interface {
 	ShouldPostProcess() bool
 
 	// NewEncoder creates an encoder of a TiDB table.
-	NewEncoder(tbl table.Table, options *kv.SessionOptions) (kv.Encoder, error)
+	NewEncoder(ctx context.Context, tbl table.Table, options *kv.SessionOptions) (kv.Encoder, error)
 
 	OpenEngine(ctx context.Context, config *EngineConfig, engineUUID uuid.UUID) error
 
@@ -260,8 +260,8 @@ func (be Backend) MakeEmptyRows() kv.Rows {
 	return be.abstract.MakeEmptyRows()
 }
 
-func (be Backend) NewEncoder(tbl table.Table, options *kv.SessionOptions) (kv.Encoder, error) {
-	return be.abstract.NewEncoder(tbl, options)
+func (be Backend) NewEncoder(ctx context.Context, tbl table.Table, options *kv.SessionOptions) (kv.Encoder, error) {
+	return be.abstract.NewEncoder(ctx, tbl, options)
 }
 
 func (be Backend) ShouldPostProcess() bool {
@@ -321,7 +321,7 @@ func (be Backend) UnsafeImportAndReset(ctx context.Context, engineUUID uuid.UUID
 	closedEngine := ClosedEngine{
 		engine: engine{
 			backend: be.abstract,
-			logger:  makeLogger("<import-and-reset>", engineUUID),
+			logger:  makeLogger(log.FromContext(ctx), "<import-and-reset>", engineUUID),
 			uuid:    engineUUID,
 		},
 	}
@@ -334,7 +334,7 @@ func (be Backend) UnsafeImportAndReset(ctx context.Context, engineUUID uuid.UUID
 // OpenEngine opens an engine with the given table name and engine ID.
 func (be Backend) OpenEngine(ctx context.Context, config *EngineConfig, tableName string, engineID int32) (*OpenedEngine, error) {
 	tag, engineUUID := MakeUUID(tableName, engineID)
-	logger := makeLogger(tag, engineUUID)
+	logger := makeLogger(log.FromContext(ctx), tag, engineUUID)
 
 	if err := be.abstract.OpenEngine(ctx, config, engineUUID); err != nil {
 		return nil, err
@@ -437,7 +437,7 @@ func (be Backend) UnsafeCloseEngine(ctx context.Context, cfg *EngineConfig, tabl
 func (be Backend) UnsafeCloseEngineWithUUID(ctx context.Context, cfg *EngineConfig, tag string, engineUUID uuid.UUID) (*ClosedEngine, error) {
 	return engine{
 		backend: be.abstract,
-		logger:  makeLogger(tag, engineUUID),
+		logger:  makeLogger(log.FromContext(ctx), tag, engineUUID),
 		uuid:    engineUUID,
 	}.unsafeClose(ctx, cfg)
 }

br/pkg/lightning/backend/backend_test.go

@@ -326,9 +326,9 @@ func TestNewEncoder(t *testing.T) {
 
 	encoder := mock.NewMockEncoder(s.controller)
 	options := &kv.SessionOptions{SQLMode: mysql.ModeANSIQuotes, Timestamp: 1234567890}
-	s.mockBackend.EXPECT().NewEncoder(nil, options).Return(encoder, nil)
+	s.mockBackend.EXPECT().NewEncoder(nil, nil, options).Return(encoder, nil)
 
-	realEncoder, err := s.mockBackend.NewEncoder(nil, options)
+	realEncoder, err := s.mockBackend.NewEncoder(nil, nil, options)
 	require.Equal(t, realEncoder, encoder)
 	require.NoError(t, err)
 }

br/pkg/lightning/backend/kv/kv2sql.go

@@ -17,6 +17,7 @@ package kv
 import (
 	"fmt"
 
+	"github.com/pingcap/tidb/br/pkg/lightning/log"
 	"github.com/pingcap/tidb/kv"
 	"github.com/pingcap/tidb/parser/model"
 	"github.com/pingcap/tidb/table"
@@ -104,8 +105,13 @@ func (t *TableKVDecoder) IterRawIndexKeys(h kv.Handle, rawRow []byte, fn func([]
 	return nil
 }
 
-func NewTableKVDecoder(tbl table.Table, tableName string, options *SessionOptions) (*TableKVDecoder, error) {
-	se := newSession(options)
+func NewTableKVDecoder(
+	tbl table.Table,
+	tableName string,
+	options *SessionOptions,
+	logger log.Logger,
+) (*TableKVDecoder, error) {
+	se := newSession(options, logger)
 	cols := tbl.Cols()
 	// Set CommonAddRecordCtx to session to reuse the slices and BufStore in AddRecord
 	recordCtx := tables.NewCommonAddRecordCtx(len(cols))

br/pkg/lightning/backend/kv/session.go

@@ -246,11 +246,11 @@ type SessionOptions struct {
 }
 
 // NewSession creates a new trimmed down Session matching the options.
-func NewSession(options *SessionOptions) sessionctx.Context {
-	return newSession(options)
+func NewSession(options *SessionOptions, logger log.Logger) sessionctx.Context {
+	return newSession(options, logger)
 }
 
-func newSession(options *SessionOptions) *session {
+func newSession(options *SessionOptions, logger log.Logger) *session {
 	sqlMode := options.SQLMode
 	vars := variable.NewSessionVars()
 	vars.SkipUTF8Check = true
@@ -265,15 +265,15 @@ func newSession(options *SessionOptions) *session {
 	if options.SysVars != nil {
 		for k, v := range options.SysVars {
 			if err := vars.SetSystemVar(k, v); err != nil {
-				log.L().DPanic("new session: failed to set system var",
+				logger.DPanic("new session: failed to set system var",
 					log.ShortError(err),
 					zap.String("key", k))
 			}
 		}
 	}
 	vars.StmtCtx.TimeZone = vars.Location()
 	if err := vars.SetSystemVar("timestamp", strconv.FormatInt(options.Timestamp, 10)); err != nil {
-		log.L().Warn("new session: failed to set timestamp",
+		logger.Warn("new session: failed to set timestamp",
 			log.ShortError(err))
 	}
 	vars.TxnCtx = nil

br/pkg/lightning/backend/kv/session_test.go

@@ -17,12 +17,13 @@ package kv
 import (
 	"testing"
 
+	"github.com/pingcap/tidb/br/pkg/lightning/log"
 	"github.com/pingcap/tidb/parser/mysql"
 	"github.com/stretchr/testify/require"
 )
 
 func TestSession(t *testing.T) {
-	session := newSession(&SessionOptions{SQLMode: mysql.ModeNone, Timestamp: 1234567890})
+	session := newSession(&SessionOptions{SQLMode: mysql.ModeNone, Timestamp: 1234567890}, log.L())
 	_, err := session.Txn(true)
 	require.NoError(t, err)
 }

br/pkg/lightning/backend/kv/sql2kv.go

@@ -66,13 +66,18 @@ type tableKVEncoder struct {
 	metrics  *metric.Metrics
 }
 
-func NewTableKVEncoder(tbl table.Table, options *SessionOptions, metrics *metric.Metrics) (Encoder, error) {
+func NewTableKVEncoder(
+	tbl table.Table,
+	options *SessionOptions,
+	metrics *metric.Metrics,
+	logger log.Logger,
+) (Encoder, error) {
 	if metrics != nil {
 		metrics.KvEncoderCounter.WithLabelValues("open").Inc()
 	}
 	meta := tbl.Meta()
 	cols := tbl.Cols()
-	se := newSession(options)
+	se := newSession(options, logger)
 	// Set CommonAddRecordCtx to session to reuse the slices and BufStore in AddRecord
 	recordCtx := tables.NewCommonAddRecordCtx(len(cols))
 	tables.SetAddRecordCtx(se, recordCtx)
@@ -267,7 +272,7 @@ func logKVConvertFailed(logger log.Logger, row []types.Datum, j int, colInfo *mo
 		log.ShortError(err),
 	)
 
-	log.L().Error("failed to covert kv value", logutil.RedactAny("origVal", original.GetValue()),
+	logger.Error("failed to convert kv value", logutil.RedactAny("origVal", original.GetValue()),
 		zap.Stringer("fieldType", &colInfo.FieldType), zap.String("column", colInfo.Name.O),
 		zap.Int("columnID", j+1))
 	return errors.Annotatef(
@@ -352,7 +357,7 @@ func (kvcodec *tableKVEncoder) Encode(
 
 	var value types.Datum
 	var err error
-	//nolint:prealloc // This is a placeholder.
+	//nolint: prealloc
 	var record []types.Datum
 
 	if kvcodec.recordCache != nil {