Merge branch 'master' into rename_recreator

Makefile

@@ -277,13 +277,14 @@ endif
 # Usage:
 #	make bench-daily TO=/path/to/file.json
 bench-daily:
-	go test github.com/pingcap/tidb/session -run TestBenchDaily --outfile bench_daily.json
-	go test github.com/pingcap/tidb/executor -run TestBenchDaily --outfile bench_daily.json
-	go test github.com/pingcap/tidb/tablecodec -run TestBenchDaily --outfile bench_daily.json
-	go test github.com/pingcap/tidb/expression -run TestBenchDaily --outfile bench_daily.json
-	go test github.com/pingcap/tidb/util/rowcodec -run TestBenchDaily --outfile bench_daily.json
-	go test github.com/pingcap/tidb/util/codec -run TestBenchDaily --outfile bench_daily.json
-	go test github.com/pingcap/tidb/util/benchdaily -run TestBenchDaily \
+	go test github.com/pingcap/tidb/session -bench BenchmarkDaily --outfile bench_daily.json
+	go test github.com/pingcap/tidb/executor -bench BenchmarkDaily --outfile bench_daily.json
+	go test github.com/pingcap/tidb/tablecodec -bench BenchmarkDaily --outfile bench_daily.json
+	go test github.com/pingcap/tidb/expression -bench BenchmarkDaily --outfile bench_daily.json
+	go test github.com/pingcap/tidb/util/rowcodec -bench BenchmarkDaily --outfile bench_daily.json
+	go test github.com/pingcap/tidb/util/codec -bench BenchmarkDaily --outfile bench_daily.json
+	go test github.com/pingcap/tidb/distsql -bench BenchmarkDaily --outfile bench_daily.json
+	go test github.com/pingcap/tidb/util/benchdaily -bench BenchmarkDaily \
 		-date `git log -n1 --date=unix --pretty=format:%cd` \
 		-commit `git log -n1 --pretty=format:%h` \
 		-outfile $(TO)

bindinfo/session_handle_serial_test.go

@@ -437,7 +437,6 @@ func TestTemporaryTable(t *testing.T) {
 	tk := testkit.NewTestKit(t, store)
 	tk.MustExec("use test")
 	tk.MustExec("drop table if exists t")
-	tk.MustExec("set tidb_enable_global_temporary_table = true")
 	tk.MustExec("create global temporary table t(a int, b int, key(a), key(b)) on commit delete rows")
 	tk.MustExec("create table t2(a int, b int, key(a), key(b))")
 	tk.MustGetErrCode("create session binding for select * from t where b = 123 using select * from t ignore index(b) where b = 123;", errno.ErrOptOnTemporaryTable)

bindinfo/temptable_serial_test.go

@@ -29,7 +29,6 @@ func TestSelectBindingOnGlobalTempTableProhibited(t *testing.T) {
 
 	tk := testkit.NewTestKit(t, store)
 	tk.MustExec("use test")
-	tk.MustExec("set tidb_enable_global_temporary_table = true")
 	tk.MustExec("drop table if exists t1,tmp1")
 	tk.MustExec("create table t1(a int(11))")
 	tk.MustExec("create global temporary table tmp1(a int(11), key idx_a(a)) on commit delete rows;")
@@ -64,7 +63,6 @@ func TestDMLBindingOnGlobalTempTableProhibited(t *testing.T) {
 
 	tk := testkit.NewTestKit(t, store)
 	tk.MustExec("use test")
-	tk.MustExec("set tidb_enable_global_temporary_table = true")
 	tk.MustExec("drop table if exists t1,tmp1,tmp2")
 	tk.MustExec("create table t1(a int(11))")
 	tk.MustExec("create global temporary table tmp1(a int(11), key idx_a(a)) on commit delete rows;")

br/pkg/conn/conn.go

@@ -184,6 +184,13 @@ func GetAllTiKVStoresWithRetry(ctx context.Context,
 				}
 			})
 
+			failpoint.Inject("hint-GetAllTiKVStores-cancel", func(val failpoint.Value) {
+				if val.(bool) {
+					logutil.CL(ctx).Debug("failpoint hint-GetAllTiKVStores-cancel injected.")
+					err = status.Error(codes.Canceled, "Cancel Retry")
+				}
+			})
+
 			return errors.Trace(err)
 		},
 		utils.NewPDReqBackoffer(),

br/pkg/conn/conn_test.go

@@ -6,10 +6,14 @@ import (
 	"context"
 	"testing"
 
+	"github.com/pingcap/errors"
+	"github.com/pingcap/failpoint"
 	"github.com/pingcap/kvproto/pkg/metapb"
 	"github.com/pingcap/tidb/br/pkg/pdutil"
 	"github.com/stretchr/testify/require"
 	pd "github.com/tikv/pd/client"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 )
 
 type fakePDClient struct {
@@ -21,9 +25,86 @@ func (c fakePDClient) GetAllStores(context.Context, ...pd.GetStoreOption) ([]*me
 	return append([]*metapb.Store{}, c.stores...), nil
 }
 
-func TestCheckStoresAlive(t *testing.T) {
-	t.Parallel()
+func TestGetAllTiKVStoresWithRetryCancel(t *testing.T) {
+	_ = failpoint.Enable("github.com/pingcap/tidb/br/pkg/conn/hint-GetAllTiKVStores-cancel", "return(true)")
+	defer func() {
+		_ = failpoint.Disable("github.com/pingcap/tidb/br/pkg/conn/hint-GetAllTiKVStores-cancel")
+	}()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	stores := []*metapb.Store{
+		{
+			Id:    1,
+			State: metapb.StoreState_Up,
+			Labels: []*metapb.StoreLabel{
+				{
+					Key:   "engine",
+					Value: "tiflash",
+				},
+			},
+		},
+		{
+			Id:    2,
+			State: metapb.StoreState_Offline,
+			Labels: []*metapb.StoreLabel{
+				{
+					Key:   "engine",
+					Value: "tiflash",
+				},
+			},
+		},
+	}
+
+	fpdc := fakePDClient{
+		stores: stores,
+	}
+
+	_, err := GetAllTiKVStoresWithRetry(ctx, fpdc, SkipTiFlash)
+	require.Error(t, err)
+	require.Equal(t, codes.Canceled, status.Code(errors.Cause(err)))
+}
 
+func TestGetAllTiKVStoresWithUnknown(t *testing.T) {
+	_ = failpoint.Enable("github.com/pingcap/tidb/br/pkg/conn/hint-GetAllTiKVStores-error", "return(true)")
+	defer func() {
+		_ = failpoint.Disable("github.com/pingcap/tidb/br/pkg/conn/hint-GetAllTiKVStores-error")
+	}()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	stores := []*metapb.Store{
+		{
+			Id:    1,
+			State: metapb.StoreState_Up,
+			Labels: []*metapb.StoreLabel{
+				{
+					Key:   "engine",
+					Value: "tiflash",
+				},
+			},
+		},
+		{
+			Id:    2,
+			State: metapb.StoreState_Offline,
+			Labels: []*metapb.StoreLabel{
+				{
+					Key:   "engine",
+					Value: "tiflash",
+				},
+			},
+		},
+	}
+
+	fpdc := fakePDClient{
+		stores: stores,
+	}
+
+	_, err := GetAllTiKVStoresWithRetry(ctx, fpdc, SkipTiFlash)
+	require.Error(t, err)
+	require.Equal(t, codes.Unknown, status.Code(errors.Cause(err)))
+}
+func TestCheckStoresAlive(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 

br/pkg/lightning/backend/backend.go

@@ -25,10 +25,10 @@ import (
 	"github.com/pingcap/failpoint"
 	"github.com/pingcap/tidb/br/pkg/lightning/backend/kv"
 	"github.com/pingcap/tidb/br/pkg/lightning/checkpoints"
-	"github.com/pingcap/tidb/br/pkg/lightning/common"
 	"github.com/pingcap/tidb/br/pkg/lightning/log"
 	"github.com/pingcap/tidb/br/pkg/lightning/metric"
 	"github.com/pingcap/tidb/br/pkg/lightning/mydump"
+	"github.com/pingcap/tidb/br/pkg/utils"
 	"github.com/pingcap/tidb/parser/model"
 	"github.com/pingcap/tidb/table"
 	"go.uber.org/zap"
@@ -442,7 +442,7 @@ func (engine *ClosedEngine) Import(ctx context.Context, regionSplitSize int64) e
 	for i := 0; i < importMaxRetryTimes; i++ {
 		task := engine.logger.With(zap.Int("retryCnt", i)).Begin(zap.InfoLevel, "import")
 		err = engine.backend.ImportEngine(ctx, engine.uuid, regionSplitSize)
-		if !common.IsRetryableError(err) {
+		if !utils.IsRetryableError(err) {
 			task.End(zap.ErrorLevel, err)
 			return err
 		}

br/pkg/lightning/backend/importer/importer.go

@@ -29,6 +29,7 @@ import (
 	"github.com/pingcap/tidb/br/pkg/lightning/common"
 	"github.com/pingcap/tidb/br/pkg/lightning/log"
 	"github.com/pingcap/tidb/br/pkg/lightning/tikv"
+	"github.com/pingcap/tidb/br/pkg/utils"
 	"github.com/pingcap/tidb/br/pkg/version"
 	"github.com/pingcap/tidb/parser/model"
 	"github.com/pingcap/tidb/table"
@@ -249,7 +250,7 @@ outside:
 			switch {
 			case err == nil:
 				continue outside
-			case common.IsRetryableError(err):
+			case utils.IsRetryableError(err):
 				// retry next loop
 			default:
 				return err

br/pkg/lightning/backend/tidb/tidb.go

@@ -416,7 +416,7 @@ rowLoop:
 			switch {
 			case err == nil:
 				continue rowLoop
-			case common.IsRetryableError(err):
+			case utils.IsRetryableError(err):
 				// retry next loop
 			default:
 				// WriteBatchRowsToDB failed in the batch mode and can not be retried,
@@ -529,7 +529,7 @@ func (be *tidbBackend) execStmts(ctx context.Context, stmtTasks []stmtTask, tabl
 					return errors.Trace(err)
 				}
 				// Retry the non-batch insert here if this is not the last retry.
-				if common.IsRetryableError(err) && i != writeRowsMaxRetryTimes-1 {
+				if utils.IsRetryableError(err) && i != writeRowsMaxRetryTimes-1 {
 					continue
 				}
 				firstRow := stmtTask.rows[0]

br/pkg/lightning/common/util.go

@@ -18,28 +18,20 @@ import (
 	"context"
 	"database/sql"
 	"encoding/json"
-	stderrors "errors"
 	"fmt"
 	"io"
-	"net"
 	"net/http"
 	"net/url"
 	"os"
-	"reflect"
-	"regexp"
 	"strings"
 	"syscall"
 	"time"
 
-	"github.com/go-sql-driver/mysql"
 	"github.com/pingcap/errors"
 	"github.com/pingcap/tidb/br/pkg/lightning/log"
 	"github.com/pingcap/tidb/br/pkg/utils"
-	tmysql "github.com/pingcap/tidb/errno"
 	"github.com/pingcap/tidb/parser/model"
 	"go.uber.org/zap"
-	"google.golang.org/grpc/codes"
-	"google.golang.org/grpc/status"
 )
 
 const (
@@ -130,7 +122,7 @@ outside:
 		// do not retry NotFound error
 		case errors.IsNotFound(err):
 			break outside
-		case IsRetryableError(err):
+		case utils.IsRetryableError(err):
 			logger.Warn(purpose+" failed but going to try again", log.ShortError(err))
 			continue
 		default:
@@ -193,61 +185,6 @@ func (t SQLWithRetry) Exec(ctx context.Context, purpose string, query string, ar
 	})
 }
 
-// sqlmock uses fmt.Errorf to produce expectation failures, which will cause
-// unnecessary retry if not specially handled >:(
-var stdFatalErrorsRegexp = regexp.MustCompile(
-	`^call to (?s:.*) was not expected|arguments do not match:|could not match actual sql|mock non-retryable error`,
-)
-var stdErrorType = reflect.TypeOf(stderrors.New(""))
-
-// IsRetryableError returns whether the error is transient (e.g. network
-// connection dropped) or irrecoverable (e.g. user pressing Ctrl+C). This
-// function returns `false` (irrecoverable) if `err == nil`.
-//
-// If the error is a multierr, returns true only if all suberrors are retryable.
-func IsRetryableError(err error) bool {
-	for _, singleError := range errors.Errors(err) {
-		if !isSingleRetryableError(singleError) {
-			return false
-		}
-	}
-	return true
-}
-
-func isSingleRetryableError(err error) bool {
-	err = errors.Cause(err)
-
-	switch err {
-	case nil, context.Canceled, context.DeadlineExceeded, io.EOF, sql.ErrNoRows:
-		return false
-	}
-
-	switch nerr := err.(type) {
-	case net.Error:
-		return nerr.Timeout()
-	case *mysql.MySQLError:
-		switch nerr.Number {
-		// ErrLockDeadlock can retry to commit while meet deadlock
-		case tmysql.ErrUnknown, tmysql.ErrLockDeadlock, tmysql.ErrWriteConflictInTiDB, tmysql.ErrPDServerTimeout, tmysql.ErrTiKVServerTimeout, tmysql.ErrTiKVServerBusy, tmysql.ErrResolveLockTimeout, tmysql.ErrRegionUnavailable:
-			return true
-		default:
-			return false
-		}
-	default:
-		switch status.Code(err) {
-		case codes.DeadlineExceeded, codes.NotFound, codes.AlreadyExists, codes.PermissionDenied, codes.ResourceExhausted, codes.Aborted, codes.OutOfRange, codes.Unavailable, codes.DataLoss:
-			return true
-		case codes.Unknown:
-			if reflect.TypeOf(err) == stdErrorType {
-				return !stdFatalErrorsRegexp.MatchString(err.Error())
-			}
-			return true
-		default:
-			return false
-		}
-	}
-}
-
 // IsContextCanceledError returns whether the error is caused by context
 // cancellation. This function should only be used when the code logic is
 // affected by whether the error is canceling or not.

br/pkg/lightning/common/util_test.go

@@ -17,23 +17,16 @@ package common_test
 import (
 	"context"
 	"encoding/json"
-	"fmt"
 	"io"
-	"net"
 	"net/http"
 	"net/http/httptest"
 	"time"
 
 	sqlmock "github.com/DATA-DOG/go-sqlmock"
-	"github.com/go-sql-driver/mysql"
 	. "github.com/pingcap/check"
 	"github.com/pingcap/errors"
 	"github.com/pingcap/tidb/br/pkg/lightning/common"
 	"github.com/pingcap/tidb/br/pkg/lightning/log"
-	tmysql "github.com/pingcap/tidb/errno"
-	"go.uber.org/multierr"
-	"google.golang.org/grpc/codes"
-	"google.golang.org/grpc/status"
 )
 
 type utilSuite struct{}
@@ -85,48 +78,6 @@ func (s *utilSuite) TestGetJSON(c *C) {
 	c.Assert(err, ErrorMatches, ".*http status code != 200.*")
 }
 
-func (s *utilSuite) TestIsRetryableError(c *C) {
-	c.Assert(common.IsRetryableError(context.Canceled), IsFalse)
-	c.Assert(common.IsRetryableError(context.DeadlineExceeded), IsFalse)
-	c.Assert(common.IsRetryableError(io.EOF), IsFalse)
-	c.Assert(common.IsRetryableError(&net.AddrError{}), IsFalse)
-	c.Assert(common.IsRetryableError(&net.DNSError{}), IsFalse)
-	c.Assert(common.IsRetryableError(&net.DNSError{IsTimeout: true}), IsTrue)
-
-	// MySQL Errors
-	c.Assert(common.IsRetryableError(&mysql.MySQLError{}), IsFalse)
-	c.Assert(common.IsRetryableError(&mysql.MySQLError{Number: tmysql.ErrUnknown}), IsTrue)
-	c.Assert(common.IsRetryableError(&mysql.MySQLError{Number: tmysql.ErrLockDeadlock}), IsTrue)
-	c.Assert(common.IsRetryableError(&mysql.MySQLError{Number: tmysql.ErrPDServerTimeout}), IsTrue)
-	c.Assert(common.IsRetryableError(&mysql.MySQLError{Number: tmysql.ErrTiKVServerTimeout}), IsTrue)
-	c.Assert(common.IsRetryableError(&mysql.MySQLError{Number: tmysql.ErrTiKVServerBusy}), IsTrue)
-	c.Assert(common.IsRetryableError(&mysql.MySQLError{Number: tmysql.ErrResolveLockTimeout}), IsTrue)
-	c.Assert(common.IsRetryableError(&mysql.MySQLError{Number: tmysql.ErrRegionUnavailable}), IsTrue)
-	c.Assert(common.IsRetryableError(&mysql.MySQLError{Number: tmysql.ErrWriteConflictInTiDB}), IsTrue)
-
-	// gRPC Errors
-	c.Assert(common.IsRetryableError(status.Error(codes.Canceled, "")), IsFalse)
-	c.Assert(common.IsRetryableError(status.Error(codes.Unknown, "")), IsTrue)
-	c.Assert(common.IsRetryableError(status.Error(codes.DeadlineExceeded, "")), IsTrue)
-	c.Assert(common.IsRetryableError(status.Error(codes.NotFound, "")), IsTrue)
-	c.Assert(common.IsRetryableError(status.Error(codes.AlreadyExists, "")), IsTrue)
-	c.Assert(common.IsRetryableError(status.Error(codes.PermissionDenied, "")), IsTrue)
-	c.Assert(common.IsRetryableError(status.Error(codes.ResourceExhausted, "")), IsTrue)
-	c.Assert(common.IsRetryableError(status.Error(codes.Aborted, "")), IsTrue)
-	c.Assert(common.IsRetryableError(status.Error(codes.OutOfRange, "")), IsTrue)
-	c.Assert(common.IsRetryableError(status.Error(codes.Unavailable, "")), IsTrue)
-	c.Assert(common.IsRetryableError(status.Error(codes.DataLoss, "")), IsTrue)
-
-	// sqlmock errors
-	c.Assert(common.IsRetryableError(fmt.Errorf("call to database Close was not expected")), IsFalse)
-	c.Assert(common.IsRetryableError(errors.New("call to database Close was not expected")), IsTrue)
-
-	// multierr
-	c.Assert(common.IsRetryableError(multierr.Combine(context.Canceled, context.Canceled)), IsFalse)
-	c.Assert(common.IsRetryableError(multierr.Combine(&net.DNSError{IsTimeout: true}, &net.DNSError{IsTimeout: true})), IsTrue)
-	c.Assert(common.IsRetryableError(multierr.Combine(context.Canceled, &net.DNSError{IsTimeout: true})), IsFalse)
-}
-
 func (s *utilSuite) TestToDSN(c *C) {
 	param := common.MySQLConnectParam{
 		Host:             "127.0.0.1",