restore: enhance error handling

go.mod

@@ -8,12 +8,14 @@ require (
 	github.com/cheggaaa/pb/v3 v3.0.1
 	github.com/coreos/go-semver v0.3.0 // indirect
 	github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect
+	github.com/fatih/color v1.9.0 // indirect
 	github.com/fsouza/fake-gcs-server v1.15.0
 	github.com/go-sql-driver/mysql v1.4.1
 	github.com/gogo/protobuf v1.3.1
 	github.com/golang/snappy v0.0.1 // indirect
 	github.com/google/btree v1.0.0
 	github.com/google/uuid v1.1.1
+	github.com/mattn/go-runewidth v0.0.7 // indirect
 	github.com/onsi/ginkgo v1.10.3 // indirect
 	github.com/onsi/gomega v1.7.1 // indirect
 	github.com/pingcap/check v0.0.0-20191216031241-8a5a85928f12
@@ -33,6 +35,7 @@ require (
 	go.uber.org/zap v1.13.0
 	golang.org/x/net v0.0.0-20191011234655-491137f69257 // indirect
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
+	golang.org/x/tools v0.0.0-20200107184032-11e9d9cc0042 // indirect
 	google.golang.org/api v0.14.0
 	google.golang.org/grpc v1.25.1
 )

go.sum

@@ -97,6 +97,8 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s=
+github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsouza/fake-gcs-server v1.15.0 h1:ss/ztlt10Y64A5qslmxZKsiqW/i28t5DkRtv6qSFaLQ=
@@ -224,12 +226,18 @@ github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czP
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
+github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM=
+github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
 github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y=
 github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
+github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54=
+github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
 github.com/matttproud/golang_protobuf_extensions v1.0.0/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
@@ -518,6 +526,7 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190909082730-f460065e899a h1:mIzbOulag9/gXacgxKlFVwpCOWSfBT3/pDyyCwGA9as=
 golang.org/x/sys v0.0.0-20190909082730-f460065e899a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191210023423-ac6580df4449 h1:gSbV7h1NRL2G1xTg/owz62CST1oJBmxy4QpMMregXVQ=
 golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
@@ -555,6 +564,8 @@ golang.org/x/tools v0.0.0-20191107010934-f79515f33823/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2 h1:EtTFh6h4SAKemS+CURDMTDIANuduG5zKEXShyy18bGA=
 golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200107184032-11e9d9cc0042 h1:BKiPVwWbEdmAh+5CBwk13CYeVJQRDJpDnKgDyMOGz9M=
+golang.org/x/tools v0.0.0-20200107184032-11e9d9cc0042/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4 h1:Toz2IK7k8rbltAXwNAxKcn9OzqyNfMUhUNjz3sL0NMk=
 golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

pkg/restore/backoff.go

@@ -0,0 +1,117 @@
+package restore
+
+import (
+	"time"
+
+	"github.com/pingcap/errors"
+	"github.com/pingcap/log"
+	"go.uber.org/zap"
+
+	"github.com/pingcap/br/pkg/utils"
+)
+
+var (
+	errNotLeader           = errors.NewNoStackError("not leader")
+	errEpochNotMatch       = errors.NewNoStackError("epoch not match")
+	errKeyNotInRegion      = errors.NewNoStackError("key not in region")
+	errRegionNotFound      = errors.NewNoStackError("region not found")
+	errResp                = errors.NewNoStackError("response error")
+	errRewriteRuleNotFound = errors.NewNoStackError("rewrite rule not found")
+	errRangeIsEmpty        = errors.NewNoStackError("range is empty")
+	errGrpc                = errors.NewNoStackError("gRPC error")
+
+	// TODO: add `error` field to `DownloadResponse` for distinguish the errors of gRPC
+	// and the errors of request
+	errBadFormat      = errors.NewNoStackError("bad format")
+	errWrongKeyPrefix = errors.NewNoStackError("wrong key prefix")
+	errFileCorrupted  = errors.NewNoStackError("file corrupted")
+	errCannotRead     = errors.NewNoStackError("cannot read externel storage")
+)
+
+const (
+	importSSTRetryTimes      = 16
+	importSSTWaitInterval    = 10 * time.Millisecond
+	importSSTMaxWaitInterval = 1 * time.Second
+
+	downloadSSTRetryTimes      = 8
+	downloadSSTWaitInterval    = 10 * time.Millisecond
+	downloadSSTMaxWaitInterval = 1 * time.Second
+
+	resetTsRetryTime       = 16
+	resetTSWaitInterval    = 50 * time.Millisecond
+	resetTSMaxWaitInterval = 500 * time.Millisecond
+)
+
+type importerBackoffer struct {
+	attempt      int
+	delayTime    time.Duration
+	maxDelayTime time.Duration
+}
+
+func newImportSSTBackoffer() utils.Backoffer {
+	return &importerBackoffer{
+		attempt:      importSSTRetryTimes,
+		delayTime:    importSSTWaitInterval,
+		maxDelayTime: importSSTMaxWaitInterval,
+	}
+}
+
+func newDownloadSSTBackoffer() utils.Backoffer {
+	return &importerBackoffer{
+		attempt:      downloadSSTRetryTimes,
+		delayTime:    downloadSSTWaitInterval,
+		maxDelayTime: downloadSSTMaxWaitInterval,
+	}
+}
+
+func (bo *importerBackoffer) NextBackoff(err error) time.Duration {
+	switch errors.Cause(err) {
+	case errResp, errGrpc, errEpochNotMatch, errNotLeader:
+		bo.delayTime = 2 * bo.delayTime
+		bo.attempt--
+	case errRangeIsEmpty, errRewriteRuleNotFound:
+		// Excepted error, finish the operation
+		bo.delayTime = 0
+		bo.attempt = 0
+	default:
+		// Unexcepted error
+		bo.delayTime = 0
+		bo.attempt = 0
+		log.Warn("unexcepted error, stop to retry", zap.Error(err))
+	}
+	if bo.delayTime > bo.maxDelayTime {
+		return bo.maxDelayTime
+	}
+	return bo.delayTime
+}
+
+func (bo *importerBackoffer) Attempt() int {
+	return bo.attempt
+}
+
+type resetTSBackoffer struct {
+	attempt      int
+	delayTime    time.Duration
+	maxDelayTime time.Duration
+}
+
+func newResetTSBackoffer() utils.Backoffer {
+	return &resetTSBackoffer{
+		attempt:      resetTsRetryTime,
+		delayTime:    resetTSWaitInterval,
+		maxDelayTime: resetTSMaxWaitInterval,
+	}
+}
+
+func (bo *resetTSBackoffer) NextBackoff(err error) time.Duration {
+	bo.delayTime = 2 * bo.delayTime
+	bo.attempt--
+	if bo.delayTime > bo.maxDelayTime {
+		return bo.maxDelayTime
+	}
+	return bo.delayTime
+}
+
+func (bo *resetTSBackoffer) Attempt() int {
+	return bo.attempt
+}

pkg/restore/backoff_test.go

@@ -0,0 +1,58 @@
+package restore
+
+import (
+	"context"
+	"time"
+
+	. "github.com/pingcap/check"
+	"github.com/pingcap/tidb/util/testleak"
+
+	"github.com/pingcap/br/pkg/utils"
+)
+
+var _ = Suite(&testBackofferSuite{})
+
+type testBackofferSuite struct {
+	mock *utils.MockCluster
+}
+
+func (s *testBackofferSuite) SetUpSuite(c *C) {
+	var err error
+	s.mock, err = utils.NewMockCluster()
+	c.Assert(err, IsNil)
+}
+
+func (s *testBackofferSuite) TearDownSuite(c *C) {
+	testleak.AfterTest(c)()
+}
+
+func (s *testBackofferSuite) TestImporterBackoffer(c *C) {
+	var counter int
+	err := utils.WithRetry(context.Background(), func() error {
+		defer func() { counter++ }()
+		switch counter {
+		case 0:
+			return errGrpc
+		case 1:
+			return errResp
+		case 2:
+			return errRangeIsEmpty
+		}
+		return nil
+	}, newImportSSTBackoffer())
+	c.Assert(counter, Equals, 3)
+	c.Assert(err, Equals, errRangeIsEmpty)
+
+	counter = 0
+	backoffer := importerBackoffer{
+		attempt:      10,
+		delayTime:    time.Nanosecond,
+		maxDelayTime: time.Nanosecond,
+	}
+	err = utils.WithRetry(context.Background(), func() error {
+		defer func() { counter++ }()
+		return errResp
+	}, &backoffer)
+	c.Assert(counter, Equals, 10)
+	c.Assert(err, Equals, errResp)
+}