Merge master to release 3.1

.codecov.yml

@@ -0,0 +1,10 @@
+codecov:
+  require_ci_to_pass: yes
+
+coverage:
+  status:
+    project:
+      default:
+        # Allow the coverage to drop by 3%
+        threshold: 3%
+    patch: off

.dockerignore

@@ -0,0 +1 @@
+.gitignore

.gitignore

@@ -7,3 +7,5 @@ backupmeta
 *.ngo
 *.coverprofile
 coverage.txt
+docker/data/
+docker/logs/

LICENSE.md

@@ -186,7 +186,7 @@ Apache License
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright {}
+   Copyright [yyyy] [name of copyright owner]
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
@@ -198,4 +198,4 @@ Apache License
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
-   limitations under the License.
+   limitations under the License.

Makefile

@@ -12,25 +12,28 @@ LDFLAGS += -X "$(BR_PKG)/pkg/utils.BRBuildTS=$(shell date -u '+%Y-%m-%d %I:%M:%S
 LDFLAGS += -X "$(BR_PKG)/pkg/utils.BRGitHash=$(shell git rev-parse HEAD)"
 LDFLAGS += -X "$(BR_PKG)/pkg/utils.BRGitBranch=$(shell git rev-parse --abbrev-ref HEAD)"
 
-all: check test build
+ifeq ("$(WITH_RACE)", "1")
+	RACEFLAG = -race
+endif
 
-release:
-	GO111MODULE=on go build -ldflags '$(LDFLAGS)' -o bin/br
+all: check test build
 
 build:
-	GO111MODULE=on go build -ldflags '$(LDFLAGS)' -race -o bin/br
+	GO111MODULE=on go build -ldflags '$(LDFLAGS)' ${RACEFLAG} -o bin/br
 
 build_for_integration_test:
 	GO111MODULE=on go test -c -cover -covermode=count \
 		-coverpkg=$(BR_PKG)/... \
 		-o bin/br.test
 	# build key locker
-	GO111MODULE=on go build -race -o bin/locker tests/br_key_locked/*.go
+	GO111MODULE=on go build ${RACEFLAG} -o bin/locker tests/br_key_locked/*.go
 	# build gc
-	GO111MODULE=on go build -race -o bin/gc tests/br_z_gc_safepoint/*.go
+	GO111MODULE=on go build ${RACEFLAG} -o bin/gc tests/br_z_gc_safepoint/*.go
+	# build rawkv client
+	GO111MODULE=on go build ${RACEFLAG} -o bin/rawkv tests/br_rawkv/*.go
 
 test:
-	GO111MODULE=on go test -race -tags leak ./...
+	GO111MODULE=on go test ${RACEFLAG} -tags leak ./...
 
 testcover:
 	GO111MODULE=on retool do overalls \
@@ -46,6 +49,7 @@ integration_test: build build_for_integration_test
 	@which bin/pd-server
 	@which bin/pd-ctl
 	@which bin/go-ycsb
+	@which bin/minio
 	@which bin/br
 	tests/run.sh
 
@@ -71,6 +75,12 @@ static:
 		--disable interfacer \
 		--disable goimports \
 		--disable gofmt \
+		--disable wsl \
+		--disable funlen \
+		--disable whitespace \
+		--disable gocognit \
+		--disable godox \
+		--disable gomnd \
 		$$($(PACKAGE_DIRECTORIES))
 
 lint:

README.md

@@ -36,6 +36,41 @@ Notice BR supports building with Go version `Go >= 1.13`
 
 When BR is built successfully, you can find binary in the `bin` directory.
 
+## Quick start
+
+```sh
+# Start TiDB cluster
+docker-compose -f docker-compose.yaml rm -s -v && \
+docker-compose -f docker-compose.yaml build && \
+docker-compose -f docker-compose.yaml up --remove-orphans
+
+# Attach to control container to run BR
+docker exec -it br_control_1 bash
+
+# Load testing data to TiDB
+go-ycsb load mysql -p workload=core \
+    -p mysql.host=tidb -p mysql.port=4000 -p mysql.user=root \
+    -p recordcount=100000 -p threadcount=100
+
+# How many rows do we get? 100000 rows.
+mysql -uroot -htidb -P4000 -E -e "SELECT COUNT(*) FROM test.usertable"
+
+# Build BR and backup!
+make release && \
+bin/br backup full --pd pd0:2379 --storage "local:///data/backup/full" \
+    --log-file "/logs/br_backup.log"
+
+# Let's drop database.
+mysql -uroot -htidb -P4000 -E -e "DROP DATABASE test; SHOW DATABASES;"
+
+# Restore!
+bin/br restore full --pd pd0:2379 --storage "local:///data/backup/full" \
+    --log-file "/logs/br_restore.log"
+
+# How many rows do we get again? Expected to be 100000 rows.
+mysql -uroot -htidb -P4000 -E -e "SELECT COUNT(*) FROM test.usertable"
+```
+
 ## Contributing
 
 Contributions are welcomed and greatly appreciated. See [CONTRIBUTING](./CONTRIBUTING.md)

cmd/backup.go

@@ -1,10 +1,13 @@
+// Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0.
+
 package cmd
 
 import (
 	"github.com/pingcap/tidb/ddl"
 	"github.com/pingcap/tidb/session"
 	"github.com/spf13/cobra"
 
+	"github.com/pingcap/br/pkg/gluetikv"
 	"github.com/pingcap/br/pkg/summary"
 	"github.com/pingcap/br/pkg/task"
 	"github.com/pingcap/br/pkg/utils"
@@ -15,14 +18,23 @@ func runBackupCommand(command *cobra.Command, cmdName string) error {
 	if err := cfg.ParseFromFlags(command.Flags()); err != nil {
 		return err
 	}
-	return task.RunBackup(GetDefaultContext(), cmdName, &cfg)
+	return task.RunBackup(GetDefaultContext(), tidbGlue, cmdName, &cfg)
+}
+
+func runBackupRawCommand(command *cobra.Command, cmdName string) error {
+	cfg := task.RawKvConfig{Config: task.Config{LogProgress: HasLogFile()}}
+	if err := cfg.ParseFromFlags(command.Flags()); err != nil {
+		return err
+	}
+	return task.RunBackupRaw(GetDefaultContext(), gluetikv.Glue{}, cmdName, &cfg)
 }
 
 // NewBackupCommand return a full backup subcommand.
 func NewBackupCommand() *cobra.Command {
 	command := &cobra.Command{
-		Use:   "backup",
-		Short: "backup a TiDB cluster",
+		Use:          "backup",
+		Short:        "backup a TiDB/TiKV cluster",
+		SilenceUsage: false,
 		PersistentPreRunE: func(c *cobra.Command, args []string) error {
 			if err := Init(c); err != nil {
 				return err
@@ -43,6 +55,7 @@ func NewBackupCommand() *cobra.Command {
 		newFullBackupCommand(),
 		newDbBackupCommand(),
 		newTableBackupCommand(),
+		newRawBackupCommand(),
 	)
 
 	task.DefineBackupFlags(command.PersistentFlags())
@@ -87,3 +100,18 @@ func newTableBackupCommand() *cobra.Command {
 	task.DefineTableFlags(command)
 	return command
 }
+
+// newRawBackupCommand return a raw kv range backup subcommand.
+func newRawBackupCommand() *cobra.Command {
+	// TODO: remove experimental tag if it's stable
+	command := &cobra.Command{
+		Use:   "raw",
+		Short: "(experimental) backup a raw kv range from TiKV cluster",
+		RunE: func(command *cobra.Command, _ []string) error {
+			return runBackupRawCommand(command, "Raw backup")
+		},
+	}
+
+	task.DefineRawBackupFlags(command)
+	return command
+}

cmd/cmd.go

@@ -1,3 +1,5 @@
+// Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0.
+
 package cmd
 
 import (
@@ -13,6 +15,7 @@ import (
 	"github.com/spf13/cobra"
 	"go.uber.org/zap"
 
+	"github.com/pingcap/br/pkg/gluetidb"
 	"github.com/pingcap/br/pkg/task"
 	"github.com/pingcap/br/pkg/utils"
 )
@@ -21,6 +24,7 @@ var (
 	initOnce       = sync.Once{}
 	defaultContext context.Context
 	hasLogFile     uint64
+	tidbGlue       = gluetidb.Glue{}
 )
 
 const (
@@ -84,16 +88,20 @@ func Init(cmd *cobra.Command) (err error) {
 			err = e
 			return
 		}
+		tidbLogCfg := logutil.LogConfig{}
 		if len(slowLogFilename) != 0 {
-			slowCfg := logutil.LogConfig{SlowQueryFile: slowLogFilename}
-			e = logutil.InitLogger(&slowCfg)
-			if e != nil {
-				err = e
-				return
-			}
+			tidbLogCfg.SlowQueryFile = slowLogFilename
 		} else {
 			// Hack! Discard slow log by setting log level to PanicLevel
 			logutil.SlowQueryLogger.SetLevel(logrus.PanicLevel)
+			// Disable annoying TiDB Log.
+			// TODO: some error logs outputs randomly, we need to fix them in TiDB.
+			tidbLogCfg.Level = "fatal"
+		}
+		e = logutil.InitLogger(&tidbLogCfg)
+		if e != nil {
+			err = e
+			return
 		}
 
 		// Initialize the pprof server.

cmd/restore.go

@@ -1,9 +1,12 @@
+// Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0.
+
 package cmd
 
 import (
 	"github.com/pingcap/tidb/session"
 	"github.com/spf13/cobra"
 
+	"github.com/pingcap/br/pkg/gluetikv"
 	"github.com/pingcap/br/pkg/summary"
 	"github.com/pingcap/br/pkg/task"
 	"github.com/pingcap/br/pkg/utils"
@@ -14,14 +17,34 @@ func runRestoreCommand(command *cobra.Command, cmdName string) error {
 	if err := cfg.ParseFromFlags(command.Flags()); err != nil {
 		return err
 	}
-	return task.RunRestore(GetDefaultContext(), cmdName, &cfg)
+	return task.RunRestore(GetDefaultContext(), tidbGlue, cmdName, &cfg)
+}
+
+func runRestoreRawCommand(command *cobra.Command, cmdName string) error {
+	cfg := task.RestoreRawConfig{
+		RawKvConfig: task.RawKvConfig{Config: task.Config{LogProgress: HasLogFile()}},
+	}
+	if err := cfg.ParseFromFlags(command.Flags()); err != nil {
+		return err
+	}
+	return task.RunRestoreRaw(GetDefaultContext(), gluetikv.Glue{}, cmdName, &cfg)
+}
+
+func runRestoreTiflashReplicaCommand(command *cobra.Command, cmdName string) error {
+	cfg := task.RestoreConfig{Config: task.Config{LogProgress: HasLogFile()}}
+	if err := cfg.ParseFromFlags(command.Flags()); err != nil {
+		return err
+	}
+
+	return task.RunRestoreTiflashReplica(GetDefaultContext(), tidbGlue, cmdName, &cfg)
 }
 
 // NewRestoreCommand returns a restore subcommand
 func NewRestoreCommand() *cobra.Command {
 	command := &cobra.Command{
-		Use:   "restore",
-		Short: "restore a TiKV cluster from a backup",
+		Use:          "restore",
+		Short:        "restore a TiDB/TiKV cluster",
+		SilenceUsage: false,
 		PersistentPreRunE: func(c *cobra.Command, args []string) error {
 			if err := Init(c); err != nil {
 				return err
@@ -40,6 +63,8 @@ func NewRestoreCommand() *cobra.Command {
 		newFullRestoreCommand(),
 		newDbRestoreCommand(),
 		newTableRestoreCommand(),
+		newRawRestoreCommand(),
+		newTiflashReplicaRestoreCommand(),
 	)
 	task.DefineRestoreFlags(command.PersistentFlags())
 
@@ -80,3 +105,27 @@ func newTableRestoreCommand() *cobra.Command {
 	task.DefineTableFlags(command)
 	return command
 }
+
+func newTiflashReplicaRestoreCommand() *cobra.Command {
+	command := &cobra.Command{
+		Use:   "tiflash-replica",
+		Short: "restore the tiflash replica before the last restore, it must only be used after the last restore failed",
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			return runRestoreTiflashReplicaCommand(cmd, "Restore TiFlash Replica")
+		},
+	}
+	return command
+}
+
+func newRawRestoreCommand() *cobra.Command {
+	command := &cobra.Command{
+		Use:   "raw",
+		Short: "(experimental) restore a raw kv range to TiKV cluster",
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			return runRestoreRawCommand(cmd, "Raw restore")
+		},
+	}
+
+	task.DefineRawRestoreFlags(command)
+	return command
+}