merge master

.github/ISSUE_TEMPLATE/bug-report.md

@@ -0,0 +1,29 @@
+---
+name: "🐛 Bug Report"
+about: Something isn't working as expected
+title: ''
+labels: 'bug'
+---
+
+Please answer these questions before submitting your issue. Thanks!
+
+1. What did you do?
+If possible, provide a recipe for reproducing the error.
+
+
+2. What did you expect to see?
+
+
+
+3. What did you see instead?
+
+
+
+4. What version of BR and TiDB/TiKV/PD are you using?
+
+<!--
+br -V
+tidb-server -V
+tikv-server -V
+pd-server -V
+-->

.github/ISSUE_TEMPLATE/feature-request.md

@@ -0,0 +1,19 @@
+---
+name: "🚀 Feature Request"
+about: I have a suggestion
+labels: enhancement
+---
+
+## Feature Request
+
+### Describe your feature request related problem:
+<!-- A description of what the problem is. -->
+
+### Describe the feature you'd like:
+<!-- A description of what you want to happen. -->
+
+### Describe alternatives you've considered:
+<!-- A description of any alternative solutions or features you've considered. -->
+
+### Teachability, Documentation, Adoption, Migration Strategy:
+<!-- If you can, explain some scenarios how users might use this, or situations in which it would be helpful. Any API designs, mockups, or diagrams are also helpful. -->

.github/pull_request_template.md

@@ -0,0 +1,37 @@
+<!--
+Thank you for working on BR! Please read BR's [CONTRIBUTING](https://github.com/pingcap/br/blob/master/CONTRIBUTING.md) document **BEFORE** filing this PR.
+-->
+
+### What problem does this PR solve? <!--add issue link with summary if exists-->
+
+
+### What is changed and how it works?
+
+
+### Check List <!--REMOVE the items that are not applicable-->
+
+Tests <!-- At least one of them must be included. -->
+
+ - Unit test
+ - Integration test
+ - Manual test (add detailed scripts or steps below)
+ - No code
+
+Code changes
+
+ - Has exported function/method change
+ - Has exported variable/fields change
+ - Has interface methods change
+ - Has persistent data change
+
+Side effects
+
+ - Possible performance regression
+ - Increased code complexity
+ - Breaking backward compatibility
+
+Related changes
+
+ - Need to cherry-pick to the release branch
+ - Need to update the documentation
+ - Need to be included in the release note

.golangci.yml

@@ -9,7 +9,8 @@ issues:
       text: "Potential HTTP request made with variable url"
       linters:
         - gosec
-    - path: .go
-      text: "Use of weak random number generator"
+    # TODO Remove it.
+    - path: split_client.go
+      text: "SA1019:"
       linters:
-        - gosec
+        - staticcheck

CHANGELOG.md

@@ -0,0 +1,18 @@
+# BR (Backup and Restore) Change Log
+All notable changes to this project are documented in this file.
+See also,
+- [TiDB Changelog](https://github.com/pingcap/tidb/blob/master/CHANGELOG.md),
+- [TiKV Changelog](https://github.com/tikv/tikv/blob/master/CHANGELOG.md),
+- [PD Changelog](https://github.com/pingcap/pd/blob/master/CHANGELOG.md).
+
+## [3.1.0-beta.1] - 2020-01-10
+
+- Fix the inaccurate backup progress information [#127](https://github.com/pingcap/br/pull/127)
+- Improve the performance of splitting Regions [#122](https://github.com/pingcap/br/pull/122)
+- Add the backup and restore feature for partitioned tables [#137](https://github.com/pingcap/br/pull/137)
+- Add the feature of automatically scheduling PD schedulers [#123](https://github.com/pingcap/br/pull/123)
+- Fix the issue that data is overwritten after non `PKIsHandle` tables are restored [#139](https://github.com/pingcap/br/pull/139)
+
+## [3.1.0-beta] - 2019-12-20
+
+Initial release of the distributed backup and restore tool

Makefile

@@ -26,6 +26,8 @@ build_for_integration_test:
 		-o bin/br.test
 	# build key locker
 	GO111MODULE=on go build -race -o bin/locker tests/br_key_locked/*.go
+	# build gc
+	GO111MODULE=on go build -race -o bin/gc tests/br_z_gc_safepoint/*.go
 
 test:
 	GO111MODULE=on go test -race -tags leak ./...

cmd/backup.go

@@ -1,165 +1,21 @@
 package cmd
 
 import (
-	"context"
-
-	"github.com/pingcap/errors"
-	"github.com/pingcap/log"
 	"github.com/pingcap/tidb/ddl"
 	"github.com/pingcap/tidb/session"
 	"github.com/spf13/cobra"
-	"github.com/spf13/pflag"
 
-	"github.com/pingcap/br/pkg/backup"
-	"github.com/pingcap/br/pkg/storage"
 	"github.com/pingcap/br/pkg/summary"
+	"github.com/pingcap/br/pkg/task"
 	"github.com/pingcap/br/pkg/utils"
 )
 
-const (
-	flagBackupTimeago     = "timeago"
-	flagBackupRateLimit   = "ratelimit"
-	flagBackupConcurrency = "concurrency"
-	flagBackupChecksum    = "checksum"
-	flagLastBackupTS      = "lastbackupts"
-)
-
-func defineBackupFlags(flagSet *pflag.FlagSet) {
-	flagSet.StringP(
-		flagBackupTimeago, "", "",
-		"The history version of the backup task, e.g. 1m, 1h. Do not exceed GCSafePoint")
-	flagSet.Uint64P(
-		flagBackupRateLimit, "", 0, "The rate limit of the backup task, MB/s per node")
-	flagSet.Uint32P(
-		flagBackupConcurrency, "", 4, "The size of thread pool on each node that execute the backup task")
-	flagSet.BoolP(flagBackupChecksum, "", true,
-		"Run checksum after backup")
-	flagSet.Uint64P(flagLastBackupTS, "", 0, "the last time backup ts")
-	_ = flagSet.MarkHidden(flagLastBackupTS)
-}
-
-func runBackup(flagSet *pflag.FlagSet, cmdName, db, table string) error {
-	ctx, cancel := context.WithCancel(defaultContext)
-	defer cancel()
-
-	mgr, err := GetDefaultMgr()
-	if err != nil {
-		return err
-	}
-	defer mgr.Close()
-
-	timeago, err := flagSet.GetString(flagBackupTimeago)
-	if err != nil {
-		return err
-	}
-
-	ratelimit, err := flagSet.GetUint64(flagBackupRateLimit)
-	if err != nil {
-		return err
-	}
-
-	concurrency, err := flagSet.GetUint32(flagBackupConcurrency)
-	if err != nil {
-		return err
-	}
-	if concurrency == 0 {
-		err = errors.New("at least one thread required")
-		return err
-	}
-
-	checksum, err := flagSet.GetBool(flagBackupChecksum)
-	if err != nil {
-		return err
-	}
-
-	lastBackupTS, err := flagSet.GetUint64(flagLastBackupTS)
-	if err != nil {
-		return nil
-	}
-
-	u, err := storage.ParseBackendFromFlags(flagSet, FlagStorage)
-	if err != nil {
-		return err
-	}
-
-	client, err := backup.NewBackupClient(ctx, mgr)
-	if err != nil {
-		return nil
-	}
-
-	err = client.SetStorage(ctx, u)
-	if err != nil {
-		return err
-	}
-
-	backupTS, err := client.GetTS(ctx, timeago)
-	if err != nil {
-		return err
-	}
-
-	defer summary.Summary(cmdName)
-
-	ranges, backupSchemas, err := backup.BuildBackupRangeAndSchema(
-		mgr.GetDomain(), mgr.GetTiKV(), backupTS, db, table)
-	if err != nil {
-		return err
-	}
-
-	// The number of regions need to backup
-	approximateRegions := 0
-	for _, r := range ranges {
-		var regionCount int
-		regionCount, err = mgr.GetRegionCount(ctx, r.StartKey, r.EndKey)
-		if err != nil {
-			return err
-		}
-		approximateRegions += regionCount
-	}
-
-	summary.CollectInt("backup total regions", approximateRegions)
-	// Backup
-	// Redirect to log if there is no log file to avoid unreadable output.
-	updateCh := utils.StartProgress(
-		ctx, cmdName, int64(approximateRegions), !HasLogFile())
-	err = client.BackupRanges(
-		ctx, ranges, lastBackupTS, backupTS, ratelimit, concurrency, updateCh)
-	if err != nil {
-		return err
-	}
-	// Backup has finished
-	close(updateCh)
-
-	// Checksum
-	backupSchemasConcurrency := backup.DefaultSchemaConcurrency
-	if backupSchemas.Len() < backupSchemasConcurrency {
-		backupSchemasConcurrency = backupSchemas.Len()
-	}
-	updateCh = utils.StartProgress(
-		ctx, "Checksum", int64(backupSchemas.Len()), !HasLogFile())
-	backupSchemas.SetSkipChecksum(!checksum)
-	backupSchemas.Start(
-		ctx, mgr.GetTiKV(), backupTS, uint(backupSchemasConcurrency), updateCh)
-
-	err = client.CompleteMeta(backupSchemas)
-	if err != nil {
-		return err
-	}
-
-	valid, err := client.FastChecksum()
-	if err != nil {
-		return err
-	}
-	if !valid {
-		log.Error("backup FastChecksum failed!")
-	}
-	// Checksum has finished
-	close(updateCh)
-
-	err = client.SaveBackupMeta(ctx)
-	if err != nil {
+func runBackupCommand(command *cobra.Command, cmdName string) error {
+	cfg := task.BackupConfig{Config: task.Config{LogProgress: HasLogFile()}}
+	if err := cfg.ParseFromFlags(command.Flags()); err != nil {
 		return err
 	}
-	return nil
+	return task.RunBackup(GetDefaultContext(), cmdName, &cfg)
 }
 
 // NewBackupCommand return a full backup subcommand.
@@ -189,7 +45,7 @@ func NewBackupCommand() *cobra.Command {
 		newTableBackupCommand(),
 	)
 
-	defineBackupFlags(command.PersistentFlags())
+	task.DefineBackupFlags(command.PersistentFlags())
 	return command
 }
 
@@ -200,7 +56,7 @@ func newFullBackupCommand() *cobra.Command {
 		Short: "backup all database",
 		RunE: func(command *cobra.Command, _ []string) error {
 			// empty db/table means full backup.
-			return runBackup(command.Flags(), "Full backup", "", "")
+			return runBackupCommand(command, "Full backup")
 		},
 	}
 	return command
@@ -212,19 +68,10 @@ func newDbBackupCommand() *cobra.Command {
 		Use:   "db",
 		Short: "backup a database",
 		RunE: func(command *cobra.Command, _ []string) error {
-			db, err := command.Flags().GetString(flagDatabase)
-			if err != nil {
-				return err
-			}
-			if len(db) == 0 {
-				return errors.Errorf("empty database name is not allowed")
-			}
-			return runBackup(command.Flags(), "Database backup", db, "")
+			return runBackupCommand(command, "Database backup")
 		},
 	}
-	command.Flags().StringP(flagDatabase, "", "", "backup a table in the specific db")
-	_ = command.MarkFlagRequired(flagDatabase)
-
+	task.DefineDatabaseFlags(command)
 	return command
 }
 
@@ -234,26 +81,9 @@ func newTableBackupCommand() *cobra.Command {
 		Use:   "table",
 		Short: "backup a table",
 		RunE: func(command *cobra.Command, _ []string) error {
-			db, err := command.Flags().GetString(flagDatabase)
-			if err != nil {
-				return err
-			}
-			if len(db) == 0 {
-				return errors.Errorf("empty database name is not allowed")
-			}
-			table, err := command.Flags().GetString(flagTable)
-			if err != nil {
-				return err
-			}
-			if len(table) == 0 {
-				return errors.Errorf("empty table name is not allowed")
-			}
-			return runBackup(command.Flags(), "Table backup", db, table)
+			return runBackupCommand(command, "Table backup")
 		},
 	}
-	command.Flags().StringP(flagDatabase, "", "", "backup a table in the specific db")
-	command.Flags().StringP(flagTable, "t", "", "backup the specific table")
-	_ = command.MarkFlagRequired(flagDatabase)
-	_ = command.MarkFlagRequired(flagTable)
+	task.DefineTableFlags(command)
 	return command
 }