Release 3.1.0.beta2 (pingcap#162)

* restore: fix alter auto increment id for no-primary-key table (pingcap#139) * restore: fix alter auto increment id for no-primary-key table Signed-off-by: 5kbpers <[email protected]> * add integration test Signed-off-by: 5kbpers <[email protected]> * address comments Signed-off-by: 5kbpers <[email protected]> * *: use oracle package to manipulate ts and test gc safe point (pingcap#121) Signed-off-by: Neil Shen <[email protected]> * cmd: convert version command to flags (pingcap#144) * cmd: convert version command to flags Signed-off-by: Neil Shen <[email protected]> * address comments Signed-off-by: Neil Shen <[email protected]> * *: add changelog and github templates (pingcap#143) Signed-off-by: Neil Shen <[email protected]> * restore: merge tidb-tools/pkg/restore-util (pingcap#146) * restore-util: Implement split/scatter (pingcap#274) * implement split/scatter Signed-off-by: 5kbpers <[email protected]> * init test Signed-off-by: 5kbpers <[email protected]> * redesign output/input of the lib Signed-off-by: 5kbpers <[email protected]> * update dependency Signed-off-by: 5kbpers <[email protected]> * add commments and more tests Signed-off-by: 5kbpers <[email protected]> * add ScanRegions interface to Client Signed-off-by: 5kbpers <[email protected]> * fix potential data race Signed-off-by: 5kbpers <[email protected]> * address comments Signed-off-by: 5kbpers <[email protected]> * address comments Signed-off-by: 5kbpers <[email protected]> * Apply suggestions from code review Co-Authored-By: kennytm <[email protected]> * Update pkg/restore-util/client.go Co-Authored-By: kennytm <[email protected]> * address comments Signed-off-by: 5kbpers <[email protected]> * address comments Signed-off-by: 5kbpers <[email protected]> * address comments Signed-off-by: 5kbpers <[email protected]> * update dependency Signed-off-by: 5kbpers <[email protected]> * resolve conflicts Signed-off-by: 5kbpers <[email protected]> * fix prefix rewrite Signed-off-by: 5kbpers <[email protected]> * add RewriteRule/skip failed scatter region/retry the SplitRegion Signed-off-by: 5kbpers <[email protected]> * fix test Signed-off-by: 5kbpers <[email protected]> * check if region has peer Signed-off-by: 5kbpers <[email protected]> * more logs Signed-off-by: 5kbpers <[email protected]> * restore-util: add split retry interval (pingcap#277) * reset dependencies to release-3.1 * add split retry interval Signed-off-by: 5kbpers <[email protected]> * fix go.sum Signed-off-by: 5kbpers <[email protected]> * restore-util: wait for scatter region sequentially (pingcap#279) * wait for scatter region sequentially Signed-off-by: 5kbpers <[email protected]> * address comments Signed-off-by: 5kbpers <[email protected]> * restore-util: add on split hook (pingcap#281) * restore-util: add on split hook Signed-off-by: Neil Shen <[email protected]> * Nil check onSplit Co-Authored-By: kennytm <[email protected]> * restore-util: fix returned new region is nil (pingcap#283) * restore-util: fix returned new region is nil Signed-off-by: 5kbpers <[email protected]> * more logs Signed-off-by: 5kbpers <[email protected]> * *: gofmt Signed-off-by: 5kbpers <[email protected]> * Apply suggestions from code review Co-Authored-By: kennytm <[email protected]> * fix log Signed-off-by: 5kbpers <[email protected]> * restore-util: call onSplit on splitByRewriteRules (pingcap#285) Signed-off-by: Neil Shen <[email protected]> * restore-util: fix overlapped error message (pingcap#293) * restore-util: fix overlapped error message Signed-off-by: 5kbpers <[email protected]> * fix log message Signed-off-by: 5kbpers <[email protected]> * reduce error trace Signed-off-by: 5kbpers <[email protected]> * fix test Signed-off-by: 5kbpers <[email protected]> * address comments Signed-off-by: 5kbpers <[email protected]> * address comments Signed-off-by: 5kbpers <[email protected]> * restore-util: log warning when cannot find matched rewrite rule (pingcap#299) * restore-util: add method to set placement rules and store labels (pingcap#301) * restore-util: add method to set placement rules and store labels Signed-off-by: disksing <[email protected]> * minor fix Signed-off-by: disksing <[email protected]> * address comment Signed-off-by: disksing <[email protected]> * add GetPlacementRules Signed-off-by: disksing <[email protected]> * fix test Signed-off-by: disksing <[email protected]> * restore-util: support batch split (pingcap#300) * restore-util: support batch split Signed-off-by: 5kbpers <[email protected]> * go fmt Signed-off-by: 5kbpers <[email protected]> * Apply suggestions from code review Co-Authored-By: kennytm <[email protected]> * address commits Signed-off-by: 5kbpers <[email protected]> * Update pkg/restore-util/split.go Co-Authored-By: kennytm <[email protected]> * add onSplit callback Signed-off-by: 5kbpers <[email protected]> * fix test Signed-off-by: 5kbpers <[email protected]> * address comments Signed-off-by: 5kbpers <[email protected]> * restore-util: add upper bound time for waiting for scatter (pingcap#305) * restore: fix scatter regions failed Signed-off-by: 5kbpers <[email protected]> * add log Signed-off-by: 5kbpers <[email protected]> * stop waiting for scatter after 3min Signed-off-by: 5kbpers <[email protected]> * address comments Signed-off-by: 5kbpers <[email protected]> * restore-util: fix wrong url (pingcap#306) Signed-off-by: disksing <[email protected]> * restore-util: add warning about unmatched table id (pingcap#313) * restore-util: support table partition Signed-off-by: 5kbpers <[email protected]> * fix log Signed-off-by: 5kbpers <[email protected]> * warn table id does not match Signed-off-by: 5kbpers <[email protected]> * add unit tests Signed-off-by: 5kbpers <[email protected]> * Apply suggestions from code review Co-Authored-By: Neil Shen <[email protected]> * fix compile error Signed-off-by: 5kbpers <[email protected]> * address comments Signed-off-by: 5kbpers <[email protected]> * address comments Signed-off-by: 5kbpers <[email protected]> * fix test Signed-off-by: 5kbpers <[email protected]> Co-authored-by: Ian <[email protected]> Co-authored-by: Neil Shen <[email protected]> * *: prune tidb-tools Signed-off-by: Neil Shen <[email protected]> * restore: address linters suggestions Signed-off-by: Neil Shen <[email protected]> * restore: merge restoreutil into restore Signed-off-by: Neil Shen <[email protected]> * address comment Signed-off-by: Neil Shen <[email protected]> Co-authored-by: 5kbpers <[email protected]> Co-authored-by: kennytm <[email protected]> Co-authored-by: disksing <[email protected]> Co-authored-by: Ian <[email protected]> * Fixed handling for a dbName that do not exist in the backup being restored (pingcap#148) * Fixed handling for a dbName that do not exist in the backup being restored * Fixed handling for a dbName that do not exist in the backup being restored * validate: fix debug meta test ci (pingcap#153) * validate: fix debug meta test ci * *: extracts runBackup/runRestore in cmd into pkg/task (pingcap#156) * *: extracts runBackup/runRestore in cmd into pkg/task Defines a "Config" structure to store the parsed flags. Use the "black-white-list" structure to define what tables/databases to backup/restore. * go.mod: update tidb to v4.0.0-beta * restore: fix restore summary log (pingcap#150) Co-authored-by: kennytm <[email protected]> * restore: enhance error handling (pingcap#152) * restore: enhance error handling Signed-off-by: 5kbpers <[email protected]> * unit test Signed-off-by: 5kbpers <[email protected]> * address comments Signed-off-by: 5kbpers <[email protected]> * fix region epoch error Signed-off-by: 5kbpers <[email protected]> * address comments Signed-off-by: 5kbpers <[email protected]> * remove `Restore*` Signed-off-by: 5kbpers <[email protected]> * address lint Signed-off-by: 5kbpers <[email protected]> * add debug log Signed-off-by: 5kbpers <[email protected]> * Apply suggestions from code review Co-Authored-By: kennytm <[email protected]> * Update pkg/restore/import.go Co-Authored-By: kennytm <[email protected]> * fix retry error Signed-off-by: 5kbpers <[email protected]> * handle RegionNotFound error Signed-off-by: 5kbpers <[email protected]> Co-authored-by: Neil Shen <[email protected]> Co-authored-by: kennytm <[email protected]> Co-authored-by: 5kbpers <[email protected]> Co-authored-by: Neil Shen <[email protected]> Co-authored-by: kennytm <[email protected]> Co-authored-by: disksing <[email protected]> Co-authored-by: Ian <[email protected]> Co-authored-by: Kolbe Kegel <[email protected]>
kennytm · Feb 20, 2020 · 78c162a · 78c162a
1 parent 328ad69
commit 78c162a
Show file tree

Hide file tree

Showing 52 changed files with 2,928 additions and 1,303 deletions.
diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md
@@ -0,0 +1,29 @@
+---
+name: "🐛 Bug Report"
+about: Something isn't working as expected
+title: ''
+labels: 'bug'
+---
+
+Please answer these questions before submitting your issue. Thanks!
+
+1. What did you do?
+If possible, provide a recipe for reproducing the error.
+
+
+2. What did you expect to see?
+
+
+
+3. What did you see instead?
+
+
+
+4. What version of BR and TiDB/TiKV/PD are you using?
+
+<!--
+br -V
+tidb-server -V
+tikv-server -V
+pd-server -V
+-->
diff --git a/.github/ISSUE_TEMPLATE/feature-request.md b/.github/ISSUE_TEMPLATE/feature-request.md
@@ -0,0 +1,19 @@
+---
+name: "🚀 Feature Request"
+about: I have a suggestion
+labels: enhancement
+---
+
+## Feature Request
+
+### Describe your feature request related problem:
+<!-- A description of what the problem is. -->
+
+### Describe the feature you'd like:
+<!-- A description of what you want to happen. -->
+
+### Describe alternatives you've considered:
+<!-- A description of any alternative solutions or features you've considered. -->
+
+### Teachability, Documentation, Adoption, Migration Strategy:
+<!-- If you can, explain some scenarios how users might use this, or situations in which it would be helpful. Any API designs, mockups, or diagrams are also helpful. -->
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
@@ -0,0 +1,37 @@
+<!--
+Thank you for working on BR! Please read BR's [CONTRIBUTING](https://github.com/pingcap/br/blob/master/CONTRIBUTING.md) document **BEFORE** filing this PR.
+-->
+
+### What problem does this PR solve? <!--add issue link with summary if exists-->
+
+
+### What is changed and how it works?
+
+
+### Check List <!--REMOVE the items that are not applicable-->
+
+Tests <!-- At least one of them must be included. -->
+
+ - Unit test
+ - Integration test
+ - Manual test (add detailed scripts or steps below)
+ - No code
+
+Code changes
+
+ - Has exported function/method change
+ - Has exported variable/fields change
+ - Has interface methods change
+ - Has persistent data change
+
+Side effects
+
+ - Possible performance regression
+ - Increased code complexity
+ - Breaking backward compatibility
+
+Related changes
+
+ - Need to cherry-pick to the release branch
+ - Need to update the documentation
+ - Need to be included in the release note
diff --git a/.golangci.yml b/.golangci.yml
@@ -9,7 +9,8 @@ issues:
       text: "Potential HTTP request made with variable url"
       linters:
         - gosec
-    - path: .go
-      text: "Use of weak random number generator"
+    # TODO Remove it.
+    - path: split_client.go
+      text: "SA1019:"
       linters:
-        - gosec
+        - staticcheck
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -0,0 +1,18 @@
+# BR (Backup and Restore) Change Log
+All notable changes to this project are documented in this file.
+See also,
+- [TiDB Changelog](https://github.com/pingcap/tidb/blob/master/CHANGELOG.md),
+- [TiKV Changelog](https://github.com/tikv/tikv/blob/master/CHANGELOG.md),
+- [PD Changelog](https://github.com/pingcap/pd/blob/master/CHANGELOG.md).
+
+## [3.1.0-beta.1] - 2020-01-10
+
+- Fix the inaccurate backup progress information [#127](https://github.com/pingcap/br/pull/127)
+- Improve the performance of splitting Regions [#122](https://github.com/pingcap/br/pull/122)
+- Add the backup and restore feature for partitioned tables [#137](https://github.com/pingcap/br/pull/137)
+- Add the feature of automatically scheduling PD schedulers [#123](https://github.com/pingcap/br/pull/123)
+- Fix the issue that data is overwritten after non `PKIsHandle` tables are restored [#139](https://github.com/pingcap/br/pull/139)
+
+## [3.1.0-beta] - 2019-12-20
+
+Initial release of the distributed backup and restore tool
diff --git a/Makefile b/Makefile
@@ -26,6 +26,8 @@ build_for_integration_test:
 		-o bin/br.test
 	# build key locker
 	GO111MODULE=on go build -race -o bin/locker tests/br_key_locked/*.go
+	# build gc
+	GO111MODULE=on go build -race -o bin/gc tests/br_z_gc_safepoint/*.go
 
 test:
 	GO111MODULE=on go test -race -tags leak ./...

diff --git a/cmd/backup.go b/cmd/backup.go
@@ -1,165 +1,21 @@
 package cmd
 
 import (
-	"context"
-
-	"github.com/pingcap/errors"
-	"github.com/pingcap/log"
 	"github.com/pingcap/tidb/ddl"
 	"github.com/pingcap/tidb/session"
 	"github.com/spf13/cobra"
-	"github.com/spf13/pflag"
 
-	"github.com/pingcap/br/pkg/backup"
-	"github.com/pingcap/br/pkg/storage"
 	"github.com/pingcap/br/pkg/summary"
+	"github.com/pingcap/br/pkg/task"
 	"github.com/pingcap/br/pkg/utils"
 )
 
-const (
-	flagBackupTimeago     = "timeago"
-	flagBackupRateLimit   = "ratelimit"
-	flagBackupConcurrency = "concurrency"
-	flagBackupChecksum    = "checksum"
-	flagLastBackupTS      = "lastbackupts"
-)
-
-func defineBackupFlags(flagSet *pflag.FlagSet) {
-	flagSet.StringP(
-		flagBackupTimeago, "", "",
-		"The history version of the backup task, e.g. 1m, 1h. Do not exceed GCSafePoint")
-	flagSet.Uint64P(
-		flagBackupRateLimit, "", 0, "The rate limit of the backup task, MB/s per node")
-	flagSet.Uint32P(
-		flagBackupConcurrency, "", 4, "The size of thread pool on each node that execute the backup task")
-	flagSet.BoolP(flagBackupChecksum, "", true,
-		"Run checksum after backup")
-	flagSet.Uint64P(flagLastBackupTS, "", 0, "the last time backup ts")
-	_ = flagSet.MarkHidden(flagLastBackupTS)
-}
-
-func runBackup(flagSet *pflag.FlagSet, cmdName, db, table string) error {
-	ctx, cancel := context.WithCancel(defaultContext)
-	defer cancel()
-
-	mgr, err := GetDefaultMgr()
-	if err != nil {
-		return err
-	}
-	defer mgr.Close()
-
-	timeago, err := flagSet.GetString(flagBackupTimeago)
-	if err != nil {
-		return err
-	}
-
-	ratelimit, err := flagSet.GetUint64(flagBackupRateLimit)
-	if err != nil {
-		return err
-	}
-
-	concurrency, err := flagSet.GetUint32(flagBackupConcurrency)
-	if err != nil {
-		return err
-	}
-	if concurrency == 0 {
-		err = errors.New("at least one thread required")
-		return err
-	}
-
-	checksum, err := flagSet.GetBool(flagBackupChecksum)
-	if err != nil {
-		return err
-	}
-
-	lastBackupTS, err := flagSet.GetUint64(flagLastBackupTS)
-	if err != nil {
-		return nil
-	}
-
-	u, err := storage.ParseBackendFromFlags(flagSet, FlagStorage)
-	if err != nil {
-		return err
-	}
-
-	client, err := backup.NewBackupClient(ctx, mgr)
-	if err != nil {
-		return nil
-	}
-
-	err = client.SetStorage(ctx, u)
-	if err != nil {
-		return err
-	}
-
-	backupTS, err := client.GetTS(ctx, timeago)
-	if err != nil {
-		return err
-	}
-
-	defer summary.Summary(cmdName)
-
-	ranges, backupSchemas, err := backup.BuildBackupRangeAndSchema(
-		mgr.GetDomain(), mgr.GetTiKV(), backupTS, db, table)
-	if err != nil {
-		return err
-	}
-
-	// The number of regions need to backup
-	approximateRegions := 0
-	for _, r := range ranges {
-		var regionCount int
-		regionCount, err = mgr.GetRegionCount(ctx, r.StartKey, r.EndKey)
-		if err != nil {
-			return err
-		}
-		approximateRegions += regionCount
-	}
-
-	summary.CollectInt("backup total regions", approximateRegions)
-	// Backup
-	// Redirect to log if there is no log file to avoid unreadable output.
-	updateCh := utils.StartProgress(
-		ctx, cmdName, int64(approximateRegions), !HasLogFile())
-	err = client.BackupRanges(
-		ctx, ranges, lastBackupTS, backupTS, ratelimit, concurrency, updateCh)
-	if err != nil {
-		return err
-	}
-	// Backup has finished
-	close(updateCh)
-
-	// Checksum
-	backupSchemasConcurrency := backup.DefaultSchemaConcurrency
-	if backupSchemas.Len() < backupSchemasConcurrency {
-		backupSchemasConcurrency = backupSchemas.Len()
-	}
-	updateCh = utils.StartProgress(
-		ctx, "Checksum", int64(backupSchemas.Len()), !HasLogFile())
-	backupSchemas.SetSkipChecksum(!checksum)
-	backupSchemas.Start(
-		ctx, mgr.GetTiKV(), backupTS, uint(backupSchemasConcurrency), updateCh)
-
-	err = client.CompleteMeta(backupSchemas)
-	if err != nil {
-		return err
-	}
-
-	valid, err := client.FastChecksum()
-	if err != nil {
-		return err
-	}
-	if !valid {
-		log.Error("backup FastChecksum failed!")
-	}
-	// Checksum has finished
-	close(updateCh)
-
-	err = client.SaveBackupMeta(ctx)
-	if err != nil {
+func runBackupCommand(command *cobra.Command, cmdName string) error {
+	cfg := task.BackupConfig{Config: task.Config{LogProgress: HasLogFile()}}
+	if err := cfg.ParseFromFlags(command.Flags()); err != nil {
 		return err
 	}
-	return nil
+	return task.RunBackup(GetDefaultContext(), cmdName, &cfg)
 }
 
 // NewBackupCommand return a full backup subcommand.
@@ -189,7 +45,7 @@ func NewBackupCommand() *cobra.Command {
 		newTableBackupCommand(),
 	)
 
-	defineBackupFlags(command.PersistentFlags())
+	task.DefineBackupFlags(command.PersistentFlags())
 	return command
 }
 
@@ -200,7 +56,7 @@ func newFullBackupCommand() *cobra.Command {
 		Short: "backup all database",
 		RunE: func(command *cobra.Command, _ []string) error {
 			// empty db/table means full backup.
-			return runBackup(command.Flags(), "Full backup", "", "")
+			return runBackupCommand(command, "Full backup")
 		},
 	}
 	return command
@@ -212,19 +68,10 @@ func newDbBackupCommand() *cobra.Command {
 		Use:   "db",
 		Short: "backup a database",
 		RunE: func(command *cobra.Command, _ []string) error {
-			db, err := command.Flags().GetString(flagDatabase)
-			if err != nil {
-				return err
-			}
-			if len(db) == 0 {
-				return errors.Errorf("empty database name is not allowed")
-			}
-			return runBackup(command.Flags(), "Database backup", db, "")
+			return runBackupCommand(command, "Database backup")
 		},
 	}
-	command.Flags().StringP(flagDatabase, "", "", "backup a table in the specific db")
-	_ = command.MarkFlagRequired(flagDatabase)
-
+	task.DefineDatabaseFlags(command)
 	return command
 }
 
@@ -234,26 +81,9 @@ func newTableBackupCommand() *cobra.Command {
 		Use:   "table",
 		Short: "backup a table",
 		RunE: func(command *cobra.Command, _ []string) error {
-			db, err := command.Flags().GetString(flagDatabase)
-			if err != nil {
-				return err
-			}
-			if len(db) == 0 {
-				return errors.Errorf("empty database name is not allowed")
-			}
-			table, err := command.Flags().GetString(flagTable)
-			if err != nil {
-				return err
-			}
-			if len(table) == 0 {
-				return errors.Errorf("empty table name is not allowed")
-			}
-			return runBackup(command.Flags(), "Table backup", db, table)
+			return runBackupCommand(command, "Table backup")
 		},
 	}
-	command.Flags().StringP(flagDatabase, "", "", "backup a table in the specific db")
-	command.Flags().StringP(flagTable, "t", "", "backup the specific table")
-	_ = command.MarkFlagRequired(flagDatabase)
-	_ = command.MarkFlagRequired(flagTable)
+	task.DefineTableFlags(command)
 	return command
 }