Merge pull request pingcap#8 from pingcap/backup-all-schemas

Backup all schemas & Auto Inc ID

cmd/raw.go

@@ -63,6 +63,11 @@ func newFullBackupCommand() *cobra.Command {
 				return errors.New("at least one thread required")
 			}
 
+			err = client.BackupAllSchemas(backupTS)
+			if err != nil {
+				return err
+			}
+
 			err = client.BackupRange([]byte(""), []byte(""), u, backupTS, rate, concurrency)
 			if err != nil {
 				return err

cmd/restore.go

@@ -185,11 +185,23 @@ func newTableRestoreCommand() *cobra.Command {
 			if err != nil {
 				return errors.Trace(err)
 			}
-			tables := db.GetTables(tableName)
-			if len(tables) <= 0 {
-				return errors.Trace(fmt.Errorf("not exists table"))
+			table := db.GetTable(tableName)
+			if table == nil {
+				return errors.New("not exists table")
 			}
-			err = client.RestoreMultipleTables(tables, restoreTS)
+			err = restore.CreateTable(db.Schema.Name.String(), table, client.GetDbDSN())
+			if err != nil {
+				return errors.Trace(err)
+			}
+			err = restore.AlterAutoIncID(db.Schema.Name.String(), table, client.GetDbDSN())
+			if err != nil {
+				return errors.Trace(err)
+			}
+			fileGroups := db.GetFileGroups(tableName)
+			if len(fileGroups) <= 0 {
+				return errors.New("not exists table")
+			}
+			err = client.RestoreMultipleTables(fileGroups, restoreTS)
 			return errors.Trace(err)
 		},
 	}

go.mod

@@ -3,6 +3,7 @@ module github.com/pingcap/br
 go 1.12
 
 require (
+	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/coreos/go-semver v0.3.0 // indirect
 	github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect
 	github.com/cznic/sortutil v0.0.0-20181122101858-f5f958428db8 // indirect
@@ -27,7 +28,7 @@ require (
 	github.com/pingcap/parser v0.0.0-20190912032624-978b8272c04e
 	github.com/pingcap/pd v0.0.0-20190806095100-82f1dd11d823
 	github.com/pingcap/tidb v0.0.0-20190912055946-5c48d93368d4
-	github.com/prometheus/client_golang v1.1.0
+	github.com/prometheus/client_golang v0.9.0
 	github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 // indirect
 	github.com/sirupsen/logrus v1.4.2 // indirect

pkg/raw/full.go

@@ -7,9 +7,10 @@ import (
 	"sync"
 	"time"
 
+	"github.com/pingcap/tidb/meta/autoid"
+
 	"github.com/gogo/protobuf/proto"
 	"github.com/google/btree"
-	"github.com/pingcap/br/pkg/meta"
 	"github.com/pingcap/errors"
 	"github.com/pingcap/kvproto/pkg/backup"
 	"github.com/pingcap/kvproto/pkg/metapb"
@@ -23,6 +24,8 @@ import (
 	"github.com/pingcap/tidb/tablecodec"
 	"github.com/pingcap/tidb/util/codec"
 	"go.uber.org/zap"
+
+	"github.com/pingcap/br/pkg/meta"
 )
 
 // Maximum total sleep time(in ms) for kv/cop commands.
@@ -113,11 +116,11 @@ func (bc *BackupClient) BackupTable(
 	rateLimit uint64,
 	concurrency uint32,
 ) error {
-	session, err := session.CreateSession(bc.backer.GetTiKV())
+	dbSession, err := session.CreateSession(bc.backer.GetTiKV())
 	if err != nil {
 		return errors.Trace(err)
 	}
-	do := domain.GetDomain(session.(sessionctx.Context))
+	do := domain.GetDomain(dbSession.(sessionctx.Context))
 	info, err := do.GetSnapshotInfoSchema(backupTS)
 	if err != nil {
 		return errors.Trace(err)
@@ -132,10 +135,16 @@ func (bc *BackupClient) BackupTable(
 	}
 	cTableName := model.NewCIStr(tableName)
 	table, err := info.TableByName(cDBName, cTableName)
+	if err != nil {
+		return errors.Trace(err)
+	}
 	tableInfo = table.Meta()
+	idAlloc := autoid.NewAllocator(bc.backer.GetTiKV(), dbInfo.ID, false)
+	globalAutoID, err := idAlloc.NextGlobalAutoID(tableInfo.ID)
 	if err != nil {
 		return errors.Trace(err)
 	}
+	tableInfo.AutoIncID = globalAutoID
 
 	dbData, err := json.Marshal(dbInfo)
 	if err != nil {
@@ -152,6 +161,11 @@ func (bc *BackupClient) BackupTable(
 		Db:    dbData,
 		Table: tableData,
 	}
+	log.Info("save table schema",
+		zap.Stringer("db", dbInfo.Name),
+		zap.Stringer("table", tableInfo.Name),
+		zap.Int64("auto_inc_id", globalAutoID),
+	)
 	bc.backupMeta.Schemas = append(bc.backupMeta.Schemas, backupSchema)
 	log.Info("backup table meta",
 		zap.Reflect("Schema", dbInfo),
@@ -199,6 +213,64 @@ func buildTableRanges(tbl *model.TableInfo) []tableRange {
 	return ranges
 }
 
+// BackupAllSchemas fetches all schemas from TiDB.
+func (bc *BackupClient) BackupAllSchemas(backupTS uint64) error {
+	SystemDatabases := [3]string{
+		"information_schema",
+		"performance_schema",
+		"mysql",
+	}
+
+	dbSession, err := session.CreateSession(bc.backer.GetTiKV())
+	if err != nil {
+		return errors.Trace(err)
+	}
+	do := domain.GetDomain(dbSession.(sessionctx.Context))
+	info, err := do.GetSnapshotInfoSchema(backupTS)
+	if err != nil {
+		return errors.Trace(err)
+	}
+
+	dbInfos := info.AllSchemas()
+LoadDb:
+	for _, dbInfo := range dbInfos {
+		// skip system databases
+		for _, sysDbName := range SystemDatabases {
+			if sysDbName == dbInfo.Name.L {
+				continue LoadDb
+			}
+		}
+		dbData, err := json.Marshal(dbInfo)
+		if err != nil {
+			return errors.Trace(err)
+		}
+		idAlloc := autoid.NewAllocator(bc.backer.GetTiKV(), dbInfo.ID, false)
+		for _, tableInfo := range dbInfo.Tables {
+			globalAutoID, err := idAlloc.NextGlobalAutoID(tableInfo.ID)
+			if err != nil {
+				return errors.Trace(err)
+			}
+			tableInfo.AutoIncID = globalAutoID
+			tableData, err := json.Marshal(tableInfo)
+			if err != nil {
+				return errors.Trace(err)
+			}
+			// Save schema.
+			backupSchema := &backup.Schema{
+				Db:    dbData,
+				Table: tableData,
+			}
+			log.Info("save table schema",
+				zap.Stringer("db", dbInfo.Name),
+				zap.Stringer("table", tableInfo.Name),
+				zap.Int64("auto_inc_id", globalAutoID),
+			)
+			bc.backupMeta.Schemas = append(bc.backupMeta.Schemas, backupSchema)
+		}
+	}
+	return nil
+}
+
 // BackupRange make a backup of the given key range.
 func (bc *BackupClient) BackupRange(
 	startKey, endKey []byte,