From b7b9b519c2042c1d919952b20da09ff2a4c9a1eb Mon Sep 17 00:00:00 2001 From: Louis Date: Sun, 1 Mar 2020 14:48:42 +0800 Subject: [PATCH 01/11] Backup: add TikvGCLifeTime field --- cmd/backup-manager/app/backup/backup.go | 29 +++++ cmd/backup-manager/app/backup/manager.go | 112 +++++++++++++++++- cmd/backup-manager/app/cmd/backup.go | 6 + cmd/backup-manager/app/export/manager.go | 66 +++++++---- images/tidb-backup-manager/Dockerfile | 6 + pkg/apis/pingcap/v1alpha1/types.go | 2 + .../pingcap/v1alpha1/zz_generated.deepcopy.go | 5 + pkg/backup/backup/backup_manager.go | 13 +- 8 files changed, 208 insertions(+), 31 deletions(-) diff --git a/cmd/backup-manager/app/backup/backup.go b/cmd/backup-manager/app/backup/backup.go index 51f1d3626d..a602f664b8 100644 --- a/cmd/backup-manager/app/backup/backup.go +++ b/cmd/backup-manager/app/backup/backup.go @@ -15,6 +15,7 @@ package backup import ( "context" + "database/sql" "fmt" "io" "os/exec" @@ -32,6 +33,10 @@ import ( type Options struct { Namespace string BackupName string + Host string + Port int32 + Password string + User string } func (bo *Options) String() string { @@ -63,6 +68,30 @@ func (bo *Options) backupData(backup *v1alpha1.Backup) (string, error) { return path, nil } +func (bo *Options) getTikvGCLifeTime(db *sql.DB) (string, error) { + var tikvGCTime string + sql := fmt.Sprintf("select variable_value from %s where variable_name= ?", constants.TidbMetaTable) + row := db.QueryRow(sql, constants.TikvGCVariable) + err := row.Scan(&tikvGCTime) + if err != nil { + return tikvGCTime, fmt.Errorf("query cluster %s %s failed, sql: %s, err: %v", bo, constants.TikvGCVariable, sql, err) + } + return tikvGCTime, nil +} + +func (bo *Options) setTikvGCLifeTime(db *sql.DB, gcTime string) error { + sql := fmt.Sprintf("update %s set variable_value = ? where variable_name = ?", constants.TidbMetaTable) + _, err := db.Exec(sql, gcTime, constants.TikvGCVariable) + if err != nil { + return fmt.Errorf("set cluster %s %s failed, sql: %s, err: %v", bo, constants.TikvGCVariable, sql, err) + } + return nil +} + +func (bo *Options) getDSN(db string) string { + return fmt.Sprintf("%s:%s@(%s:%d)/%s?charset=utf8", bo.User, bo.Password, bo.Host, bo.Port, db) +} + // getCommitTs get backup position from `EndVersion` in BR backup meta func getCommitTs(backup *v1alpha1.Backup) (uint64, error) { var commitTs uint64 diff --git a/cmd/backup-manager/app/backup/manager.go b/cmd/backup-manager/app/backup/manager.go index 0fb2543eba..c575255675 100644 --- a/cmd/backup-manager/app/backup/manager.go +++ b/cmd/backup-manager/app/backup/manager.go @@ -14,14 +14,18 @@ package backup import ( + "database/sql" "fmt" "time" + "github.com/pingcap/tidb-operator/cmd/backup-manager/app/constants" + "github.com/pingcap/tidb-operator/cmd/backup-manager/app/util" "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" listers "github.com/pingcap/tidb-operator/pkg/client/listers/pingcap/v1alpha1" "github.com/pingcap/tidb-operator/pkg/controller" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog" ) @@ -60,10 +64,32 @@ func (bm *Manager) ProcessBackup() error { if backup.Spec.BR == nil { return fmt.Errorf("no br config in %s", bm) } - return bm.performBackup(backup.DeepCopy()) + + var db *sql.DB + err = wait.PollImmediate(constants.PollInterval, constants.CheckTimeout, func() (done bool, err error) { + db, err = util.OpenDB(bm.getDSN(constants.TidbMetaDB)) + if err := db.Ping(); err != nil { + klog.Warningf("can't connect to tidb cluster %s, err: %s", bm, err) + return false, nil + } + return true, nil + }) + + if err != nil { + klog.Errorf("cluster %s connect failed, err: %s", bm, err) + return bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ + Type: v1alpha1.BackupFailed, + Status: corev1.ConditionTrue, + Reason: "ConnectTidbFailed", + Message: err.Error(), + }) + } + + defer db.Close() + return bm.performBackup(backup.DeepCopy(), db) } -func (bm *Manager) performBackup(backup *v1alpha1.Backup) error { +func (bm *Manager) performBackup(backup *v1alpha1.Backup, db *sql.DB) error { started := time.Now() err := bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ @@ -74,14 +100,92 @@ func (bm *Manager) performBackup(backup *v1alpha1.Backup) error { return err } - backupFullPath, err := bm.backupData(backup) + oldTikvGCTime, err := bm.getTikvGCLifeTime(db) + if err != nil { + klog.Errorf("cluster %s get %s failed, err: %s", bm, constants.TikvGCVariable, err) + return bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ + Type: v1alpha1.BackupFailed, + Status: corev1.ConditionTrue, + Reason: "GetTikvGCLifeTimeFailed", + Message: err.Error(), + }) + } + klog.Infof("cluster %s %s is %s", bm, constants.TikvGCVariable, oldTikvGCTime) + + oldTikvGCTimeDuration, err := time.ParseDuration(oldTikvGCTime) if err != nil { + klog.Errorf("cluster %s parse old %s failed, err: %s", bm, constants.TikvGCVariable, err) + return bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ + Type: v1alpha1.BackupFailed, + Status: corev1.ConditionTrue, + Reason: "ParseOldTikvGCLifeTimeFailed", + Message: err.Error(), + }) + } + + var tikvGCTimeDuration time.Duration + var tikvGCLifeTime string + if backup.Spec.TikvGCLifeTime != nil { + tikvGCLifeTime = *backup.Spec.TikvGCLifeTime + tikvGCTimeDuration, err = time.ParseDuration(tikvGCLifeTime) + if err != nil { + klog.Errorf("cluster %s parse configured %s failed, err: %s", bm, constants.TikvGCVariable, err) + return bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ + Type: v1alpha1.BackupFailed, + Status: corev1.ConditionTrue, + Reason: "ParseConfiguredTikvGCLifeTimeFailed", + Message: err.Error(), + }) + } + } else { + tikvGCLifeTime = constants.TikvGCLifeTime + tikvGCTimeDuration, err = time.ParseDuration(tikvGCLifeTime) + if err != nil { + klog.Errorf("cluster %s parse default %s failed, err: %s", bm, constants.TikvGCVariable, err) + return bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ + Type: v1alpha1.BackupFailed, + Status: corev1.ConditionTrue, + Reason: "ParseDefaultTikvGCLifeTimeFailed", + Message: err.Error(), + }) + } + } + + if oldTikvGCTimeDuration < tikvGCTimeDuration { + err = bm.setTikvGCLifeTime(db, tikvGCLifeTime) + if err != nil { + klog.Errorf("cluster %s set tikv GC life time to %s failed, err: %s", bm, tikvGCLifeTime, err) + return bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ + Type: v1alpha1.BackupFailed, + Status: corev1.ConditionTrue, + Reason: "SetTikvGCLifeTimeFailed", + Message: err.Error(), + }) + } + klog.Infof("set cluster %s %s to %s success", bm, constants.TikvGCVariable, tikvGCLifeTime) + } + + backupFullPath, backupErr := bm.backupData(backup) + if oldTikvGCTimeDuration < tikvGCTimeDuration { + err = bm.setTikvGCLifeTime(db, oldTikvGCTime) + if err != nil { + klog.Errorf("cluster %s reset tikv GC life time to %s failed, err: %s", bm, oldTikvGCTime, err) + return bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ + Type: v1alpha1.BackupFailed, + Status: corev1.ConditionTrue, + Reason: "ResetTikvGCLifeTimeFailed", + Message: err.Error(), + }) + } + klog.Infof("reset cluster %s %s to %s success", bm, constants.TikvGCVariable, oldTikvGCTime) + } + if backupErr != nil { klog.Errorf("backup cluster %s data failed, err: %s", bm, err) return bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ Type: v1alpha1.BackupFailed, Status: corev1.ConditionTrue, Reason: "BackupDataToRemoteFailed", - Message: err.Error(), + Message: backupErr.Error(), }) } klog.Infof("backup cluster %s data to %s success", bm, backupFullPath) diff --git a/cmd/backup-manager/app/cmd/backup.go b/cmd/backup-manager/app/cmd/backup.go index a67c95dcb1..5e4cd26a7d 100644 --- a/cmd/backup-manager/app/cmd/backup.go +++ b/cmd/backup-manager/app/cmd/backup.go @@ -19,6 +19,7 @@ import ( "github.com/pingcap/tidb-operator/cmd/backup-manager/app/backup" "github.com/pingcap/tidb-operator/cmd/backup-manager/app/constants" "github.com/pingcap/tidb-operator/cmd/backup-manager/app/util" + bkconstants "github.com/pingcap/tidb-operator/pkg/backup/constants" informers "github.com/pingcap/tidb-operator/pkg/client/informers/externalversions" "github.com/pingcap/tidb-operator/pkg/controller" "github.com/spf13/cobra" @@ -42,6 +43,11 @@ func NewBackupCommand() *cobra.Command { cmd.Flags().StringVar(&bo.Namespace, "namespace", "", "Backup CR's namespace") cmd.Flags().StringVar(&bo.BackupName, "backupName", "", "Backup CRD object name") + cmd.Flags().StringVar(&bo.Host, "host", "", "Tidb cluster access address") + cmd.Flags().Int32Var(&bo.Port, "port", bkconstants.DefaultTidbPort, "Port number to use for connecting tidb cluster") + cmd.Flags().StringVar(&bo.Password, bkconstants.TidbPasswordKey, "", "Password to use when connecting to tidb cluster") + cmd.Flags().StringVar(&bo.User, "user", "", "User for login tidb cluster") + util.SetFlagsFromEnv(cmd.Flags(), bkconstants.BackupManagerEnvVarPrefix) return cmd } diff --git a/cmd/backup-manager/app/export/manager.go b/cmd/backup-manager/app/export/manager.go index 22148294e2..436e8bcc7b 100644 --- a/cmd/backup-manager/app/export/manager.go +++ b/cmd/backup-manager/app/export/manager.go @@ -64,11 +64,6 @@ func (bm *BackupManager) ProcessBackup() error { var db *sql.DB err = wait.PollImmediate(constants.PollInterval, constants.CheckTimeout, func() (done bool, err error) { db, err = util.OpenDB(bm.getDSN(constants.TidbMetaDB)) - if err != nil { - klog.Warningf("can't open connection to tidb cluster %s, err: %v", bm, err) - return false, nil - } - if err := db.Ping(); err != nil { klog.Warningf("can't connect to tidb cluster %s, err: %s", bm, err) return false, nil @@ -123,16 +118,35 @@ func (bm *BackupManager) performBackup(backup *v1alpha1.Backup, db *sql.DB) erro Message: err.Error(), }) } - tikvGCTimeDuration, err := time.ParseDuration(constants.TikvGCLifeTime) - if err != nil { - klog.Errorf("cluster %s parse default %s failed, err: %s", bm, constants.TikvGCVariable, err) - return bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ - Type: v1alpha1.BackupFailed, - Status: corev1.ConditionTrue, - Reason: "ParseDefaultTikvGCLifeTimeFailed", - Message: err.Error(), - }) + + var tikvGCTimeDuration time.Duration + var tikvGCLifeTime string + if backup.Spec.TikvGCLifeTime != nil { + tikvGCLifeTime = *backup.Spec.TikvGCLifeTime + tikvGCTimeDuration, err = time.ParseDuration(tikvGCLifeTime) + if err != nil { + klog.Errorf("cluster %s parse configured %s failed, err: %s", bm, constants.TikvGCVariable, err) + return bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ + Type: v1alpha1.BackupFailed, + Status: corev1.ConditionTrue, + Reason: "ParseConfiguredTikvGCLifeTimeFailed", + Message: err.Error(), + }) + } + } else { + tikvGCLifeTime = constants.TikvGCLifeTime + tikvGCTimeDuration, err = time.ParseDuration(tikvGCLifeTime) + if err != nil { + klog.Errorf("cluster %s parse default %s failed, err: %s", bm, constants.TikvGCVariable, err) + return bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ + Type: v1alpha1.BackupFailed, + Status: corev1.ConditionTrue, + Reason: "ParseDefaultTikvGCLifeTimeFailed", + Message: err.Error(), + }) + } } + if oldTikvGCTimeDuration < tikvGCTimeDuration { err = bm.setTikvGCLifeTime(db, constants.TikvGCLifeTime) if err != nil { @@ -147,18 +161,7 @@ func (bm *BackupManager) performBackup(backup *v1alpha1.Backup, db *sql.DB) erro klog.Infof("set cluster %s %s to %s success", bm, constants.TikvGCVariable, constants.TikvGCLifeTime) } - backupFullPath, err := bm.dumpTidbClusterData() - if err != nil { - klog.Errorf("dump cluster %s data failed, err: %s", bm, err) - return bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ - Type: v1alpha1.BackupFailed, - Status: corev1.ConditionTrue, - Reason: "DumpTidbClusterFailed", - Message: err.Error(), - }) - } - klog.Infof("dump cluster %s data to %s success", bm, backupFullPath) - + backupFullPath, backupErr := bm.dumpTidbClusterData() if oldTikvGCTimeDuration < tikvGCTimeDuration { err = bm.setTikvGCLifeTime(db, oldTikvGCTime) if err != nil { @@ -172,6 +175,17 @@ func (bm *BackupManager) performBackup(backup *v1alpha1.Backup, db *sql.DB) erro } klog.Infof("reset cluster %s %s to %s success", bm, constants.TikvGCVariable, oldTikvGCTime) } + if backupErr != nil { + klog.Errorf("dump cluster %s data failed, err: %s", bm, err) + return bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ + Type: v1alpha1.BackupFailed, + Status: corev1.ConditionTrue, + Reason: "DumpTidbClusterFailed", + Message: backupErr.Error(), + }) + } + klog.Infof("dump cluster %s data to %s success", bm, backupFullPath) + // TODO: Concurrent get file size and upload backup data to speed up processing time archiveBackupPath := backupFullPath + constants.DefaultArchiveExtention err = archiveBackupData(backupFullPath, archiveBackupPath) diff --git a/images/tidb-backup-manager/Dockerfile b/images/tidb-backup-manager/Dockerfile index 32f0940baa..cbfcff05c4 100644 --- a/images/tidb-backup-manager/Dockerfile +++ b/images/tidb-backup-manager/Dockerfile @@ -9,6 +9,12 @@ RUN wget -nv https://github.com/ncw/rclone/releases/download/${VERSION}/rclone-$ && chmod 755 /usr/local/bin/rclone \ && rm -rf rclone-${VERSION}-linux-amd64.zip rclone-${VERSION}-linux-amd64 +RUN wget -nv http://fileserver.pingcap.net/download/builds/pingcap/br/latest/centos7/br.tar.gz \ + && tar -xzf br.tar.gz \ + && mv bin/br /usr/local/bin \ + && chmod 755 /usr/local/bin/br \ + && rm -rf br.tar.gz + COPY bin/tidb-backup-manager /tidb-backup-manager COPY entrypoint.sh /entrypoint.sh diff --git a/pkg/apis/pingcap/v1alpha1/types.go b/pkg/apis/pingcap/v1alpha1/types.go index 80b26c0f5c..5cfc9cc451 100644 --- a/pkg/apis/pingcap/v1alpha1/types.go +++ b/pkg/apis/pingcap/v1alpha1/types.go @@ -765,6 +765,8 @@ type BackupSpec struct { From TiDBAccessConfig `json:"from,omitempty"` // Type is the backup type for tidb cluster. Type BackupType `json:"backupType,omitempty"` + // TikvGCLifeTime is to specify the safe gc life time for backup.The time limit during which data is retained for each GC, in the format of Go Duration. When a GC happens, the current time minus this value is the safe point. + TikvGCLifeTime *string `json:"tikvGCLifeTime,omitempty"` // StorageProvider configures where and how backups should be stored. StorageProvider `json:",inline"` // The storageClassName of the persistent volume for Backup data storage. diff --git a/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go index 6f152d33b2..8e5f22fdfe 100644 --- a/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go @@ -265,6 +265,11 @@ func (in *BackupScheduleStatus) DeepCopy() *BackupScheduleStatus { func (in *BackupSpec) DeepCopyInto(out *BackupSpec) { *out = *in out.From = in.From + if in.TikvGCLifeTime != nil { + in, out := &in.TikvGCLifeTime, &out.TikvGCLifeTime + *out = new(string) + **out = **in + } in.StorageProvider.DeepCopyInto(&out.StorageProvider) if in.StorageClassName != nil { in, out := &in.StorageClassName, &out.StorageClassName diff --git a/pkg/backup/backup/backup_manager.go b/pkg/backup/backup/backup_manager.go index 9cbf246041..3971c63269 100644 --- a/pkg/backup/backup/backup_manager.go +++ b/pkg/backup/backup/backup_manager.go @@ -250,19 +250,30 @@ func (bm *backupManager) makeExportJob(backup *v1alpha1.Backup) (*batchv1.Job, s return job, "", nil } + func (bm *backupManager) makeBackupJob(backup *v1alpha1.Backup) (*batchv1.Job, string, error) { ns := backup.GetNamespace() name := backup.GetName() - envVars, reason, err := backuputil.GenerateStorageCertEnv(ns, backup.Spec.StorageProvider, bm.secretLister) + envVars, reason, err := backuputil.GenerateTidbPasswordEnv(ns, name, backup.Spec.From.SecretName, bm.secretLister) + if err != nil { + return nil, reason, err + } + + storageEnv, reason, err := backuputil.GenerateStorageCertEnv(ns, backup.Spec.StorageProvider, bm.secretLister) if err != nil { return nil, reason, fmt.Errorf("backup %s/%s, %v", ns, name, err) } + envVars = append(envVars, storageEnv...) + args := []string{ "backup", fmt.Sprintf("--namespace=%s", ns), fmt.Sprintf("--backupName=%s", name), + fmt.Sprintf("--host=%s", backup.Spec.From.Host), + fmt.Sprintf("--port=%d", backup.Spec.From.Port), + fmt.Sprintf("--user=%s", backup.Spec.From.User), } backupLabel := label.NewBackup().Instance(backup.GetInstanceName()).BackupJob().Backup(name) From e8cd6d6304406ea3f3ae789966f77cdfda9d4505 Mon Sep 17 00:00:00 2001 From: Louis Date: Sun, 1 Mar 2020 17:26:57 +0800 Subject: [PATCH 02/11] Backup: add GenericBackupOptions for br and mydumper --- cmd/backup-manager/app/backup/backup.go | 42 ++--------------- cmd/backup-manager/app/backup/manager.go | 14 +++--- cmd/backup-manager/app/cmd/backup.go | 4 +- cmd/backup-manager/app/export/export.go | 36 +-------------- cmd/backup-manager/app/export/manager.go | 8 ++-- cmd/backup-manager/app/util/backup.go | 59 ++++++++++++++++++++++++ images/tidb-backup-manager/Dockerfile | 11 +++-- 7 files changed, 83 insertions(+), 91 deletions(-) create mode 100644 cmd/backup-manager/app/util/backup.go diff --git a/cmd/backup-manager/app/backup/backup.go b/cmd/backup-manager/app/backup/backup.go index a602f664b8..506ba30e7c 100644 --- a/cmd/backup-manager/app/backup/backup.go +++ b/cmd/backup-manager/app/backup/backup.go @@ -15,7 +15,6 @@ package backup import ( "context" - "database/sql" "fmt" "io" "os/exec" @@ -29,21 +28,12 @@ import ( "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" ) -// Options contains the input arguments to the backup command -type Options struct { - Namespace string - BackupName string - Host string - Port int32 - Password string - User string +// BackupOpts contains the input arguments to the backup command +type BackupOpts struct { + util.GenericBackupOptions } -func (bo *Options) String() string { - return fmt.Sprintf("%s/%s", bo.Namespace, bo.BackupName) -} - -func (bo *Options) backupData(backup *v1alpha1.Backup) (string, error) { +func (bo *BackupOpts) backupData(backup *v1alpha1.Backup) (string, error) { args, path, err := constructOptions(backup) if err != nil { return "", err @@ -68,30 +58,6 @@ func (bo *Options) backupData(backup *v1alpha1.Backup) (string, error) { return path, nil } -func (bo *Options) getTikvGCLifeTime(db *sql.DB) (string, error) { - var tikvGCTime string - sql := fmt.Sprintf("select variable_value from %s where variable_name= ?", constants.TidbMetaTable) - row := db.QueryRow(sql, constants.TikvGCVariable) - err := row.Scan(&tikvGCTime) - if err != nil { - return tikvGCTime, fmt.Errorf("query cluster %s %s failed, sql: %s, err: %v", bo, constants.TikvGCVariable, sql, err) - } - return tikvGCTime, nil -} - -func (bo *Options) setTikvGCLifeTime(db *sql.DB, gcTime string) error { - sql := fmt.Sprintf("update %s set variable_value = ? where variable_name = ?", constants.TidbMetaTable) - _, err := db.Exec(sql, gcTime, constants.TikvGCVariable) - if err != nil { - return fmt.Errorf("set cluster %s %s failed, sql: %s, err: %v", bo, constants.TikvGCVariable, sql, err) - } - return nil -} - -func (bo *Options) getDSN(db string) string { - return fmt.Sprintf("%s:%s@(%s:%d)/%s?charset=utf8", bo.User, bo.Password, bo.Host, bo.Port, db) -} - // getCommitTs get backup position from `EndVersion` in BR backup meta func getCommitTs(backup *v1alpha1.Backup) (uint64, error) { var commitTs uint64 diff --git a/cmd/backup-manager/app/backup/manager.go b/cmd/backup-manager/app/backup/manager.go index c575255675..b84c2fda75 100644 --- a/cmd/backup-manager/app/backup/manager.go +++ b/cmd/backup-manager/app/backup/manager.go @@ -33,14 +33,14 @@ import ( type Manager struct { backupLister listers.BackupLister StatusUpdater controller.BackupConditionUpdaterInterface - Options + BackupOpts } // NewManager return a Manager func NewManager( backupLister listers.BackupLister, statusUpdater controller.BackupConditionUpdaterInterface, - backupOpts Options) *Manager { + backupOpts BackupOpts) *Manager { return &Manager{ backupLister, statusUpdater, @@ -67,8 +67,8 @@ func (bm *Manager) ProcessBackup() error { var db *sql.DB err = wait.PollImmediate(constants.PollInterval, constants.CheckTimeout, func() (done bool, err error) { - db, err = util.OpenDB(bm.getDSN(constants.TidbMetaDB)) - if err := db.Ping(); err != nil { + db, err = util.OpenDB(bm.GetDSN(constants.TidbMetaDB)) + if err != nil { klog.Warningf("can't connect to tidb cluster %s, err: %s", bm, err) return false, nil } @@ -100,7 +100,7 @@ func (bm *Manager) performBackup(backup *v1alpha1.Backup, db *sql.DB) error { return err } - oldTikvGCTime, err := bm.getTikvGCLifeTime(db) + oldTikvGCTime, err := bm.GetTikvGCLifeTime(db) if err != nil { klog.Errorf("cluster %s get %s failed, err: %s", bm, constants.TikvGCVariable, err) return bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ @@ -152,7 +152,7 @@ func (bm *Manager) performBackup(backup *v1alpha1.Backup, db *sql.DB) error { } if oldTikvGCTimeDuration < tikvGCTimeDuration { - err = bm.setTikvGCLifeTime(db, tikvGCLifeTime) + err = bm.SetTikvGCLifeTime(db, tikvGCLifeTime) if err != nil { klog.Errorf("cluster %s set tikv GC life time to %s failed, err: %s", bm, tikvGCLifeTime, err) return bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ @@ -167,7 +167,7 @@ func (bm *Manager) performBackup(backup *v1alpha1.Backup, db *sql.DB) error { backupFullPath, backupErr := bm.backupData(backup) if oldTikvGCTimeDuration < tikvGCTimeDuration { - err = bm.setTikvGCLifeTime(db, oldTikvGCTime) + err = bm.SetTikvGCLifeTime(db, oldTikvGCTime) if err != nil { klog.Errorf("cluster %s reset tikv GC life time to %s failed, err: %s", bm, oldTikvGCTime, err) return bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ diff --git a/cmd/backup-manager/app/cmd/backup.go b/cmd/backup-manager/app/cmd/backup.go index 5e4cd26a7d..32bc79900d 100644 --- a/cmd/backup-manager/app/cmd/backup.go +++ b/cmd/backup-manager/app/cmd/backup.go @@ -30,7 +30,7 @@ import ( // NewBackupCommand implements the backup command func NewBackupCommand() *cobra.Command { - bo := backup.Options{} + bo := backup.BackupOpts{} cmd := &cobra.Command{ Use: "backup", @@ -51,7 +51,7 @@ func NewBackupCommand() *cobra.Command { return cmd } -func runBackup(backupOpts backup.Options, kubecfg string) error { +func runBackup(backupOpts backup.BackupOpts, kubecfg string) error { kubeCli, cli, err := util.NewKubeAndCRCli(kubecfg) cmdutil.CheckErr(err) options := []informers.SharedInformerOption{ diff --git a/cmd/backup-manager/app/export/export.go b/cmd/backup-manager/app/export/export.go index 91e73ca175..26abe89135 100644 --- a/cmd/backup-manager/app/export/export.go +++ b/cmd/backup-manager/app/export/export.go @@ -14,7 +14,6 @@ package export import ( - "database/sql" "fmt" "io/ioutil" "os/exec" @@ -32,20 +31,11 @@ import ( // BackupOpts contains the input arguments to the backup command type BackupOpts struct { - Namespace string - BackupName string + util.GenericBackupOptions Bucket string - Host string - Port int32 - Password string - User string StorageType string } -func (bo *BackupOpts) String() string { - return fmt.Sprintf("%s/%s", bo.Namespace, bo.BackupName) -} - func (bo *BackupOpts) getBackupFullPath() string { return filepath.Join(constants.BackupRootPath, bo.getBackupRelativePath()) } @@ -59,26 +49,6 @@ func (bo *BackupOpts) getDestBucketURI(remotePath string) string { return fmt.Sprintf("%s://%s", bo.StorageType, remotePath) } -func (bo *BackupOpts) getTikvGCLifeTime(db *sql.DB) (string, error) { - var tikvGCTime string - sql := fmt.Sprintf("select variable_value from %s where variable_name= ?", constants.TidbMetaTable) - row := db.QueryRow(sql, constants.TikvGCVariable) - err := row.Scan(&tikvGCTime) - if err != nil { - return tikvGCTime, fmt.Errorf("query cluster %s %s failed, sql: %s, err: %v", bo, constants.TikvGCVariable, sql, err) - } - return tikvGCTime, nil -} - -func (bo *BackupOpts) setTikvGCLifeTime(db *sql.DB, gcTime string) error { - sql := fmt.Sprintf("update %s set variable_value = ? where variable_name = ?", constants.TidbMetaTable) - _, err := db.Exec(sql, gcTime, constants.TikvGCVariable) - if err != nil { - return fmt.Errorf("set cluster %s %s failed, sql: %s, err: %v", bo, constants.TikvGCVariable, sql, err) - } - return nil -} - func (bo *BackupOpts) dumpTidbClusterData() (string, error) { bfPath := bo.getBackupFullPath() err := util.EnsureDirectoryExist(bfPath) @@ -125,10 +95,6 @@ func (bo *BackupOpts) backupDataToRemote(source, bucketURI string) error { return nil } -func (bo *BackupOpts) getDSN(db string) string { - return fmt.Sprintf("%s:%s@(%s:%d)/%s?charset=utf8", bo.User, bo.Password, bo.Host, bo.Port, db) -} - /* getCommitTsFromMetadata get commitTs from mydumper's metadata file diff --git a/cmd/backup-manager/app/export/manager.go b/cmd/backup-manager/app/export/manager.go index 436e8bcc7b..061fb9e622 100644 --- a/cmd/backup-manager/app/export/manager.go +++ b/cmd/backup-manager/app/export/manager.go @@ -63,7 +63,7 @@ func (bm *BackupManager) ProcessBackup() error { var db *sql.DB err = wait.PollImmediate(constants.PollInterval, constants.CheckTimeout, func() (done bool, err error) { - db, err = util.OpenDB(bm.getDSN(constants.TidbMetaDB)) + db, err = util.OpenDB(bm.GetDSN(constants.TidbMetaDB)) if err := db.Ping(); err != nil { klog.Warningf("can't connect to tidb cluster %s, err: %s", bm, err) return false, nil @@ -96,7 +96,7 @@ func (bm *BackupManager) performBackup(backup *v1alpha1.Backup, db *sql.DB) erro return err } - oldTikvGCTime, err := bm.getTikvGCLifeTime(db) + oldTikvGCTime, err := bm.GetTikvGCLifeTime(db) if err != nil { klog.Errorf("cluster %s get %s failed, err: %s", bm, constants.TikvGCVariable, err) return bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ @@ -148,7 +148,7 @@ func (bm *BackupManager) performBackup(backup *v1alpha1.Backup, db *sql.DB) erro } if oldTikvGCTimeDuration < tikvGCTimeDuration { - err = bm.setTikvGCLifeTime(db, constants.TikvGCLifeTime) + err = bm.SetTikvGCLifeTime(db, constants.TikvGCLifeTime) if err != nil { klog.Errorf("cluster %s set tikv GC life time to %s failed, err: %s", bm, constants.TikvGCLifeTime, err) return bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ @@ -163,7 +163,7 @@ func (bm *BackupManager) performBackup(backup *v1alpha1.Backup, db *sql.DB) erro backupFullPath, backupErr := bm.dumpTidbClusterData() if oldTikvGCTimeDuration < tikvGCTimeDuration { - err = bm.setTikvGCLifeTime(db, oldTikvGCTime) + err = bm.SetTikvGCLifeTime(db, oldTikvGCTime) if err != nil { klog.Errorf("cluster %s reset tikv GC life time to %s failed, err: %s", bm, oldTikvGCTime, err) return bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ diff --git a/cmd/backup-manager/app/util/backup.go b/cmd/backup-manager/app/util/backup.go new file mode 100644 index 0000000000..8874f61c70 --- /dev/null +++ b/cmd/backup-manager/app/util/backup.go @@ -0,0 +1,59 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "database/sql" + "fmt" + + "github.com/pingcap/tidb-operator/cmd/backup-manager/app/constants" +) + +// GenericBackupOptions contains the generic input arguments to the backup command +type GenericBackupOptions struct { + Namespace string + BackupName string + Host string + Port int32 + Password string + User string +} + +func (bo *GenericBackupOptions) String() string { + return fmt.Sprintf("%s/%s", bo.Namespace, bo.BackupName) +} + +func (bo *GenericBackupOptions) GetDSN(db string) string { + return fmt.Sprintf("%s:%s@(%s:%d)/%s?charset=utf8", bo.User, bo.Password, bo.Host, bo.Port, db) +} + +func (bo *GenericBackupOptions) GetTikvGCLifeTime(db *sql.DB) (string, error) { + var tikvGCTime string + sql := fmt.Sprintf("select variable_value from %s where variable_name= ?", constants.TidbMetaTable) + row := db.QueryRow(sql, constants.TikvGCVariable) + err := row.Scan(&tikvGCTime) + if err != nil { + return tikvGCTime, fmt.Errorf("query cluster %s %s failed, sql: %s, err: %v", bo, constants.TikvGCVariable, sql, err) + } + return tikvGCTime, nil +} + +func (bo *GenericBackupOptions) SetTikvGCLifeTime(db *sql.DB, gcTime string) error { + sql := fmt.Sprintf("update %s set variable_value = ? where variable_name = ?", constants.TidbMetaTable) + _, err := db.Exec(sql, gcTime, constants.TikvGCVariable) + if err != nil { + return fmt.Errorf("set cluster %s %s failed, sql: %s, err: %v", bo, constants.TikvGCVariable, sql, err) + } + return nil +} diff --git a/images/tidb-backup-manager/Dockerfile b/images/tidb-backup-manager/Dockerfile index cbfcff05c4..a9119b37e3 100644 --- a/images/tidb-backup-manager/Dockerfile +++ b/images/tidb-backup-manager/Dockerfile @@ -9,11 +9,12 @@ RUN wget -nv https://github.com/ncw/rclone/releases/download/${VERSION}/rclone-$ && chmod 755 /usr/local/bin/rclone \ && rm -rf rclone-${VERSION}-linux-amd64.zip rclone-${VERSION}-linux-amd64 -RUN wget -nv http://fileserver.pingcap.net/download/builds/pingcap/br/latest/centos7/br.tar.gz \ - && tar -xzf br.tar.gz \ - && mv bin/br /usr/local/bin \ - && chmod 755 /usr/local/bin/br \ - && rm -rf br.tar.gz +# TODO: change br's download url to public url +# RUN wget -nv http://fileserver.pingcap.net/download/builds/pingcap/br/latest/centos7/br.tar.gz \ +# && tar -xzf br.tar.gz \ +# && mv bin/br /usr/local/bin \ +# && chmod 755 /usr/local/bin/br \ +# && rm -rf br.tar.gz COPY bin/tidb-backup-manager /tidb-backup-manager COPY entrypoint.sh /entrypoint.sh From 6bae935660d4b346f08f1af0b45d3a6a8fe1d043 Mon Sep 17 00:00:00 2001 From: Louis Date: Mon, 2 Mar 2020 00:51:25 +0800 Subject: [PATCH 03/11] update crd.yaml and openapi --- manifests/crd.yaml | 12 ++++++++++++ pkg/apis/pingcap/v1alpha1/openapi_generated.go | 7 +++++++ 2 files changed, 19 insertions(+) diff --git a/manifests/crd.yaml b/manifests/crd.yaml index 8f1351d5ce..a80ab8ab2a 100644 --- a/manifests/crd.yaml +++ b/manifests/crd.yaml @@ -6904,6 +6904,12 @@ spec: storageSize: description: StorageSize is the request storage size for backup job type: string + tikvGCLifeTime: + description: TikvGCLifeTime is to specify the safe gc life time for + backup.The time limit during which data is retained for each GC, in + the format of Go Duration. When a GC happens, the current time minus + this value is the safe point. + type: string tolerations: description: Base tolerations of backup Pods, components may add more tolerations upon this respectively @@ -8620,6 +8626,12 @@ spec: description: StorageSize is the request storage size for backup job type: string + tikvGCLifeTime: + description: TikvGCLifeTime is to specify the safe gc life time + for backup.The time limit during which data is retained for each + GC, in the format of Go Duration. When a GC happens, the current + time minus this value is the safe point. + type: string tolerations: description: Base tolerations of backup Pods, components may add more tolerations upon this respectively diff --git a/pkg/apis/pingcap/v1alpha1/openapi_generated.go b/pkg/apis/pingcap/v1alpha1/openapi_generated.go index f76834fed6..08865456d6 100644 --- a/pkg/apis/pingcap/v1alpha1/openapi_generated.go +++ b/pkg/apis/pingcap/v1alpha1/openapi_generated.go @@ -723,6 +723,13 @@ func schema_pkg_apis_pingcap_v1alpha1_BackupSpec(ref common.ReferenceCallback) c Format: "", }, }, + "tikvGCLifeTime": { + SchemaProps: spec.SchemaProps{ + Description: "TikvGCLifeTime is to specify the safe gc life time for backup.The time limit during which data is retained for each GC, in the format of Go Duration. When a GC happens, the current time minus this value is the safe point.", + Type: []string{"string"}, + Format: "", + }, + }, "s3": { SchemaProps: spec.SchemaProps{ Ref: ref("github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.S3StorageProvider"), From 1767a5b0b7fcfb3dbfb9ffe41ec3e24ebfaeea92 Mon Sep 17 00:00:00 2001 From: Louis Date: Mon, 2 Mar 2020 10:56:40 +0800 Subject: [PATCH 04/11] Update pkg/apis/pingcap/v1alpha1/types.go Co-Authored-By: onlymellb --- pkg/apis/pingcap/v1alpha1/types.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/apis/pingcap/v1alpha1/types.go b/pkg/apis/pingcap/v1alpha1/types.go index 5cfc9cc451..a436a55206 100644 --- a/pkg/apis/pingcap/v1alpha1/types.go +++ b/pkg/apis/pingcap/v1alpha1/types.go @@ -765,7 +765,9 @@ type BackupSpec struct { From TiDBAccessConfig `json:"from,omitempty"` // Type is the backup type for tidb cluster. Type BackupType `json:"backupType,omitempty"` - // TikvGCLifeTime is to specify the safe gc life time for backup.The time limit during which data is retained for each GC, in the format of Go Duration. When a GC happens, the current time minus this value is the safe point. + // TikvGCLifeTime is to specify the safe gc life time for backup. + // The time limit during which data is retained for each GC, in the format of Go Duration. + // When a GC happens, the current time minus this value is the safe point. TikvGCLifeTime *string `json:"tikvGCLifeTime,omitempty"` // StorageProvider configures where and how backups should be stored. StorageProvider `json:",inline"` From 0651330b233efc9a78c56d6c66a4efc765c66efd Mon Sep 17 00:00:00 2001 From: Louis Date: Mon, 2 Mar 2020 11:46:04 +0800 Subject: [PATCH 05/11] update crd.yml and openapi --- manifests/crd.yaml | 8 ++++---- pkg/apis/pingcap/v1alpha1/openapi_generated.go | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/manifests/crd.yaml b/manifests/crd.yaml index a80ab8ab2a..35573ac50e 100644 --- a/manifests/crd.yaml +++ b/manifests/crd.yaml @@ -6906,9 +6906,9 @@ spec: type: string tikvGCLifeTime: description: TikvGCLifeTime is to specify the safe gc life time for - backup.The time limit during which data is retained for each GC, in - the format of Go Duration. When a GC happens, the current time minus - this value is the safe point. + backup. The time limit during which data is retained for each GC, + in the format of Go Duration. When a GC happens, the current time + minus this value is the safe point. type: string tolerations: description: Base tolerations of backup Pods, components may add more @@ -8628,7 +8628,7 @@ spec: type: string tikvGCLifeTime: description: TikvGCLifeTime is to specify the safe gc life time - for backup.The time limit during which data is retained for each + for backup. The time limit during which data is retained for each GC, in the format of Go Duration. When a GC happens, the current time minus this value is the safe point. type: string diff --git a/pkg/apis/pingcap/v1alpha1/openapi_generated.go b/pkg/apis/pingcap/v1alpha1/openapi_generated.go index 08865456d6..73e4b9a0ba 100644 --- a/pkg/apis/pingcap/v1alpha1/openapi_generated.go +++ b/pkg/apis/pingcap/v1alpha1/openapi_generated.go @@ -725,7 +725,7 @@ func schema_pkg_apis_pingcap_v1alpha1_BackupSpec(ref common.ReferenceCallback) c }, "tikvGCLifeTime": { SchemaProps: spec.SchemaProps{ - Description: "TikvGCLifeTime is to specify the safe gc life time for backup.The time limit during which data is retained for each GC, in the format of Go Duration. When a GC happens, the current time minus this value is the safe point.", + Description: "TikvGCLifeTime is to specify the safe gc life time for backup. The time limit during which data is retained for each GC, in the format of Go Duration. When a GC happens, the current time minus this value is the safe point.", Type: []string{"string"}, Format: "", }, From 4d9791ddb3809ff615d537cbe36a53bdddf05c3b Mon Sep 17 00:00:00 2001 From: Louis Date: Mon, 2 Mar 2020 11:55:11 +0800 Subject: [PATCH 06/11] fix check-static --- pkg/apis/pingcap/v1alpha1/types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/apis/pingcap/v1alpha1/types.go b/pkg/apis/pingcap/v1alpha1/types.go index a436a55206..951df10715 100644 --- a/pkg/apis/pingcap/v1alpha1/types.go +++ b/pkg/apis/pingcap/v1alpha1/types.go @@ -766,7 +766,7 @@ type BackupSpec struct { // Type is the backup type for tidb cluster. Type BackupType `json:"backupType,omitempty"` // TikvGCLifeTime is to specify the safe gc life time for backup. - // The time limit during which data is retained for each GC, in the format of Go Duration. + // The time limit during which data is retained for each GC, in the format of Go Duration. // When a GC happens, the current time minus this value is the safe point. TikvGCLifeTime *string `json:"tikvGCLifeTime,omitempty"` // StorageProvider configures where and how backups should be stored. From e6c3c1668fd90bdab1727dc5b422b02265322211 Mon Sep 17 00:00:00 2001 From: Louis Date: Mon, 2 Mar 2020 12:25:13 +0800 Subject: [PATCH 07/11] add br to backup-manager image --- images/tidb-backup-manager/Dockerfile | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/images/tidb-backup-manager/Dockerfile b/images/tidb-backup-manager/Dockerfile index a9119b37e3..861490bb88 100644 --- a/images/tidb-backup-manager/Dockerfile +++ b/images/tidb-backup-manager/Dockerfile @@ -9,12 +9,11 @@ RUN wget -nv https://github.com/ncw/rclone/releases/download/${VERSION}/rclone-$ && chmod 755 /usr/local/bin/rclone \ && rm -rf rclone-${VERSION}-linux-amd64.zip rclone-${VERSION}-linux-amd64 -# TODO: change br's download url to public url -# RUN wget -nv http://fileserver.pingcap.net/download/builds/pingcap/br/latest/centos7/br.tar.gz \ -# && tar -xzf br.tar.gz \ -# && mv bin/br /usr/local/bin \ -# && chmod 755 /usr/local/bin/br \ -# && rm -rf br.tar.gz +RUN wget -nv http://download.pingcap.org/br-latest-linux-amd64.tar.gz \ + && tar -xzf br-latest-linux-amd64.tar.gz \ + && mv bin/br /usr/local/bin \ + && chmod 755 /usr/local/bin/br \ + && rm -rf br-latest-linux-amd64.tar.gz COPY bin/tidb-backup-manager /tidb-backup-manager COPY entrypoint.sh /entrypoint.sh From 2c7416dd048e064e1c537b4babc4e9576fdf79dd Mon Sep 17 00:00:00 2001 From: Louis Date: Mon, 2 Mar 2020 12:35:59 +0800 Subject: [PATCH 08/11] fix backupErr in log --- cmd/backup-manager/app/backup/manager.go | 2 +- cmd/backup-manager/app/export/manager.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/backup-manager/app/backup/manager.go b/cmd/backup-manager/app/backup/manager.go index b84c2fda75..5067d2398b 100644 --- a/cmd/backup-manager/app/backup/manager.go +++ b/cmd/backup-manager/app/backup/manager.go @@ -180,7 +180,7 @@ func (bm *Manager) performBackup(backup *v1alpha1.Backup, db *sql.DB) error { klog.Infof("reset cluster %s %s to %s success", bm, constants.TikvGCVariable, oldTikvGCTime) } if backupErr != nil { - klog.Errorf("backup cluster %s data failed, err: %s", bm, err) + klog.Errorf("backup cluster %s data failed, err: %s", bm, backupErr) return bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ Type: v1alpha1.BackupFailed, Status: corev1.ConditionTrue, diff --git a/cmd/backup-manager/app/export/manager.go b/cmd/backup-manager/app/export/manager.go index 061fb9e622..c542057a86 100644 --- a/cmd/backup-manager/app/export/manager.go +++ b/cmd/backup-manager/app/export/manager.go @@ -176,7 +176,7 @@ func (bm *BackupManager) performBackup(backup *v1alpha1.Backup, db *sql.DB) erro klog.Infof("reset cluster %s %s to %s success", bm, constants.TikvGCVariable, oldTikvGCTime) } if backupErr != nil { - klog.Errorf("dump cluster %s data failed, err: %s", bm, err) + klog.Errorf("dump cluster %s data failed, err: %s", bm, backupErr) return bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ Type: v1alpha1.BackupFailed, Status: corev1.ConditionTrue, From ce4ee4e0645a6cf47980d25cd033227e9ae415f1 Mon Sep 17 00:00:00 2001 From: Louis Date: Mon, 2 Mar 2020 15:27:35 +0800 Subject: [PATCH 09/11] fix OpenDB error --- cmd/backup-manager/app/export/manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/backup-manager/app/export/manager.go b/cmd/backup-manager/app/export/manager.go index c542057a86..1991056657 100644 --- a/cmd/backup-manager/app/export/manager.go +++ b/cmd/backup-manager/app/export/manager.go @@ -64,7 +64,7 @@ func (bm *BackupManager) ProcessBackup() error { var db *sql.DB err = wait.PollImmediate(constants.PollInterval, constants.CheckTimeout, func() (done bool, err error) { db, err = util.OpenDB(bm.GetDSN(constants.TidbMetaDB)) - if err := db.Ping(); err != nil { + if err != nil { klog.Warningf("can't connect to tidb cluster %s, err: %s", bm, err) return false, nil } From e9f2010c0d74aea93313d3d4cc5f61e2fbac5bda Mon Sep 17 00:00:00 2001 From: Louis Date: Mon, 2 Mar 2020 17:22:46 +0800 Subject: [PATCH 10/11] address comments --- cmd/backup-manager/app/backup/backup.go | 6 +++--- cmd/backup-manager/app/backup/manager.go | 4 ++-- cmd/backup-manager/app/cmd/backup.go | 4 ++-- cmd/backup-manager/app/cmd/export.go | 4 ++-- cmd/backup-manager/app/export/export.go | 14 +++++++------- cmd/backup-manager/app/export/manager.go | 4 ++-- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/cmd/backup-manager/app/backup/backup.go b/cmd/backup-manager/app/backup/backup.go index 506ba30e7c..54467b2034 100644 --- a/cmd/backup-manager/app/backup/backup.go +++ b/cmd/backup-manager/app/backup/backup.go @@ -28,12 +28,12 @@ import ( "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" ) -// BackupOpts contains the input arguments to the backup command -type BackupOpts struct { +// Options contains the input arguments to the backup command +type Options struct { util.GenericBackupOptions } -func (bo *BackupOpts) backupData(backup *v1alpha1.Backup) (string, error) { +func (bo *Options) backupData(backup *v1alpha1.Backup) (string, error) { args, path, err := constructOptions(backup) if err != nil { return "", err diff --git a/cmd/backup-manager/app/backup/manager.go b/cmd/backup-manager/app/backup/manager.go index 5067d2398b..59b678710e 100644 --- a/cmd/backup-manager/app/backup/manager.go +++ b/cmd/backup-manager/app/backup/manager.go @@ -33,14 +33,14 @@ import ( type Manager struct { backupLister listers.BackupLister StatusUpdater controller.BackupConditionUpdaterInterface - BackupOpts + Options } // NewManager return a Manager func NewManager( backupLister listers.BackupLister, statusUpdater controller.BackupConditionUpdaterInterface, - backupOpts BackupOpts) *Manager { + backupOpts Options) *Manager { return &Manager{ backupLister, statusUpdater, diff --git a/cmd/backup-manager/app/cmd/backup.go b/cmd/backup-manager/app/cmd/backup.go index 32bc79900d..5e4cd26a7d 100644 --- a/cmd/backup-manager/app/cmd/backup.go +++ b/cmd/backup-manager/app/cmd/backup.go @@ -30,7 +30,7 @@ import ( // NewBackupCommand implements the backup command func NewBackupCommand() *cobra.Command { - bo := backup.BackupOpts{} + bo := backup.Options{} cmd := &cobra.Command{ Use: "backup", @@ -51,7 +51,7 @@ func NewBackupCommand() *cobra.Command { return cmd } -func runBackup(backupOpts backup.BackupOpts, kubecfg string) error { +func runBackup(backupOpts backup.Options, kubecfg string) error { kubeCli, cli, err := util.NewKubeAndCRCli(kubecfg) cmdutil.CheckErr(err) options := []informers.SharedInformerOption{ diff --git a/cmd/backup-manager/app/cmd/export.go b/cmd/backup-manager/app/cmd/export.go index 6cc0c57fb2..0d391e0eba 100644 --- a/cmd/backup-manager/app/cmd/export.go +++ b/cmd/backup-manager/app/cmd/export.go @@ -32,7 +32,7 @@ import ( // NewExportCommand implements the backup command func NewExportCommand() *cobra.Command { - bo := export.BackupOpts{} + bo := export.Options{} cmd := &cobra.Command{ Use: "export", @@ -55,7 +55,7 @@ func NewExportCommand() *cobra.Command { return cmd } -func runExport(backupOpts export.BackupOpts, kubecfg string) error { +func runExport(backupOpts export.Options, kubecfg string) error { kubeCli, cli, err := util.NewKubeAndCRCli(kubecfg) cmdutil.CheckErr(err) options := []informers.SharedInformerOption{ diff --git a/cmd/backup-manager/app/export/export.go b/cmd/backup-manager/app/export/export.go index 26abe89135..92bd67763d 100644 --- a/cmd/backup-manager/app/export/export.go +++ b/cmd/backup-manager/app/export/export.go @@ -29,27 +29,27 @@ import ( "github.com/pingcap/tidb-operator/cmd/backup-manager/app/util" ) -// BackupOpts contains the input arguments to the backup command -type BackupOpts struct { +// Options contains the input arguments to the backup command +type Options struct { util.GenericBackupOptions Bucket string StorageType string } -func (bo *BackupOpts) getBackupFullPath() string { +func (bo *Options) getBackupFullPath() string { return filepath.Join(constants.BackupRootPath, bo.getBackupRelativePath()) } -func (bo *BackupOpts) getBackupRelativePath() string { +func (bo *Options) getBackupRelativePath() string { backupName := fmt.Sprintf("backup-%s", time.Now().UTC().Format(time.RFC3339)) return fmt.Sprintf("%s/%s", bo.Bucket, backupName) } -func (bo *BackupOpts) getDestBucketURI(remotePath string) string { +func (bo *Options) getDestBucketURI(remotePath string) string { return fmt.Sprintf("%s://%s", bo.StorageType, remotePath) } -func (bo *BackupOpts) dumpTidbClusterData() (string, error) { +func (bo *Options) dumpTidbClusterData() (string, error) { bfPath := bo.getBackupFullPath() err := util.EnsureDirectoryExist(bfPath) if err != nil { @@ -75,7 +75,7 @@ func (bo *BackupOpts) dumpTidbClusterData() (string, error) { return bfPath, nil } -func (bo *BackupOpts) backupDataToRemote(source, bucketURI string) error { +func (bo *Options) backupDataToRemote(source, bucketURI string) error { destBucket := util.NormalizeBucketURI(bucketURI) tmpDestBucket := fmt.Sprintf("%s.tmp", destBucket) // TODO: We may need to use exec.CommandContext to control timeouts. diff --git a/cmd/backup-manager/app/export/manager.go b/cmd/backup-manager/app/export/manager.go index 1991056657..779a7f8494 100644 --- a/cmd/backup-manager/app/export/manager.go +++ b/cmd/backup-manager/app/export/manager.go @@ -33,14 +33,14 @@ import ( type BackupManager struct { backupLister listers.BackupLister StatusUpdater controller.BackupConditionUpdaterInterface - BackupOpts + Options } // NewBackupManager return a BackupManager func NewBackupManager( backupLister listers.BackupLister, statusUpdater controller.BackupConditionUpdaterInterface, - backupOpts BackupOpts) *BackupManager { + backupOpts Options) *BackupManager { return &BackupManager{ backupLister, statusUpdater, From 414597d2012e269ed5842e27d9d273b7e441e691 Mon Sep 17 00:00:00 2001 From: Louis Date: Tue, 3 Mar 2020 03:19:26 +0800 Subject: [PATCH 11/11] Restore: support configuring TiKV GC life time --- cmd/backup-manager/app/backup/backup.go | 5 +- cmd/backup-manager/app/backup/manager.go | 27 +++- cmd/backup-manager/app/cmd/backup.go | 8 +- cmd/backup-manager/app/cmd/export.go | 8 +- cmd/backup-manager/app/cmd/import.go | 12 +- cmd/backup-manager/app/cmd/restore.go | 4 +- cmd/backup-manager/app/export/export.go | 5 +- cmd/backup-manager/app/export/manager.go | 27 +++- cmd/backup-manager/app/import/manager.go | 44 ++++-- cmd/backup-manager/app/import/restore.go | 27 +--- cmd/backup-manager/app/restore/manager.go | 146 ++++++++++++++++-- cmd/backup-manager/app/restore/restore.go | 10 +- .../app/util/{backup.go => generic.go} | 29 ++-- cmd/backup-manager/app/util/util.go | 15 +- manifests/crd.yaml | 6 + .../pingcap/v1alpha1/openapi_generated.go | 7 + pkg/apis/pingcap/v1alpha1/types.go | 4 + .../pingcap/v1alpha1/zz_generated.deepcopy.go | 5 + pkg/backup/backup/backup_manager.go | 8 +- pkg/backup/restore/restore_manager.go | 11 +- pkg/backup/util/util.go | 26 ++-- 21 files changed, 296 insertions(+), 138 deletions(-) rename cmd/backup-manager/app/util/{backup.go => generic.go} (67%) diff --git a/cmd/backup-manager/app/backup/backup.go b/cmd/backup-manager/app/backup/backup.go index 54467b2034..18cc163a97 100644 --- a/cmd/backup-manager/app/backup/backup.go +++ b/cmd/backup-manager/app/backup/backup.go @@ -20,17 +20,16 @@ import ( "os/exec" "github.com/gogo/protobuf/proto" - "k8s.io/klog" - kvbackup "github.com/pingcap/kvproto/pkg/backup" "github.com/pingcap/tidb-operator/cmd/backup-manager/app/constants" "github.com/pingcap/tidb-operator/cmd/backup-manager/app/util" "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" + "k8s.io/klog" ) // Options contains the input arguments to the backup command type Options struct { - util.GenericBackupOptions + util.GenericOptions } func (bo *Options) backupData(backup *v1alpha1.Backup) (string, error) { diff --git a/cmd/backup-manager/app/backup/manager.go b/cmd/backup-manager/app/backup/manager.go index 59b678710e..f4389b1d48 100644 --- a/cmd/backup-manager/app/backup/manager.go +++ b/cmd/backup-manager/app/backup/manager.go @@ -21,6 +21,7 @@ import ( "github.com/pingcap/tidb-operator/cmd/backup-manager/app/constants" "github.com/pingcap/tidb-operator/cmd/backup-manager/app/util" "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" + bkconstants "github.com/pingcap/tidb-operator/pkg/backup/constants" listers "github.com/pingcap/tidb-operator/pkg/client/listers/pingcap/v1alpha1" "github.com/pingcap/tidb-operator/pkg/controller" corev1 "k8s.io/api/core/v1" @@ -48,11 +49,29 @@ func NewManager( } } +func (bm *Manager) setOptions(backup *v1alpha1.Backup) { + bm.Options.Host = backup.Spec.From.Host + + if backup.Spec.From.Port != 0 { + bm.Options.Port = backup.Spec.From.Port + } else { + bm.Options.Port = bkconstants.DefaultTidbPort + } + + if backup.Spec.From.User != "" { + bm.Options.User = backup.Spec.From.User + } else { + bm.Options.User = bkconstants.DefaultTidbUser + } + + bm.Options.Password = util.GetOptionValueFromEnv(bkconstants.TidbPasswordKey, bkconstants.BackupManagerEnvVarPrefix) +} + // ProcessBackup used to process the backup logic func (bm *Manager) ProcessBackup() error { - backup, err := bm.backupLister.Backups(bm.Namespace).Get(bm.BackupName) + backup, err := bm.backupLister.Backups(bm.Namespace).Get(bm.ResourceName) if err != nil { - klog.Errorf("can't find cluster %s backup %s CRD object, err: %v", bm, bm.BackupName, err) + klog.Errorf("can't find cluster %s backup %s CRD object, err: %v", bm, bm.ResourceName, err) return bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ Type: v1alpha1.BackupFailed, Status: corev1.ConditionTrue, @@ -65,9 +84,11 @@ func (bm *Manager) ProcessBackup() error { return fmt.Errorf("no br config in %s", bm) } + bm.setOptions(backup) + var db *sql.DB err = wait.PollImmediate(constants.PollInterval, constants.CheckTimeout, func() (done bool, err error) { - db, err = util.OpenDB(bm.GetDSN(constants.TidbMetaDB)) + db, err = util.OpenDB(bm.GetDSN()) if err != nil { klog.Warningf("can't connect to tidb cluster %s, err: %s", bm, err) return false, nil diff --git a/cmd/backup-manager/app/cmd/backup.go b/cmd/backup-manager/app/cmd/backup.go index 5e4cd26a7d..621103a7be 100644 --- a/cmd/backup-manager/app/cmd/backup.go +++ b/cmd/backup-manager/app/cmd/backup.go @@ -19,7 +19,6 @@ import ( "github.com/pingcap/tidb-operator/cmd/backup-manager/app/backup" "github.com/pingcap/tidb-operator/cmd/backup-manager/app/constants" "github.com/pingcap/tidb-operator/cmd/backup-manager/app/util" - bkconstants "github.com/pingcap/tidb-operator/pkg/backup/constants" informers "github.com/pingcap/tidb-operator/pkg/client/informers/externalversions" "github.com/pingcap/tidb-operator/pkg/controller" "github.com/spf13/cobra" @@ -42,12 +41,7 @@ func NewBackupCommand() *cobra.Command { } cmd.Flags().StringVar(&bo.Namespace, "namespace", "", "Backup CR's namespace") - cmd.Flags().StringVar(&bo.BackupName, "backupName", "", "Backup CRD object name") - cmd.Flags().StringVar(&bo.Host, "host", "", "Tidb cluster access address") - cmd.Flags().Int32Var(&bo.Port, "port", bkconstants.DefaultTidbPort, "Port number to use for connecting tidb cluster") - cmd.Flags().StringVar(&bo.Password, bkconstants.TidbPasswordKey, "", "Password to use when connecting to tidb cluster") - cmd.Flags().StringVar(&bo.User, "user", "", "User for login tidb cluster") - util.SetFlagsFromEnv(cmd.Flags(), bkconstants.BackupManagerEnvVarPrefix) + cmd.Flags().StringVar(&bo.ResourceName, "backupName", "", "Backup CRD object name") return cmd } diff --git a/cmd/backup-manager/app/cmd/export.go b/cmd/backup-manager/app/cmd/export.go index 0d391e0eba..f4c19f2d63 100644 --- a/cmd/backup-manager/app/cmd/export.go +++ b/cmd/backup-manager/app/cmd/export.go @@ -21,7 +21,6 @@ import ( "github.com/pingcap/tidb-operator/cmd/backup-manager/app/constants" "github.com/pingcap/tidb-operator/cmd/backup-manager/app/export" "github.com/pingcap/tidb-operator/cmd/backup-manager/app/util" - bkconstants "github.com/pingcap/tidb-operator/pkg/backup/constants" informers "github.com/pingcap/tidb-operator/pkg/client/informers/externalversions" "github.com/pingcap/tidb-operator/pkg/controller" "github.com/spf13/cobra" @@ -44,14 +43,9 @@ func NewExportCommand() *cobra.Command { } cmd.Flags().StringVar(&bo.Namespace, "namespace", "", "Backup CR's namespace") - cmd.Flags().StringVar(&bo.Host, "host", "", "Tidb cluster access address") - cmd.Flags().Int32Var(&bo.Port, "port", bkconstants.DefaultTidbPort, "Port number to use for connecting tidb cluster") + cmd.Flags().StringVar(&bo.ResourceName, "backupName", "", "Backup CRD object name") cmd.Flags().StringVar(&bo.Bucket, "bucket", "", "Bucket in which to store the backup data") - cmd.Flags().StringVar(&bo.Password, bkconstants.TidbPasswordKey, "", "Password to use when connecting to tidb cluster") - cmd.Flags().StringVar(&bo.User, "user", "", "User for login tidb cluster") cmd.Flags().StringVar(&bo.StorageType, "storageType", "", "Backend storage type") - cmd.Flags().StringVar(&bo.BackupName, "backupName", "", "Backup CRD object name") - util.SetFlagsFromEnv(cmd.Flags(), bkconstants.BackupManagerEnvVarPrefix) return cmd } diff --git a/cmd/backup-manager/app/cmd/import.go b/cmd/backup-manager/app/cmd/import.go index 9efbe03946..89a796128a 100644 --- a/cmd/backup-manager/app/cmd/import.go +++ b/cmd/backup-manager/app/cmd/import.go @@ -21,7 +21,6 @@ import ( "github.com/pingcap/tidb-operator/cmd/backup-manager/app/constants" "github.com/pingcap/tidb-operator/cmd/backup-manager/app/import" "github.com/pingcap/tidb-operator/cmd/backup-manager/app/util" - bkconstants "github.com/pingcap/tidb-operator/pkg/backup/constants" informers "github.com/pingcap/tidb-operator/pkg/client/informers/externalversions" "github.com/pingcap/tidb-operator/pkg/controller" "github.com/spf13/cobra" @@ -32,7 +31,7 @@ import ( // NewImportCommand implements the restore command func NewImportCommand() *cobra.Command { - ro := _import.RestoreOpts{} + ro := _import.Options{} cmd := &cobra.Command{ Use: "import", @@ -44,17 +43,12 @@ func NewImportCommand() *cobra.Command { } cmd.Flags().StringVar(&ro.Namespace, "namespace", "", "Restore CR's namespace") - cmd.Flags().StringVar(&ro.Host, "host", "", "Tidb cluster access address") - cmd.Flags().Int32Var(&ro.Port, "port", bkconstants.DefaultTidbPort, "Port number to use for connecting tidb cluster") - cmd.Flags().StringVar(&ro.Password, bkconstants.TidbPasswordKey, "", "Password to use when connecting to tidb cluster") - cmd.Flags().StringVar(&ro.User, "user", "", "User for login tidb cluster") - cmd.Flags().StringVar(&ro.RestoreName, "restoreName", "", "Restore CRD object name") + cmd.Flags().StringVar(&ro.ResourceName, "restoreName", "", "Restore CRD object name") cmd.Flags().StringVar(&ro.BackupPath, "backupPath", "", "The location of the backup") - util.SetFlagsFromEnv(cmd.Flags(), bkconstants.BackupManagerEnvVarPrefix) return cmd } -func runImport(restoreOpts _import.RestoreOpts, kubecfg string) error { +func runImport(restoreOpts _import.Options, kubecfg string) error { kubeCli, cli, err := util.NewKubeAndCRCli(kubecfg) cmdutil.CheckErr(err) options := []informers.SharedInformerOption{ diff --git a/cmd/backup-manager/app/cmd/restore.go b/cmd/backup-manager/app/cmd/restore.go index d5835c2ad6..7b6226083b 100644 --- a/cmd/backup-manager/app/cmd/restore.go +++ b/cmd/backup-manager/app/cmd/restore.go @@ -21,7 +21,6 @@ import ( "github.com/pingcap/tidb-operator/cmd/backup-manager/app/constants" "github.com/pingcap/tidb-operator/cmd/backup-manager/app/restore" "github.com/pingcap/tidb-operator/cmd/backup-manager/app/util" - bkconstants "github.com/pingcap/tidb-operator/pkg/backup/constants" informers "github.com/pingcap/tidb-operator/pkg/client/informers/externalversions" "github.com/pingcap/tidb-operator/pkg/controller" "github.com/spf13/cobra" @@ -44,8 +43,7 @@ func NewRestoreCommand() *cobra.Command { } cmd.Flags().StringVar(&ro.Namespace, "namespace", "", "Restore CR's namespace") - cmd.Flags().StringVar(&ro.RestoreName, "restoreName", "", "Restore CRD object name") - util.SetFlagsFromEnv(cmd.Flags(), bkconstants.BackupManagerEnvVarPrefix) + cmd.Flags().StringVar(&ro.ResourceName, "restoreName", "", "Restore CRD object name") return cmd } diff --git a/cmd/backup-manager/app/export/export.go b/cmd/backup-manager/app/export/export.go index 92bd67763d..ff7a0e19f5 100644 --- a/cmd/backup-manager/app/export/export.go +++ b/cmd/backup-manager/app/export/export.go @@ -23,15 +23,14 @@ import ( "time" "github.com/mholt/archiver" - "k8s.io/klog" - "github.com/pingcap/tidb-operator/cmd/backup-manager/app/constants" "github.com/pingcap/tidb-operator/cmd/backup-manager/app/util" + "k8s.io/klog" ) // Options contains the input arguments to the backup command type Options struct { - util.GenericBackupOptions + util.GenericOptions Bucket string StorageType string } diff --git a/cmd/backup-manager/app/export/manager.go b/cmd/backup-manager/app/export/manager.go index 779a7f8494..b4a488e6ef 100644 --- a/cmd/backup-manager/app/export/manager.go +++ b/cmd/backup-manager/app/export/manager.go @@ -21,6 +21,7 @@ import ( "github.com/pingcap/tidb-operator/cmd/backup-manager/app/constants" "github.com/pingcap/tidb-operator/cmd/backup-manager/app/util" "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" + bkconstants "github.com/pingcap/tidb-operator/pkg/backup/constants" listers "github.com/pingcap/tidb-operator/pkg/client/listers/pingcap/v1alpha1" "github.com/pingcap/tidb-operator/pkg/controller" corev1 "k8s.io/api/core/v1" @@ -48,11 +49,29 @@ func NewBackupManager( } } +func (bm *BackupManager) setOptions(backup *v1alpha1.Backup) { + bm.Options.Host = backup.Spec.From.Host + + if backup.Spec.From.Port != 0 { + bm.Options.Port = backup.Spec.From.Port + } else { + bm.Options.Port = bkconstants.DefaultTidbPort + } + + if backup.Spec.From.User != "" { + bm.Options.User = backup.Spec.From.User + } else { + bm.Options.User = bkconstants.DefaultTidbUser + } + + bm.Options.Password = util.GetOptionValueFromEnv(bkconstants.TidbPasswordKey, bkconstants.BackupManagerEnvVarPrefix) +} + // ProcessBackup used to process the backup logic func (bm *BackupManager) ProcessBackup() error { - backup, err := bm.backupLister.Backups(bm.Namespace).Get(bm.BackupName) + backup, err := bm.backupLister.Backups(bm.Namespace).Get(bm.ResourceName) if err != nil { - klog.Errorf("can't find cluster %s backup %s CRD object, err: %v", bm, bm.BackupName, err) + klog.Errorf("can't find cluster %s backup %s CRD object, err: %v", bm, bm.ResourceName, err) return bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ Type: v1alpha1.BackupFailed, Status: corev1.ConditionTrue, @@ -61,9 +80,11 @@ func (bm *BackupManager) ProcessBackup() error { }) } + bm.setOptions(backup) + var db *sql.DB err = wait.PollImmediate(constants.PollInterval, constants.CheckTimeout, func() (done bool, err error) { - db, err = util.OpenDB(bm.GetDSN(constants.TidbMetaDB)) + db, err = util.OpenDB(bm.GetDSN()) if err != nil { klog.Warningf("can't connect to tidb cluster %s, err: %s", bm, err) return false, nil diff --git a/cmd/backup-manager/app/import/manager.go b/cmd/backup-manager/app/import/manager.go index cc17881052..3d6ac2ce31 100644 --- a/cmd/backup-manager/app/import/manager.go +++ b/cmd/backup-manager/app/import/manager.go @@ -14,6 +14,7 @@ package _import import ( + "database/sql" "fmt" "path/filepath" "time" @@ -21,6 +22,7 @@ import ( "github.com/pingcap/tidb-operator/cmd/backup-manager/app/constants" "github.com/pingcap/tidb-operator/cmd/backup-manager/app/util" "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" + bkconstants "github.com/pingcap/tidb-operator/pkg/backup/constants" listers "github.com/pingcap/tidb-operator/pkg/client/listers/pingcap/v1alpha1" "github.com/pingcap/tidb-operator/pkg/controller" corev1 "k8s.io/api/core/v1" @@ -29,30 +31,48 @@ import ( "k8s.io/klog" ) -// RestoreManager mainly used to manage backup related work +// RestoreManager mainly used to manage restore related work type RestoreManager struct { restoreLister listers.RestoreLister StatusUpdater controller.RestoreConditionUpdaterInterface - RestoreOpts + Options } // NewRestoreManager return a RestoreManager func NewRestoreManager( restoreLister listers.RestoreLister, statusUpdater controller.RestoreConditionUpdaterInterface, - backupOpts RestoreOpts) *RestoreManager { + restoreOpts Options) *RestoreManager { return &RestoreManager{ restoreLister, statusUpdater, - backupOpts, + restoreOpts, } } +func (rm *RestoreManager) setOptions(restore *v1alpha1.Restore) { + rm.Options.Host = restore.Spec.To.Host + + if restore.Spec.To.Port != 0 { + rm.Options.Port = restore.Spec.To.Port + } else { + rm.Options.Port = bkconstants.DefaultTidbPort + } + + if restore.Spec.To.User != "" { + rm.Options.User = restore.Spec.To.User + } else { + rm.Options.User = bkconstants.DefaultTidbUser + } + + rm.Options.Password = util.GetOptionValueFromEnv(bkconstants.TidbPasswordKey, bkconstants.BackupManagerEnvVarPrefix) +} + // ProcessRestore used to process the restore logic func (rm *RestoreManager) ProcessRestore() error { - restore, err := rm.restoreLister.Restores(rm.Namespace).Get(rm.RestoreName) + restore, err := rm.restoreLister.Restores(rm.Namespace).Get(rm.ResourceName) if err != nil { - klog.Errorf("can't find cluster %s restore %s CRD object, err: %v", rm, rm.RestoreName, err) + klog.Errorf("can't find cluster %s restore %s CRD object, err: %v", rm, rm.ResourceName, err) return rm.StatusUpdater.Update(restore, &v1alpha1.RestoreCondition{ Type: v1alpha1.RestoreFailed, Status: corev1.ConditionTrue, @@ -61,18 +81,15 @@ func (rm *RestoreManager) ProcessRestore() error { }) } + rm.setOptions(restore) + + var db *sql.DB err = wait.PollImmediate(constants.PollInterval, constants.CheckTimeout, func() (done bool, err error) { - db, err := util.OpenDB(rm.getDSN(constants.TidbMetaDB)) + db, err = util.OpenDB(rm.GetDSN()) if err != nil { - klog.Warningf("can't open connection to tidb cluster %s, err: %v", rm, err) - return false, nil - } - - if err := db.Ping(); err != nil { klog.Warningf("can't connect to tidb cluster %s, err: %s", rm, err) return false, nil } - db.Close() return true, nil }) @@ -86,6 +103,7 @@ func (rm *RestoreManager) ProcessRestore() error { }) } + defer db.Close() return rm.performRestore(restore.DeepCopy()) } diff --git a/cmd/backup-manager/app/import/restore.go b/cmd/backup-manager/app/import/restore.go index 9db1c30a6d..dfcee888c4 100644 --- a/cmd/backup-manager/app/import/restore.go +++ b/cmd/backup-manager/app/import/restore.go @@ -24,28 +24,19 @@ import ( "github.com/pingcap/tidb-operator/cmd/backup-manager/app/util" ) -// RestoreOpts contains the input arguments to the restore command -type RestoreOpts struct { - Namespace string - RestoreName string - Password string - Host string - Port int32 - User string - BackupPath string +// Options contains the input arguments to the restore command +type Options struct { + util.GenericOptions + BackupPath string } -func (ro *RestoreOpts) String() string { - return fmt.Sprintf("%s/%s", ro.Namespace, ro.RestoreName) -} - -func (ro *RestoreOpts) getRestoreDataPath() string { +func (ro *Options) getRestoreDataPath() string { backupName := filepath.Base(ro.BackupPath) bucketName := filepath.Base(filepath.Dir(ro.BackupPath)) return filepath.Join(constants.BackupRootPath, bucketName, backupName) } -func (ro *RestoreOpts) downloadBackupData(localPath string) error { +func (ro *Options) downloadBackupData(localPath string) error { if err := util.EnsureDirectoryExist(filepath.Dir(localPath)); err != nil { return err } @@ -62,7 +53,7 @@ func (ro *RestoreOpts) downloadBackupData(localPath string) error { return nil } -func (ro *RestoreOpts) loadTidbClusterData(restorePath string) error { +func (ro *Options) loadTidbClusterData(restorePath string) error { if exist := util.IsDirExist(restorePath); !exist { return fmt.Errorf("dir %s does not exist or is not a dir", restorePath) } @@ -81,10 +72,6 @@ func (ro *RestoreOpts) loadTidbClusterData(restorePath string) error { return nil } -func (ro *RestoreOpts) getDSN(db string) string { - return fmt.Sprintf("%s:%s@(%s:%d)/%s?charset=utf8", ro.User, ro.Password, ro.Host, ro.Port, db) -} - // unarchiveBackupData unarchive backup data to dest dir func unarchiveBackupData(backupFile, destDir string) (string, error) { var unarchiveBackupPath string diff --git a/cmd/backup-manager/app/restore/manager.go b/cmd/backup-manager/app/restore/manager.go index ec034badcc..56b06b66f5 100644 --- a/cmd/backup-manager/app/restore/manager.go +++ b/cmd/backup-manager/app/restore/manager.go @@ -14,16 +14,20 @@ package restore import ( + "database/sql" "fmt" "time" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/klog" - + "github.com/pingcap/tidb-operator/cmd/backup-manager/app/constants" + "github.com/pingcap/tidb-operator/cmd/backup-manager/app/util" "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" + bkconstants "github.com/pingcap/tidb-operator/pkg/backup/constants" listers "github.com/pingcap/tidb-operator/pkg/client/listers/pingcap/v1alpha1" "github.com/pingcap/tidb-operator/pkg/controller" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/klog" ) type Manager struct { @@ -44,11 +48,29 @@ func NewManager( } } +func (rm *Manager) setOptions(restore *v1alpha1.Restore) { + rm.Options.Host = restore.Spec.To.Host + + if restore.Spec.To.Port != 0 { + rm.Options.Port = restore.Spec.To.Port + } else { + rm.Options.Port = bkconstants.DefaultTidbPort + } + + if restore.Spec.To.User != "" { + rm.Options.User = restore.Spec.To.User + } else { + rm.Options.User = bkconstants.DefaultTidbUser + } + + rm.Options.Password = util.GetOptionValueFromEnv(bkconstants.TidbPasswordKey, bkconstants.BackupManagerEnvVarPrefix) +} + // ProcessRestore used to process the restore logic func (rm *Manager) ProcessRestore() error { - restore, err := rm.restoreLister.Restores(rm.Namespace).Get(rm.RestoreName) + restore, err := rm.restoreLister.Restores(rm.Namespace).Get(rm.ResourceName) if err != nil { - klog.Errorf("can't find cluster %s restore %s CRD object, err: %v", rm, rm.RestoreName, err) + klog.Errorf("can't find cluster %s restore %s CRD object, err: %v", rm, rm.ResourceName, err) return rm.StatusUpdater.Update(restore, &v1alpha1.RestoreCondition{ Type: v1alpha1.RestoreFailed, Status: corev1.ConditionTrue, @@ -60,10 +82,33 @@ func (rm *Manager) ProcessRestore() error { return fmt.Errorf("no br config in %s", rm) } - return rm.performRestore(restore.DeepCopy()) + rm.setOptions(restore) + + var db *sql.DB + err = wait.PollImmediate(constants.PollInterval, constants.CheckTimeout, func() (done bool, err error) { + db, err = util.OpenDB(rm.GetDSN()) + if err != nil { + klog.Warningf("can't connect to tidb cluster %s, err: %s", rm, err) + return false, nil + } + return true, nil + }) + + if err != nil { + klog.Errorf("cluster %s connect failed, err: %s", rm, err) + return rm.StatusUpdater.Update(restore, &v1alpha1.RestoreCondition{ + Type: v1alpha1.RestoreFailed, + Status: corev1.ConditionTrue, + Reason: "ConnectTidbFailed", + Message: err.Error(), + }) + } + + defer db.Close() + return rm.performRestore(restore.DeepCopy(), db) } -func (rm *Manager) performRestore(restore *v1alpha1.Restore) error { +func (rm *Manager) performRestore(restore *v1alpha1.Restore, db *sql.DB) error { started := time.Now() err := rm.StatusUpdater.Update(restore, &v1alpha1.RestoreCondition{ @@ -74,15 +119,94 @@ func (rm *Manager) performRestore(restore *v1alpha1.Restore) error { return err } - if err := rm.restoreData(restore); err != nil { - klog.Errorf("restore cluster %s from %s failed, err: %s", rm, restore.Spec.Type, err) + oldTikvGCTime, err := rm.GetTikvGCLifeTime(db) + if err != nil { + klog.Errorf("cluster %s get %s failed, err: %s", rm, constants.TikvGCVariable, err) return rm.StatusUpdater.Update(restore, &v1alpha1.RestoreCondition{ Type: v1alpha1.RestoreFailed, Status: corev1.ConditionTrue, - Reason: "RestoreDataFromRemoteFailed", + Reason: "GetTikvGCLifeTimeFailed", + Message: err.Error(), + }) + } + klog.Infof("cluster %s %s is %s", rm, constants.TikvGCVariable, oldTikvGCTime) + + oldTikvGCTimeDuration, err := time.ParseDuration(oldTikvGCTime) + if err != nil { + klog.Errorf("cluster %s parse old %s failed, err: %s", rm, constants.TikvGCVariable, err) + return rm.StatusUpdater.Update(restore, &v1alpha1.RestoreCondition{ + Type: v1alpha1.RestoreFailed, + Status: corev1.ConditionTrue, + Reason: "ParseOldTikvGCLifeTimeFailed", Message: err.Error(), }) } + + var tikvGCTimeDuration time.Duration + var tikvGCLifeTime string + if restore.Spec.TikvGCLifeTime != nil { + tikvGCLifeTime = *restore.Spec.TikvGCLifeTime + tikvGCTimeDuration, err = time.ParseDuration(tikvGCLifeTime) + if err != nil { + klog.Errorf("cluster %s parse configured %s failed, err: %s", rm, constants.TikvGCVariable, err) + return rm.StatusUpdater.Update(restore, &v1alpha1.RestoreCondition{ + Type: v1alpha1.RestoreFailed, + Status: corev1.ConditionTrue, + Reason: "ParseConfiguredTikvGCLifeTimeFailed", + Message: err.Error(), + }) + } + } else { + tikvGCLifeTime = constants.TikvGCLifeTime + tikvGCTimeDuration, err = time.ParseDuration(tikvGCLifeTime) + if err != nil { + klog.Errorf("cluster %s parse default %s failed, err: %s", rm, constants.TikvGCVariable, err) + return rm.StatusUpdater.Update(restore, &v1alpha1.RestoreCondition{ + Type: v1alpha1.RestoreFailed, + Status: corev1.ConditionTrue, + Reason: "ParseDefaultTikvGCLifeTimeFailed", + Message: err.Error(), + }) + } + } + + if oldTikvGCTimeDuration < tikvGCTimeDuration { + err = rm.SetTikvGCLifeTime(db, tikvGCLifeTime) + if err != nil { + klog.Errorf("cluster %s set tikv GC life time to %s failed, err: %s", rm, tikvGCLifeTime, err) + return rm.StatusUpdater.Update(restore, &v1alpha1.RestoreCondition{ + Type: v1alpha1.RestoreFailed, + Status: corev1.ConditionTrue, + Reason: "SetTikvGCLifeTimeFailed", + Message: err.Error(), + }) + } + klog.Infof("set cluster %s %s to %s success", rm, constants.TikvGCVariable, tikvGCLifeTime) + } + + restoreErr := rm.restoreData(restore) + if oldTikvGCTimeDuration < tikvGCTimeDuration { + err = rm.SetTikvGCLifeTime(db, oldTikvGCTime) + if err != nil { + klog.Errorf("cluster %s reset tikv GC life time to %s failed, err: %s", rm, oldTikvGCTime, err) + return rm.StatusUpdater.Update(restore, &v1alpha1.RestoreCondition{ + Type: v1alpha1.RestoreFailed, + Status: corev1.ConditionTrue, + Reason: "ResetTikvGCLifeTimeFailed", + Message: err.Error(), + }) + } + klog.Infof("reset cluster %s %s to %s success", rm, constants.TikvGCVariable, oldTikvGCTime) + } + if restoreErr != nil { + klog.Errorf("restore cluster %s from %s failed, err: %s", rm, restore.Spec.Type, restoreErr) + return rm.StatusUpdater.Update(restore, &v1alpha1.RestoreCondition{ + Type: v1alpha1.RestoreFailed, + Status: corev1.ConditionTrue, + Reason: "RestoreDataFromRemoteFailed", + Message: restoreErr.Error(), + }) + } klog.Infof("restore cluster %s from %s succeed", rm, restore.Spec.Type) finish := time.Now() diff --git a/cmd/backup-manager/app/restore/restore.go b/cmd/backup-manager/app/restore/restore.go index fac9d5e4f7..347bebcb43 100644 --- a/cmd/backup-manager/app/restore/restore.go +++ b/cmd/backup-manager/app/restore/restore.go @@ -17,19 +17,13 @@ import ( "fmt" "os/exec" - "k8s.io/klog" - "github.com/pingcap/tidb-operator/cmd/backup-manager/app/util" "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" + "k8s.io/klog" ) type Options struct { - Namespace string - RestoreName string -} - -func (ro *Options) String() string { - return fmt.Sprintf("%s/%s", ro.Namespace, ro.RestoreName) + util.GenericOptions } func (ro *Options) restoreData(restore *v1alpha1.Restore) error { diff --git a/cmd/backup-manager/app/util/backup.go b/cmd/backup-manager/app/util/generic.go similarity index 67% rename from cmd/backup-manager/app/util/backup.go rename to cmd/backup-manager/app/util/generic.go index 8874f61c70..04bebc5b3b 100644 --- a/cmd/backup-manager/app/util/backup.go +++ b/cmd/backup-manager/app/util/generic.go @@ -20,25 +20,26 @@ import ( "github.com/pingcap/tidb-operator/cmd/backup-manager/app/constants" ) -// GenericBackupOptions contains the generic input arguments to the backup command -type GenericBackupOptions struct { - Namespace string - BackupName string - Host string - Port int32 - Password string - User string +// GenericOptions contains the generic input arguments to the backup/restore command +type GenericOptions struct { + Namespace string + // ResourceName can be the name of a backup or restore resource + ResourceName string + Host string + Port int32 + Password string + User string } -func (bo *GenericBackupOptions) String() string { - return fmt.Sprintf("%s/%s", bo.Namespace, bo.BackupName) +func (bo *GenericOptions) String() string { + return fmt.Sprintf("%s/%s", bo.Namespace, bo.ResourceName) } -func (bo *GenericBackupOptions) GetDSN(db string) string { - return fmt.Sprintf("%s:%s@(%s:%d)/%s?charset=utf8", bo.User, bo.Password, bo.Host, bo.Port, db) +func (bo *GenericOptions) GetDSN() string { + return fmt.Sprintf("%s:%s@(%s:%d)/%s?charset=utf8", bo.User, bo.Password, bo.Host, bo.Port, constants.TidbMetaDB) } -func (bo *GenericBackupOptions) GetTikvGCLifeTime(db *sql.DB) (string, error) { +func (bo *GenericOptions) GetTikvGCLifeTime(db *sql.DB) (string, error) { var tikvGCTime string sql := fmt.Sprintf("select variable_value from %s where variable_name= ?", constants.TidbMetaTable) row := db.QueryRow(sql, constants.TikvGCVariable) @@ -49,7 +50,7 @@ func (bo *GenericBackupOptions) GetTikvGCLifeTime(db *sql.DB) (string, error) { return tikvGCTime, nil } -func (bo *GenericBackupOptions) SetTikvGCLifeTime(db *sql.DB, gcTime string) error { +func (bo *GenericOptions) SetTikvGCLifeTime(db *sql.DB, gcTime string) error { sql := fmt.Sprintf("update %s set variable_value = ? where variable_name = ?", constants.TidbMetaTable) _, err := db.Exec(sql, gcTime, constants.TikvGCVariable) if err != nil { diff --git a/cmd/backup-manager/app/util/util.go b/cmd/backup-manager/app/util/util.go index e7d436c14c..093b09e8b3 100644 --- a/cmd/backup-manager/app/util/util.go +++ b/cmd/backup-manager/app/util/util.go @@ -98,17 +98,10 @@ func NormalizeBucketURI(bucket string) string { return strings.Replace(bucket, "://", ":", 1) } -// SetFlagsFromEnv set the environment variable. Will override default values, but be overridden by command line parameters. -func SetFlagsFromEnv(flags *pflag.FlagSet, prefix string) error { - flags.VisitAll(func(f *pflag.Flag) { - envVar := prefix + "_" + strings.Replace(strings.ToUpper(f.Name), "-", "_", -1) - value := os.Getenv(envVar) - if value != "" { - flags.Set(f.Name, value) - } - }) - - return nil +// GetOptionValueFromEnv get option's value from environment variable. If unset, return empty string. +func GetOptionValueFromEnv(option, envPrefix string) string { + envVar := envPrefix + "_" + strings.Replace(strings.ToUpper(option), "-", "_", -1) + return os.Getenv(envVar) } // ConstructBRGlobalOptionsForBackup constructs BR global options for backup and also return the remote path. diff --git a/manifests/crd.yaml b/manifests/crd.yaml index 35573ac50e..844449d3e6 100644 --- a/manifests/crd.yaml +++ b/manifests/crd.yaml @@ -7723,6 +7723,12 @@ spec: storageSize: description: StorageSize is the request storage size for backup job type: string + tikvGCLifeTime: + description: TikvGCLifeTime is to specify the safe gc life time for + restore. The time limit during which data is retained for each GC, + in the format of Go Duration. When a GC happens, the current time + minus this value is the safe point. + type: string to: description: TiDBAccessConfig defines the configuration for access tidb cluster diff --git a/pkg/apis/pingcap/v1alpha1/openapi_generated.go b/pkg/apis/pingcap/v1alpha1/openapi_generated.go index 73e4b9a0ba..a0d4a41c2b 100644 --- a/pkg/apis/pingcap/v1alpha1/openapi_generated.go +++ b/pkg/apis/pingcap/v1alpha1/openapi_generated.go @@ -2779,6 +2779,13 @@ func schema_pkg_apis_pingcap_v1alpha1_RestoreSpec(ref common.ReferenceCallback) Format: "", }, }, + "tikvGCLifeTime": { + SchemaProps: spec.SchemaProps{ + Description: "TikvGCLifeTime is to specify the safe gc life time for restore. The time limit during which data is retained for each GC, in the format of Go Duration. When a GC happens, the current time minus this value is the safe point.", + Type: []string{"string"}, + Format: "", + }, + }, "s3": { SchemaProps: spec.SchemaProps{ Ref: ref("github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.S3StorageProvider"), diff --git a/pkg/apis/pingcap/v1alpha1/types.go b/pkg/apis/pingcap/v1alpha1/types.go index 951df10715..1a8ffbe3a1 100644 --- a/pkg/apis/pingcap/v1alpha1/types.go +++ b/pkg/apis/pingcap/v1alpha1/types.go @@ -985,6 +985,10 @@ type RestoreSpec struct { To TiDBAccessConfig `json:"to,omitempty"` // Type is the backup type for tidb cluster. Type BackupType `json:"backupType,omitempty"` + // TikvGCLifeTime is to specify the safe gc life time for restore. + // The time limit during which data is retained for each GC, in the format of Go Duration. + // When a GC happens, the current time minus this value is the safe point. + TikvGCLifeTime *string `json:"tikvGCLifeTime,omitempty"` // StorageProvider configures where and how backups should be stored. StorageProvider `json:",inline"` // The storageClassName of the persistent volume for Restore data storage. diff --git a/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go index 8e5f22fdfe..262eafc18c 100644 --- a/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go @@ -1913,6 +1913,11 @@ func (in *RestoreList) DeepCopyObject() runtime.Object { func (in *RestoreSpec) DeepCopyInto(out *RestoreSpec) { *out = *in out.To = in.To + if in.TikvGCLifeTime != nil { + in, out := &in.TikvGCLifeTime, &out.TikvGCLifeTime + *out = new(string) + **out = **in + } in.StorageProvider.DeepCopyInto(&out.StorageProvider) if in.StorageClassName != nil { in, out := &in.StorageClassName, &out.StorageClassName diff --git a/pkg/backup/backup/backup_manager.go b/pkg/backup/backup/backup_manager.go index 3971c63269..3922099e3b 100644 --- a/pkg/backup/backup/backup_manager.go +++ b/pkg/backup/backup/backup_manager.go @@ -188,11 +188,8 @@ func (bm *backupManager) makeExportJob(backup *v1alpha1.Backup) (*batchv1.Job, s args := []string{ "export", fmt.Sprintf("--namespace=%s", ns), - fmt.Sprintf("--host=%s", backup.Spec.From.Host), - fmt.Sprintf("--port=%d", backup.Spec.From.Port), - fmt.Sprintf("--user=%s", backup.Spec.From.User), - fmt.Sprintf("--bucket=%s", bucketName), fmt.Sprintf("--backupName=%s", name), + fmt.Sprintf("--bucket=%s", bucketName), fmt.Sprintf("--storageType=%s", backuputil.GetStorageType(backup.Spec.StorageProvider)), } @@ -271,9 +268,6 @@ func (bm *backupManager) makeBackupJob(backup *v1alpha1.Backup) (*batchv1.Job, s "backup", fmt.Sprintf("--namespace=%s", ns), fmt.Sprintf("--backupName=%s", name), - fmt.Sprintf("--host=%s", backup.Spec.From.Host), - fmt.Sprintf("--port=%d", backup.Spec.From.Port), - fmt.Sprintf("--user=%s", backup.Spec.From.User), } backupLabel := label.NewBackup().Instance(backup.GetInstanceName()).BackupJob().Backup(name) diff --git a/pkg/backup/restore/restore_manager.go b/pkg/backup/restore/restore_manager.go index 619d719e0f..1dbd11b94f 100644 --- a/pkg/backup/restore/restore_manager.go +++ b/pkg/backup/restore/restore_manager.go @@ -174,9 +174,6 @@ func (rm *restoreManager) makeImportJob(restore *v1alpha1.Restore) (*batchv1.Job "import", fmt.Sprintf("--namespace=%s", ns), fmt.Sprintf("--restoreName=%s", name), - fmt.Sprintf("--host=%s", restore.Spec.To.Host), - fmt.Sprintf("--port=%d", restore.Spec.To.Port), - fmt.Sprintf("--user=%s", restore.Spec.To.User), fmt.Sprintf("--backupPath=%s", backupPath), } @@ -238,11 +235,17 @@ func (rm *restoreManager) makeRestoreJob(restore *v1alpha1.Restore) (*batchv1.Jo ns := restore.GetNamespace() name := restore.GetName() - envVars, reason, err := backuputil.GenerateStorageCertEnv(ns, restore.Spec.StorageProvider, rm.secretLister) + envVars, reason, err := backuputil.GenerateTidbPasswordEnv(ns, name, restore.Spec.To.SecretName, rm.secretLister) + if err != nil { + return nil, reason, err + } + + storageEnv, reason, err := backuputil.GenerateStorageCertEnv(ns, restore.Spec.StorageProvider, rm.secretLister) if err != nil { return nil, reason, fmt.Errorf("restore %s/%s, %v", ns, name, err) } + envVars = append(envVars, storageEnv...) args := []string{ "restore", fmt.Sprintf("--namespace=%s", ns), diff --git a/pkg/backup/util/util.go b/pkg/backup/util/util.go index b22ebdefad..d62307b7bb 100644 --- a/pkg/backup/util/util.go +++ b/pkg/backup/util/util.go @@ -287,13 +287,14 @@ func GetBackupDataPath(provider v1alpha1.StorageProvider) (string, string, error func ValidateBackup(backup *v1alpha1.Backup) error { ns := backup.Namespace name := backup.Name + + if backup.Spec.From.Host == "" { + return fmt.Errorf("missing cluster config in spec of %s/%s", ns, name) + } + if backup.Spec.From.SecretName == "" { + return fmt.Errorf("missing tidbSecretName config in spec of %s/%s", ns, name) + } if backup.Spec.BR == nil { - if backup.Spec.From.Host == "" { - return fmt.Errorf("missing cluster config in spec of %s/%s", ns, name) - } - if backup.Spec.From.SecretName == "" { - return fmt.Errorf("missing tidbSecretName config in spec of %s/%s", ns, name) - } if backup.Spec.StorageSize == "" { return fmt.Errorf("missing StorageSize config in spec of %s/%s", ns, name) } @@ -338,13 +339,14 @@ func ValidateBackup(backup *v1alpha1.Backup) error { func ValidateRestore(restore *v1alpha1.Restore) error { ns := restore.Namespace name := restore.Name + + if restore.Spec.To.Host == "" { + return fmt.Errorf("missing cluster config in spec of %s/%s", ns, name) + } + if restore.Spec.To.SecretName == "" { + return fmt.Errorf("missing tidbSecretName config in spec of %s/%s", ns, name) + } if restore.Spec.BR == nil { - if restore.Spec.To.Host == "" { - return fmt.Errorf("missing cluster config in spec of %s/%s", ns, name) - } - if restore.Spec.To.SecretName == "" { - return fmt.Errorf("missing tidbSecretName config in spec of %s/%s", ns, name) - } if restore.Spec.StorageSize == "" { return fmt.Errorf("missing StorageSize config in spec of %s/%s", ns, name) }