ebs br: check status of global nodes before doing data recovery

pkg/apis/pingcap/v1alpha1/tidbcluster.go

@@ -531,6 +531,17 @@ func (tc *TidbCluster) PDAllMembersReady() bool {
 	return true
 }
 
+// PDAllPeerMembersReady return whether all peer members of PD are ready.
+func (tc *TidbCluster) PDAllPeerMembersReady() bool {
+
+	for _, member := range tc.Status.PD.PeerMembers {
+		if !member.Health {
+			return false
+		}
+	}
+	return true
+}
+
 func (tc *TidbCluster) PDAutoFailovering() bool {
 	if len(tc.Status.PD.FailureMembers) == 0 {
 		return false
@@ -908,6 +919,16 @@ func (tc *TidbCluster) AllTiKVsAreAvailable() bool {
 	return true
 }
 
+func (tc *TidbCluster) AllPeerTiKVsAreAvailable() bool {
+	for _, store := range tc.Status.TiKV.PeerStores {
+		if store.State != TiKVStateUp {
+			return false
+		}
+	}
+
+	return true
+}
+
 func (tc *TidbCluster) PumpIsAvailable() bool {
 	lowerLimit := 1
 	if len(tc.Status.Pump.Members) < lowerLimit {

pkg/backup/restore/restore_manager.go

@@ -131,12 +131,12 @@ func (rm *restoreManager) syncRestoreJob(restore *v1alpha1.Restore) error {
 			}, nil)
 			return err
 		}
-		if !tc.PDAllMembersReady() {
+		if !tc.PDAllMembersReady() || !tc.PDAllPeerMembersReady() {
 			return controller.RequeueErrorf("restore %s/%s: waiting for all PD members are ready in tidbcluster %s/%s", ns, name, tc.Namespace, tc.Name)
 		}
 
 		if v1alpha1.IsRestoreVolumeComplete(restore) && !v1alpha1.IsRestoreTiKVComplete(restore) {
-			if !tc.AllTiKVsAreAvailable() {
+			if !tc.AllTiKVsAreAvailable() || !tc.AllPeerTiKVsAreAvailable() {
 				return controller.RequeueErrorf("restore %s/%s: waiting for all TiKVs are available in tidbcluster %s/%s", ns, name, tc.Namespace, tc.Name)
 			} else {
 				sel, err := label.New().Instance(tc.Name).TiKV().Selector()