retry on recoverable propagation failure

pkg/controller/sync/controller.go

@@ -450,6 +450,15 @@ func (s *KubeFedSyncController) setFederatedStatus(fedResource FederatedResource
 		return util.StatusError
 	}
 
+	// return Error to trigger a retry with back off on recoverable propagation failure
+	if reason == status.AggregateSuccess {
+		for _, value := range collectedStatus.StatusMap {
+			if status.IsRecoverableError(value) {
+				return util.StatusError
+			}
+		}
+	}
+
 	return util.StatusAllOK
 }
 

pkg/controller/sync/status/status.go

@@ -164,6 +164,24 @@ func SetFederatedStatus(fedObject *unstructured.Unstructured, reason AggregateRe
 	return true, nil
 }
 
+// IsRecoverableError returns whether the given PropagationStatus is a possibly recoverable error.
+func IsRecoverableError(status PropagationStatus) bool {
+	switch status {
+	case
+		CreationFailed,
+		UpdateFailed,
+		DeletionFailed,
+		LabelRemovalFailed,
+		RetrievalFailed,
+		CreationTimedOut,
+		UpdateTimedOut,
+		DeletionTimedOut,
+		LabelRemovalTimedOut:
+		return true
+	}
+	return false
+}
+
 // update ensures that the status reflects the given generation, reason
 // and collected status. Returns a boolean indication of whether the
 // status has been changed.