Add retry on cloud functions error when pulling source from GCS

google/common_operation.go

@@ -9,6 +9,15 @@ import (
 	cloudresourcemanager "google.golang.org/api/cloudresourcemanager/v1"
 )
 
+// Wraps Op.Error in an implementation of built-in Error
+type CommonOpError struct {
+	*cloudresourcemanager.Status
+}
+
+func (e *CommonOpError) Error() string {
+	return fmt.Sprintf("Error code %v, message: %s", e.Code, e.Message)
+}
+
 type Waiter interface {
 	// State returns the current status of the operation.
 	State() string
@@ -56,7 +65,7 @@ func (w *CommonOperationWaiter) State() string {
 
 func (w *CommonOperationWaiter) Error() error {
 	if w != nil && w.Op.Error != nil {
-		return fmt.Errorf("Error code %v, message: %s", w.Op.Error.Code, w.Op.Error.Message)
+		return &CommonOpError{w.Op.Error}
 	}
 	return nil
 }

google/error_retry_predicates.go

@@ -257,3 +257,12 @@ func isPeeringOperationInProgress(err error) (bool, string) {
 	}
 	return false, ""
 }
+
+func isCloudFunctionsSourceCodeError(err error) (bool, string) {
+	if operr, ok := err.(*CommonOpError); ok {
+		if operr.Code == 3 && operr.Message == "Failed to retrieve function source code" {
+			return true, fmt.Sprintf("Retry on Function failing to pull code from GCS")
+		}
+	}
+	return false, ""
+}