From a824303cbc6df14e0de1f2741691190ba5049788 Mon Sep 17 00:00:00 2001 From: Modular Magician Date: Wed, 27 May 2020 19:20:41 +0000 Subject: [PATCH] Add retry on cloud functions error when pulling source from GCS (#3570) Signed-off-by: Modular Magician --- .changelog/3570.txt | 3 ++ google-beta/common_operation.go | 11 ++++++- google-beta/error_retry_predicates.go | 9 ++++++ .../resource_cloudfunctions_function.go | 30 +++++++++++-------- 4 files changed, 40 insertions(+), 13 deletions(-) create mode 100644 .changelog/3570.txt diff --git a/.changelog/3570.txt b/.changelog/3570.txt new file mode 100644 index 0000000000..aa80412746 --- /dev/null +++ b/.changelog/3570.txt @@ -0,0 +1,3 @@ +```release-note:bug +functions: Added retry to `google_cloudfunctions_function` creation when API returns error while pulling source from GCS +``` diff --git a/google-beta/common_operation.go b/google-beta/common_operation.go index 087f8cae13..4cf66af84d 100644 --- a/google-beta/common_operation.go +++ b/google-beta/common_operation.go @@ -9,6 +9,15 @@ import ( cloudresourcemanager "google.golang.org/api/cloudresourcemanager/v1" ) +// Wraps Op.Error in an implementation of built-in Error +type CommonOpError struct { + *cloudresourcemanager.Status +} + +func (e *CommonOpError) Error() string { + return fmt.Sprintf("Error code %v, message: %s", e.Code, e.Message) +} + type Waiter interface { // State returns the current status of the operation. State() string @@ -56,7 +65,7 @@ func (w *CommonOperationWaiter) State() string { func (w *CommonOperationWaiter) Error() error { if w != nil && w.Op.Error != nil { - return fmt.Errorf("Error code %v, message: %s", w.Op.Error.Code, w.Op.Error.Message) + return &CommonOpError{w.Op.Error} } return nil } diff --git a/google-beta/error_retry_predicates.go b/google-beta/error_retry_predicates.go index 967a1a2256..300df90043 100644 --- a/google-beta/error_retry_predicates.go +++ b/google-beta/error_retry_predicates.go @@ -257,3 +257,12 @@ func isPeeringOperationInProgress(err error) (bool, string) { } return false, "" } + +func isCloudFunctionsSourceCodeError(err error) (bool, string) { + if operr, ok := err.(*CommonOpError); ok { + if operr.Code == 3 && operr.Message == "Failed to retrieve function source code" { + return true, fmt.Sprintf("Retry on Function failing to pull code from GCS") + } + } + return false, "" +} diff --git a/google-beta/resource_cloudfunctions_function.go b/google-beta/resource_cloudfunctions_function.go index cc778c31aa..73b4888d6e 100644 --- a/google-beta/resource_cloudfunctions_function.go +++ b/google-beta/resource_cloudfunctions_function.go @@ -401,21 +401,27 @@ func resourceCloudFunctionsCreate(d *schema.ResourceData, meta interface{}) erro } log.Printf("[DEBUG] Creating cloud function: %s", function.Name) - op, err := config.clientCloudFunctions.Projects.Locations.Functions.Create( - cloudFuncId.locationId(), function).Do() - if err != nil { - return err - } - // Name of function should be unique - d.SetId(cloudFuncId.cloudFunctionId()) + // We retry the whole create-and-wait because Cloud Functions + // will sometimes fail a creation operation entirely if it fails to pull + // source code and we need to try the whole creation again. + rerr := retryTimeDuration(func() error { + op, err := config.clientCloudFunctions.Projects.Locations.Functions.Create( + cloudFuncId.locationId(), function).Do() + if err != nil { + return err + } + + // Name of function should be unique + d.SetId(cloudFuncId.cloudFunctionId()) - err = cloudFunctionsOperationWait(config, op, "Creating CloudFunctions Function", - d.Timeout(schema.TimeoutCreate)) - if err != nil { - return err + return cloudFunctionsOperationWait(config, op, "Creating CloudFunctions Function", + d.Timeout(schema.TimeoutCreate)) + }, d.Timeout(schema.TimeoutCreate), isCloudFunctionsSourceCodeError) + if rerr != nil { + return rerr } - + log.Printf("[DEBUG] Finished creating cloud function: %s", function.Name) return resourceCloudFunctionsRead(d, meta) }