Delete GCS objects in parallel (#2944)

<!-- This change is generated by MagicModules. -->
/cc @sethvargo

go.mod

@@ -7,6 +7,8 @@ require (
 	github.com/blang/semver v3.5.1+incompatible // indirect
 	github.com/davecgh/go-spew v1.1.1
 	github.com/dustinkirkland/golang-petname v0.0.0-20170921220637-d3c2ba80e75e // indirect
+	github.com/gammazero/deque v0.0.0-20180920172122-f6adf94963e4 // indirect
+	github.com/gammazero/workerpool v0.0.0-20181230203049-86a96b5d5d92
 	github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db // indirect
 	github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c // indirect
 	github.com/googleapis/gax-go v2.0.2+incompatible // indirect

go.sum

@@ -32,6 +32,10 @@ github.com/dustinkirkland/golang-petname v0.0.0-20170921220637-d3c2ba80e75e/go.m
 github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/gammazero/deque v0.0.0-20180920172122-f6adf94963e4 h1:R+19WKQClnfMXS60cP5BmMe1wjZ4u0evY2p2Ar0ZTXo=
+github.com/gammazero/deque v0.0.0-20180920172122-f6adf94963e4/go.mod h1:GeIq9qoE43YdGnDXURnmKTnGg15pQz4mYkXSTChbneI=
+github.com/gammazero/workerpool v0.0.0-20181230203049-86a96b5d5d92 h1:EipXK6U05IQ2wtuFRn4k3h0+2lXypzItoXGVyf4r9Io=
+github.com/gammazero/workerpool v0.0.0-20181230203049-86a96b5d5d92/go.mod h1:w9RqFVO2BM3xwWEcAB8Fwp0OviTBBEiRmSBDfbXnd3w=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-test/deep v1.0.1/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=

google/resource_storage_bucket.go

@@ -5,10 +5,12 @@ import (
 	"errors"
 	"fmt"
 	"log"
+	"runtime"
 	"strconv"
 	"strings"
 	"time"
 
+	"github.com/gammazero/workerpool"
 	"github.com/hashicorp/terraform/helper/hashcode"
 	"github.com/hashicorp/terraform/helper/resource"
 	"github.com/hashicorp/terraform/helper/schema"
@@ -516,22 +518,46 @@ func resourceStorageBucketDelete(d *schema.ResourceData, meta interface{}) error
 
 		if len(res.Items) != 0 {
 			if d.Get("force_destroy").(bool) {
-				// purge the bucket...
+				// GCS requires that a bucket be empty (have no objects or object
+				// versions) before it can be deleted.
 				log.Printf("[DEBUG] GCS Bucket attempting to forceDestroy\n\n")
 
+				// Create a workerpool for parallel deletion of resources. In the
+				// future, it would be great to expose Terraform's global parallelism
+				// flag here, but that's currently reserved for core use. Testing
+				// shows that NumCPUs-1 is the most performant on average networks.
+				//
+				// The challenge with making this user-configurable is that the
+				// configuration would reside in the Terraform configuration file,
+				// decreasing its portability. Ideally we'd want this to connect to
+				// Terraform's top-level -parallelism flag, but that's not plumbed nor
+				// is it scheduled to be plumbed to individual providers.
+				wp := workerpool.New(runtime.NumCPU() - 1)
+
 				for _, object := range res.Items {
 					log.Printf("[DEBUG] Found %s", object.Name)
-					if err := config.clientStorage.Objects.Delete(bucket, object.Name).Generation(object.Generation).Do(); err != nil {
-						log.Fatalf("Error trying to delete object: %s %s\n\n", object.Name, err)
-					} else {
-						log.Printf("Object deleted: %s \n\n", object.Name)
-					}
+					object := object
+
+					wp.Submit(func() {
+						log.Printf("[TRACE] Attempting to delete %s", object.Name)
+						if err := config.clientStorage.Objects.Delete(bucket, object.Name).Generation(object.Generation).Do(); err != nil {
+							// We should really return an error here, but it doesn't really
+							// matter since the following step (bucket deletion) will fail
+							// with an error indicating objects are still present, and this
+							// log line will point to that object.
+							log.Printf("[ERR] Failed to delete storage object %s: %s", object.Name, err)
+						} else {
+							log.Printf("[TRACE] Successfully deleted %s", object.Name)
+						}
+					})
 				}
 
+				// Wait for everything to finish.
+				wp.StopWait()
 			} else {
-				delete_err := errors.New("Error trying to delete a bucket containing objects without `force_destroy` set to true")
-				log.Printf("Error! %s : %s\n\n", bucket, delete_err)
-				return delete_err
+				deleteErr := errors.New("Error trying to delete a bucket containing objects without `force_destroy` set to true")
+				log.Printf("Error! %s : %s\n\n", bucket, deleteErr)
+				return deleteErr
 			}
 		} else {
 			break // 0 items, bucket empty