Improve perf with parallel image deletion

go.mod

@@ -3,6 +3,7 @@ module github.com/graillus/gcrgc
 go 1.15
 
 require (
+	github.com/gammazero/workerpool v1.1.1
 	github.com/google/go-containerregistry v0.2.1
 	github.com/k1LoW/duration v1.1.0
 	github.com/spf13/pflag v1.0.5

go.sum

@@ -133,6 +133,10 @@ github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoD
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/gammazero/deque v0.0.0-20200721202602-07291166fe33 h1:UG4wNrJX9xSKnm/Gck5yTbxnOhpNleuE4MQRdmcGySo=
+github.com/gammazero/deque v0.0.0-20200721202602-07291166fe33/go.mod h1:D90+MBHVc9Sk1lJAbEVgws0eYEurY4mv2TDso3Nxh3w=
+github.com/gammazero/workerpool v1.1.1 h1:MN29GcZtZZAgzTU+Zk54Y+J9XkE54MoXON/NCZvNulo=
+github.com/gammazero/workerpool v1.1.1/go.mod h1:5BN0IJVRjSFAypo9QTJCaWdijjNz9Jjl6VFS1PRjCeg=
 github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=

internal/gcrgc/gcrgc.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/gammazero/workerpool"
 	"github.com/google/go-containerregistry/pkg/authn"
 	"github.com/google/go-containerregistry/pkg/v1/google"
 	"github.com/graillus/gcrgc/internal/docker"
@@ -75,12 +76,13 @@ func createAuthenticator() authn.Authenticator {
 	return auth
 }
 
-type taskList map[string][]docker.Image
+type task struct {
+	repository string
+	image      *docker.Image
+}
 
 // Browse the registry to find out which images have to be deleted
-func getTaskList(gcr docker.Provider, repository []docker.Repository, s *Settings) taskList {
-	tasks := make(taskList)
-
+func getTaskList(gcr docker.Provider, repository []docker.Repository, s *Settings) []task {
 	// By default all images pushed before the current time will be taken into account
 	var date = time.Now()
 	if s.Date != (time.Time{}) {
@@ -96,43 +98,49 @@ func getTaskList(gcr docker.Provider, repository []docker.Repository, s *Setting
 		NewSemVerTagNameFilter(s.ExcludeSemVerTags),
 	}
 
+	tasks := []task{}
+
 	for _, repo := range repository {
 		imgs := gcr.ListImages(repo.Name, date)
 
 		filteredImgs := filterImages(imgs, filters)
 
-		tasks[repo.Name] = filteredImgs
 		fmt.Printf("%d matches for repository [%s]\n", len(filteredImgs), repo.Name)
+
+		for _, i := range filteredImgs {
+			img := i
+			tasks = append(tasks, task{repo.Name, &img})
+		}
 	}
 
 	return tasks
 }
 
 // Delete the images from remote registry
-func doDelete(gcr docker.Provider, tasks taskList, dryRun bool) {
+func doDelete(gcr docker.Provider, tasks []task, dryRun bool) {
 	if len(tasks) == 0 {
 		fmt.Println("Nothing to do.")
 
 		return
 	}
 
 	r := newReport()
-	for k, v := range tasks {
-		if len(v) == 0 {
-			fmt.Printf("No images to clean in repository [%s]\n", k)
-
-			continue
-		}
 
-		fmt.Printf("Cleaning repository [%s] (%d matches)\n", k, len(v))
+	// Create a pool of 8 workers. The number is arbitrary, but it seems that inscreasing the worker count doesn't affect
+	// the performance since the google container registry API has some kind of per-user rate limiting.
+	wp := workerpool.New(8)
 
-		for _, i := range v {
-			fmt.Printf("Deleting %s %s\n", i.Digest, strings.Join(i.Tags, ", "))
-			gcr.DeleteImage(k, &i, dryRun)
+	// Let's submit the tasks to the workers pool
+	for _, t := range tasks {
+		task := t
+		wp.Submit(func() {
+			fmt.Printf("Deleting %s %s\n", task.image.Digest, strings.Join(task.image.Tags, ", "))
+			gcr.DeleteImage(task.repository, task.image, dryRun)
 
-			r.reportImage(i)
-		}
+			r.reportImage(*task.image)
+		})
 	}
+	wp.StopWait()
 
 	fmt.Printf("Done\n\n")
 	fmt.Printf("Deleted images: %d/%d\n", r.TotalDeleted(), r.Total())