chore: Refactor benchmark execution code

Refactor the benchmark execution code to allow for different type of
benchmark scenarios. Currently, all benchmarks assume they will be run
against an IMAP server. For the Sync benchmark, this is not the case.

The new benchmark code makes no assumptions about what the benchmark
does internally. All IMAP benchmarks now implement the IMAPBenchmark
interface and the IMAPBenchmarkRunner ensures that all the previous
state requirements are set properly for those benchmarks.

Some other utility code has been moved to a more appropriate location as
well.

benchmarks/gluon_bench/benchmark/bench_dir_config.go

@@ -0,0 +1,45 @@
+package benchmark
+
+import (
+	"os"
+)
+
+// BenchDirConfig controls the directory where the benchmark data will be generated.
+type BenchDirConfig interface {
+	Get() (string, error)
+}
+
+// FixedBenchDirConfig always returns a known path.
+type FixedBenchDirConfig struct {
+	path string
+}
+
+func NewFixedBenchDirConfig(path string) *FixedBenchDirConfig {
+	return &FixedBenchDirConfig{path: path}
+}
+
+func (p *FixedBenchDirConfig) Get() (string, error) {
+	if err := os.MkdirAll(p.path, 0o777); err != nil {
+		return "", err
+	}
+
+	return p.path, nil
+}
+
+// TmpBenchDirConfig returns a temporary path that is generated on first use.
+type TmpBenchDirConfig struct {
+	path string
+}
+
+func (t *TmpBenchDirConfig) Get() (string, error) {
+	if len(t.path) == 0 {
+		path, err := os.MkdirTemp("", "gluon-bench-*")
+		if err != nil {
+			return "", err
+		}
+
+		t.path = path
+	}
+
+	return t.path, nil
+}

benchmarks/gluon_bench/benchmark/benchmark.go

@@ -0,0 +1,21 @@
+package benchmark
+
+import (
+	"context"
+
+	"github.com/ProtonMail/gluon/benchmarks/gluon_bench/reporter"
+)
+
+type Benchmark interface {
+	// Name should return the name of the benchmark. It will also be used to match against cli args.
+	Name() string
+
+	// Setup sets up the benchmark state, this is not timed.
+	Setup(ctx context.Context, benchmarkDir string) error
+
+	// Run performs the actual benchmark, this is timed.
+	Run(ctx context.Context) (*reporter.BenchmarkRun, error)
+
+	// TearDown clear the benchmark state, this is not timed.
+	TearDown(ctx context.Context) error
+}

benchmarks/gluon_bench/flags/general.go

@@ -3,15 +3,9 @@ package flags
 import "flag"
 
 var (
-	StorePath             = flag.String("path", "", "Filepath where to write the database data. If not set a temp folder will be used.")
-	Verbose               = flag.Bool("verbose", false, "Enable verbose logging.")
-	JsonReporter          = flag.String("json-reporter", "", "If specified, will generate a json report with the given filename.")
-	BenchmarkRuns         = flag.Uint("bench-runs", 1, "Number of runs per benchmark.")
-	ReuseState            = flag.Bool("reuse-state", false, "When present, benchmarks will re-use previous run state, rather than a clean slate.")
-	RemoteServer          = flag.String("remote-server", "", "IP address and port of the remote IMAP server to run against. E.g. 127.0.0.1:1143.")
-	Mailbox               = flag.String("mailbox", "INBOX", "If not specified will use INBOX as the mailbox to run benchmarks against.")
-	ParallelClients       = flag.Uint("parallel-clients", 1, "Set the number of clients to be run in parallel during the benchmark.")
-	FillSourceMailbox     = flag.Uint("fill-src-mailbox", 1000, "Number of messages to add to the source inbox before each benchmark, set to 0 to skip.")
-	RandomSeqSetIntervals = flag.Bool("random-seqset-intervals", false, "When set, generate random sequence intervals rather than single numbers.")
-	UIDMode               = flag.Bool("uid-mode", false, "When set, will run benchmarks in UID mode if available.")
+	BenchPath     = flag.String("path", "", "Filepath where to write the database data. If not set a temp folder will be used.")
+	Verbose       = flag.Bool("verbose", false, "Enable verbose logging.")
+	JsonReporter  = flag.String("json-reporter", "", "If specified, will generate a json report with the given filename.")
+	BenchmarkRuns = flag.Uint("bench-runs", 1, "Number of runs per benchmark.")
+	ReuseState    = flag.Bool("reuse-state", false, "When present, benchmarks will re-use previous run state, rather than a clean slate.")
 )

benchmarks/gluon_bench/flags/imap_benchmarks.go

@@ -0,0 +1,12 @@
+package flags
+
+import "flag"
+
+var (
+	RemoteServer          = flag.String("remote-server", "", "IP address and port of the remote IMAP server to run against. E.g. 127.0.0.1:1143.")
+	Mailbox               = flag.String("mailbox", "INBOX", "If not specified will use INBOX as the mailbox to run benchmarks against.")
+	FillSourceMailbox     = flag.Uint("fill-src-mailbox", 1000, "Number of messages to add to the source inbox before each benchmark, set to 0 to skip.")
+	RandomSeqSetIntervals = flag.Bool("random-seqset-intervals", false, "When set, generate random sequence intervals rather than single numbers.")
+	UIDMode               = flag.Bool("uid-mode", false, "When set, will run benchmarks in UID mode if available.")
+	ParallelClients       = flag.Uint("parallel-clients", 1, "Set the number of clients to be run in parallel during the benchmark.")
+)

benchmarks/gluon_bench/utils/build_mailbox.go → benchmarks/gluon_bench/imap_benchmarks/build_mailbox.go

@@ -1,15 +1,17 @@
-package utils
+package imap_benchmarks
 
 import (
 	"time"
 
+	"github.com/ProtonMail/gluon/benchmarks/gluon_bench/utils"
+
 	"github.com/emersion/go-imap/client"
 	"golang.org/x/exp/rand"
 )
 
 // BuildMailbox creates a mailbox of name `mailbox` and fills it up with `messageCount` random messages.
 func BuildMailbox(cl *client.Client, mailbox string, messageCount int) error {
-	messages := []string{MessageAfterNoonMeeting, MessageMultiPartMixed, MessageEmbedded}
+	messages := []string{utils.MessageAfterNoonMeeting, utils.MessageMultiPartMixed, utils.MessageEmbedded}
 	messagesLen := len(messages)
 
 	for i := 0; i < messageCount; i++ {

benchmarks/gluon_bench/utils/client.go → benchmarks/gluon_bench/imap_benchmarks/client.go

@@ -1,4 +1,4 @@
-package utils
+package imap_benchmarks
 
 import (
 	"bufio"
@@ -10,11 +10,10 @@ import (
 	"sync"
 	"time"
 
-	"github.com/bradenaw/juniper/xslices"
-
 	"github.com/ProtonMail/gluon/benchmarks/gluon_bench/flags"
+	"github.com/ProtonMail/gluon/benchmarks/gluon_bench/utils"
+	"github.com/bradenaw/juniper/xslices"
 	"github.com/emersion/go-imap"
-
 	"github.com/emersion/go-imap/client"
 )
 
@@ -24,7 +23,7 @@ func NewClient(addr string) (*client.Client, error) {
 		return nil, err
 	}
 
-	if err := client.Login(UserName, UserPassword); err != nil {
+	if err := client.Login(utils.UserName, utils.UserPassword); err != nil {
 		return nil, err
 	}
 
@@ -221,3 +220,13 @@ func RunParallelClientsWithMailboxes(addr net.Addr, mailboxes []MailboxInfo, fn
 
 	wg.Wait()
 }
+
+func FillBenchmarkSourceMailbox(cl *client.Client) error {
+	if *flags.FillSourceMailbox != 0 {
+		if err := BuildMailbox(cl, *flags.Mailbox, int(*flags.FillSourceMailbox)); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

benchmarks/gluon_bench/benchmarks/copy.go → benchmarks/gluon_bench/imap_benchmarks/copy.go

@@ -1,13 +1,13 @@
-package benchmarks
+package imap_benchmarks
 
 import (
 	"context"
 	"flag"
 	"fmt"
 	"net"
 
+	"github.com/ProtonMail/gluon/benchmarks/gluon_bench/benchmark"
 	"github.com/ProtonMail/gluon/benchmarks/gluon_bench/flags"
-	"github.com/ProtonMail/gluon/benchmarks/gluon_bench/utils"
 	"github.com/emersion/go-imap"
 	"github.com/emersion/go-imap/client"
 	"github.com/google/uuid"
@@ -24,25 +24,25 @@ type Copy struct {
 	dstMailbox string
 }
 
-func NewCopy() *Copy {
-	return &Copy{
+func NewCopy() benchmark.Benchmark {
+	return NewIMAPBenchmarkRunner(&Copy{
 		dstMailbox: uuid.NewString(),
-	}
+	})
 }
 
 func (*Copy) Name() string {
 	return "copy"
 }
 
 func (c *Copy) Setup(ctx context.Context, addr net.Addr) error {
-	cl, err := utils.NewClient(addr.String())
+	cl, err := NewClient(addr.String())
 	if err != nil {
 		return err
 	}
 
-	defer utils.CloseClient(cl)
+	defer CloseClient(cl)
 
-	if err := utils.FillBenchmarkSourceMailbox(cl); err != nil {
+	if err := FillBenchmarkSourceMailbox(cl); err != nil {
 		return err
 	}
 
@@ -89,12 +89,12 @@ func (c *Copy) Setup(ctx context.Context, addr net.Addr) error {
 }
 
 func (c *Copy) TearDown(ctx context.Context, addr net.Addr) error {
-	cl, err := utils.NewClient(addr.String())
+	cl, err := NewClient(addr.String())
 	if err != nil {
 		return err
 	}
 
-	defer utils.CloseClient(cl)
+	defer CloseClient(cl)
 
 	if err := cl.Delete(c.dstMailbox); err != nil {
 		return err
@@ -104,7 +104,7 @@ func (c *Copy) TearDown(ctx context.Context, addr net.Addr) error {
 }
 
 func (c *Copy) Run(ctx context.Context, addr net.Addr) error {
-	utils.RunParallelClients(addr, true, func(cl *client.Client, index uint) {
+	RunParallelClients(addr, true, func(cl *client.Client, index uint) {
 		var copyFn func(*client.Client, *imap.SeqSet, string) error
 		if *flags.UIDMode {
 			copyFn = func(cl *client.Client, set *imap.SeqSet, mailbox string) error {

benchmarks/gluon_bench/benchmarks/expunge.go → benchmarks/gluon_bench/imap_benchmarks/expunge.go

@@ -1,18 +1,17 @@
-package benchmarks
+package imap_benchmarks
 
 import (
 	"context"
 	"flag"
 	"fmt"
 	"net"
 
-	"github.com/bradenaw/juniper/xslices"
-	"github.com/google/uuid"
-
+	"github.com/ProtonMail/gluon/benchmarks/gluon_bench/benchmark"
 	"github.com/ProtonMail/gluon/benchmarks/gluon_bench/flags"
-	"github.com/ProtonMail/gluon/benchmarks/gluon_bench/utils"
+	"github.com/bradenaw/juniper/xslices"
 	"github.com/emersion/go-imap"
 	"github.com/emersion/go-imap/client"
+	"github.com/google/uuid"
 )
 
 var (
@@ -27,24 +26,24 @@ type Expunge struct {
 	mailboxes []string
 }
 
-func NewExpunge() *Expunge {
-	return &Expunge{}
+func NewExpunge() benchmark.Benchmark {
+	return NewIMAPBenchmarkRunner(&Expunge{})
 }
 
 func (*Expunge) Name() string {
 	return "expunge"
 }
 
 func (e *Expunge) Setup(ctx context.Context, addr net.Addr) error {
-	cl, err := utils.NewClient(addr.String())
+	cl, err := NewClient(addr.String())
 	if err != nil {
 		return err
 	}
 
-	defer utils.CloseClient(cl)
+	defer CloseClient(cl)
 
 	if *expungeSameMBoxFlag {
-		if err := utils.FillBenchmarkSourceMailbox(cl); err != nil {
+		if err := FillBenchmarkSourceMailbox(cl); err != nil {
 			return err
 		}
 
@@ -84,7 +83,7 @@ func (e *Expunge) Setup(ctx context.Context, addr net.Addr) error {
 				return err
 			}
 
-			if err := utils.BuildMailbox(cl, v, int(*flags.FillSourceMailbox)); err != nil {
+			if err := BuildMailbox(cl, v, int(*flags.FillSourceMailbox)); err != nil {
 				return err
 			}
 		}
@@ -108,12 +107,12 @@ func (e *Expunge) Setup(ctx context.Context, addr net.Addr) error {
 }
 
 func (e *Expunge) TearDown(ctx context.Context, addr net.Addr) error {
-	cl, err := utils.NewClient(addr.String())
+	cl, err := NewClient(addr.String())
 	if err != nil {
 		return err
 	}
 
-	defer utils.CloseClient(cl)
+	defer CloseClient(cl)
 
 	if !*expungeSameMBoxFlag {
 		for _, v := range e.mailboxes {
@@ -127,22 +126,22 @@ func (e *Expunge) TearDown(ctx context.Context, addr net.Addr) error {
 }
 
 func (e *Expunge) Run(ctx context.Context, addr net.Addr) error {
-	mboxInfo := xslices.Map(e.mailboxes, func(m string) utils.MailboxInfo {
-		return utils.MailboxInfo{Name: m, ReadOnly: false}
+	mboxInfo := xslices.Map(e.mailboxes, func(m string) MailboxInfo {
+		return MailboxInfo{Name: m, ReadOnly: false}
 	})
 
-	utils.RunParallelClientsWithMailboxes(addr, mboxInfo, func(cl *client.Client, index uint) {
+	RunParallelClientsWithMailboxes(addr, mboxInfo, func(cl *client.Client, index uint) {
 		var expungeFn func(*client.Client, *imap.SeqSet) error
 		if *flags.UIDMode {
 			expungeFn = func(cl *client.Client, set *imap.SeqSet) error {
-				if err := utils.UIDStore(cl, set, "+FLAGS", true, imap.DeletedFlag); err != nil {
+				if err := UIDStore(cl, set, "+FLAGS", true, imap.DeletedFlag); err != nil {
 					return err
 				}
 				return cl.Expunge(nil)
 			}
 		} else {
 			expungeFn = func(cl *client.Client, set *imap.SeqSet) error {
-				if err := utils.Store(cl, set, "+FLAGS", true, imap.DeletedFlag); err != nil {
+				if err := Store(cl, set, "+FLAGS", true, imap.DeletedFlag); err != nil {
 					return err
 				}
 				return cl.Expunge(nil)