From d07d8ad014b3c1e60f03d757dcd126db396644b1 Mon Sep 17 00:00:00 2001 From: Jakub Date: Tue, 11 Oct 2022 17:22:12 +0200 Subject: [PATCH] fix(GODT-1761): imaptest coverage benchmark. --- benchmarks/imaptest/README.md | 50 ++++--- benchmarks/imaptest/benchmark.yml | 94 +++++++++++++ benchmarks/imaptest/default.sh | 10 -- benchmarks/imaptest/full.sh | 25 ---- benchmarks/imaptest/main_test.go | 226 ++++++++++++++++++++++++++++++ demo/demo.go | 34 +++-- internal/state/responders.go | 12 +- 7 files changed, 386 insertions(+), 65 deletions(-) create mode 100644 benchmarks/imaptest/benchmark.yml delete mode 100755 benchmarks/imaptest/default.sh delete mode 100755 benchmarks/imaptest/full.sh create mode 100644 benchmarks/imaptest/main_test.go diff --git a/benchmarks/imaptest/README.md b/benchmarks/imaptest/README.md index 37cb98e5..3c9f6830 100644 --- a/benchmarks/imaptest/README.md +++ b/benchmarks/imaptest/README.md @@ -3,39 +3,51 @@ This "benchmark" uses [Dovecot's ImapTest tool](https://imapwiki.org/ImapTest) to profile the performance of Gluon. The information present here can also be used to verify the compliance of Gluon with the IMAP protocol. -## Installation +Build gluon demo in project root folder: + +```bash +go build -o gluon-demo ./demo/demo.go +``` + +## Installation of ImapTest Follow the instructions outlined in [the tools' installation page](https://imapwiki.org/ImapTest/Installation) to build the binary. The test mailbox is already present in this folder. -## Running ImapTest +## Simple ImapTest run -The bare minimum required for running ImapTest is a username and a password: +Assuming gluon demo is running, the bare minimum required for running ImapTest is a username and a password: ```bash imaptest host=127.0.0.1 port=1143 user=user1@example.com pass=password1 ``` -For convenience, we have provided a few test scripts to quickly test Gluon with the demo binary present in this -repository. +## Advance testing + +The multiple scenario coverage can be run by - * **default.sh**: Runs the default ImapTest benchmarks. - * **full.sh**: Runs every available ImapTest operation. +``` +go test +``` -Both of these scripts can be customized with the following environment variables: +The test cases are defined in `benchmark.yml`. Each case defines a number of +clients and users to be used by ImapTest. One case can have multiple +settings defined by name. The options are specified in `settings` section. The +settings reflects ImapTest options as described in +[here](https://imapwiki.org/ImapTest/Running). +The ImapTest states are described in +[here](https://imapwiki.org/ImapTest/States). -* **IMAPTEST_BIN**: Location of the ImapTest binary. -* **SECS**: Number of seconds for which to run ImapTest. -* **CLIENTS**: Number of concurrent clients used by ImapTest. +We don't use for now the ImapTest scriptable scenarios but it is +possible by defining the new test settings in file `./benchmark.yml` and +creating separate definition file like example +[here](https://github.com/dovecot/imaptest/tree/main/src/tests) -Example: -```bash -# Run imaptest for 60s with one conccurent client connection. -IMAPTEST_BIN=/bin/imaptest SECS=60 CLIENTS=1 ./default.sh -``` ## Note about this tool -The execution of this tool is non-deterministic, this means it can't be used to compare profile runs of two different -versions. -It should be only be used to profile and/or stress the codebase in an isolation. \ No newline at end of file +The execution of this tool is non-deterministic, this means it can't be used to +compare profile runs of two different versions. + +It should be only be used to profile and/or stress the codebase in an +isolation. diff --git a/benchmarks/imaptest/benchmark.yml b/benchmarks/imaptest/benchmark.yml new file mode 100644 index 00000000..47cd2805 --- /dev/null +++ b/benchmarks/imaptest/benchmark.yml @@ -0,0 +1,94 @@ + +--- + +cases: + - users: 1 + clients: 1 + settings: + - simple + - users: 10 + clients: 10 + settings: + - simple + + +.OFF.cases: + - users: 1 + clients: 10 + settings: + - full + - users: 1 + clients: 100 + settings: + - simple + - full + - users: 1 + clients: 1000 + settings: + - simple + - full + + - users: 10 + clients: 10 + settings: + - full + - users: 10 + clients: 100 + settings: + - simple + - full + + - users: 100 + clients: 100 + settings: + - simple + - full + - users: 100 + clients: 1000 + settings: + - simple + - full + + - users: 1000 + clients: 1000 + settings: + - simple + - full + - users: 1000 + clients: 10000 + settings: + - simple + - full + +settings: + simple: + mbox: dovecot-crlf + secs: 10 + no_pipelining: true + simple-with-checks: + mbox: dovecot-crlf + secs: 10 + checkpoint: 3 + no_pipelining: true + own_msgs: true + own_flags: true + full: + mbox: dovecot-crlf + no_pipelining: false + secs: 60 + mcreate: 50 + mdelete: 50 + uidf: 50 + search: 30 + noop: 15 + fetch: 50 + login: 100 + logout: 100 + list: 50 + select: 100 + fet2: 100,30 + copy: 30,5 + store: 50 + delete: 100 + expunge: 100 + append: 100,5 diff --git a/benchmarks/imaptest/default.sh b/benchmarks/imaptest/default.sh deleted file mode 100755 index 167067a4..00000000 --- a/benchmarks/imaptest/default.sh +++ /dev/null @@ -1,10 +0,0 @@ -#/bin/bash - -CLIENTS=${CLIENTS:-"200"} -SECS=${SECS:-"60"} -IMAPTEST_BIN=${IMAPTEST_BIN:-imaptest} - -${IMAPTEST_BIN} host=127.0.0.1 port=1143 \ -user=user1@example.com pass=password1 \ -mbox=dovecot-crlf \ -no_pipelining secs=${SECS} clients=${CLIENTS} diff --git a/benchmarks/imaptest/full.sh b/benchmarks/imaptest/full.sh deleted file mode 100755 index 08b9a289..00000000 --- a/benchmarks/imaptest/full.sh +++ /dev/null @@ -1,25 +0,0 @@ -#/bin/bash - -CLIENTS=${CLIENTS:-"200"} -SECS=${SECS:-"60"} -IMAPTEST_BIN=${IMAPTEST_BIN:-imaptest} - -$IMAPTEST_BIN host=127.0.0.1 port=1143 \ -user=user1@example.com pass=password1 mbox=dovecot-crlf \ -no_pipelining secs=${SECS} clients=${CLIENTS} \ -- mcreate=50 \ -mdelete=50 \ -uidf=50 \ -search=30 \ -noop=15 \ -fetch=50 \ -login=100 \ -logout=100 \ -list=50 \ -select=100 \ -fet2=100,30 \ -copy=30,5 \ -store=50 \ -delete=100 \ -expunge=100 \ -append=100,5 diff --git a/benchmarks/imaptest/main_test.go b/benchmarks/imaptest/main_test.go new file mode 100644 index 00000000..40deb36e --- /dev/null +++ b/benchmarks/imaptest/main_test.go @@ -0,0 +1,226 @@ +package main + +import ( + "bytes" + "context" + "fmt" + "os" + "strconv" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/sys/execabs" + "gopkg.in/yaml.v3" +) + +const ( + allowParallel = false + doFullIMAPtestLog = false + gluon_log_level = "warn" +) + +func TestIMAPTest(t *testing.T) { + if path, err := execabs.LookPath("imaptest"); err != nil || path == "" { + t.Skip("imaptest is not installed") + } + + r := require.New(t) + + c, err := newConfig("./benchmark.yml") + r.NoError(err) + + scenarios, err := c.generateScenarios() + r.NoError(err) + + for _, scenario := range scenarios { + t.Run(scenario.name, scenario.test) + } +} + +type config struct { + Cases []caseConfig + Settings map[string]settingsConfig +} + +type caseConfig struct { + Users, Clients int + Settings []string + allowFail bool +} + +type settingsConfig map[string]string + +func newConfig(path string) (*config, error) { + rawYAML, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + bm := &config{} + + if err := yaml.Unmarshal(rawYAML, bm); err != nil { + return nil, err + } + + return bm, nil +} + +func (conf *config) generateScenarios() ([]*scenario, error) { + var scenarios []*scenario + + i := 0 + + for _, c := range conf.Cases { + for _, settingName := range c.Settings { + s, ok := conf.Settings[settingName] + if !ok { + return nil, fmt.Errorf("wrong config format: cannot find definition for %q setting", settingName) + } + + sc, err := newScenario(c, settingName, s, 10143+i) + if err != nil { + return nil, err + } + + scenarios = append(scenarios, sc) + i += 1 + } + } + + return scenarios, nil +} + +type scenario struct { + allowFail bool + port string + users int + name string + imaptestParams []string + timeout time.Duration + + t *testing.T + + ctx context.Context +} + +func newScenario(c caseConfig, settingName string, s settingsConfig, port int) (*scenario, error) { + sc := &scenario{ + allowFail: c.allowFail, + port: fmt.Sprintf("%d", port), + users: c.Users, + name: fmt.Sprintf("u%d_c%d_%s", c.Users, c.Clients, settingName), + timeout: time.Duration(time.Second), + } + + if secs, err := strconv.Atoi(s["secs"]); err == nil { + sc.timeout = time.Duration(secs) * 2 * time.Second + } + + // coomon arguments + sc.imaptestParams = []string{ + "host=127.0.0.1", + fmt.Sprintf("port=%d", port), + "user=user%d@example.com", + fmt.Sprintf("users=%d", c.Users), + "pass=pass", + fmt.Sprintf("clients=%d", c.Clients), + } + + // scenario specific arguments + for k, val := range s { + if val == "false" { + continue + } + + if val == "true" { + sc.imaptestParams = append(sc.imaptestParams, k) + continue + } + + sc.imaptestParams = append(sc.imaptestParams, fmt.Sprintf("%s=%s", k, val)) + } + + return sc, nil +} + +func (s *scenario) test(t *testing.T) { + s.t = t + + if allowParallel { + t.Parallel() + } + + var cancel context.CancelFunc + s.ctx, cancel = context.WithTimeout(context.Background(), s.timeout) + + wg := sync.WaitGroup{} + wg.Add(1) + + defer func() { + cancel() + wg.Wait() // to printout log + }() + + go func() { + s.runGluon() + cancel() + wg.Done() + }() + + // wait for gluon demo to setup users + time.Sleep(time.Second) + + s.runIMAPTest() +} + +func (s *scenario) runGluon() { + cmd := execabs.CommandContext(s.ctx, "./gluon-demo") + cmd.Dir = "../.." + cmd.Env = append(cmd.Env, + fmt.Sprintf("GLUON_DIR=%s", s.t.TempDir()), + fmt.Sprintf("GLUON_USER_COUNT=%d", s.users), + fmt.Sprintf("GLUON_HOST=127.0.0.1:%s", s.port), + "GLUON_LOG_LEVEL="+gluon_log_level, + ) + + out := bytes.NewBuffer(nil) + cmd.Stderr = out + cmd.Stdout = out + + err := cmd.Run() + + fmt.Printf("Gluon[%s]:\n%s\nGluonEnd[%s]\n", s.name, out.String(), s.name) + + assert.Error(s.t, err) + assert.Equal(s.t, "signal: killed", err.Error()) +} + +func (s *scenario) runIMAPTest() { + logPath := "" + if doFullIMAPtestLog { + logPath = s.t.TempDir() + "imaptest.log" + s.imaptestParams = append(s.imaptestParams, "output="+logPath) + } + + cmd := execabs.CommandContext(s.ctx, "imaptest", s.imaptestParams...) + + out := bytes.NewBuffer(nil) + cmd.Stderr = out + cmd.Stdout = out + + err := cmd.Run() + + fmt.Printf("IMAPTEST[%s]: %q\n%s\nIMAPTESTEND[%s]\n", s.name, s.imaptestParams, out.String(), s.name) + + assert.NoError(s.t, err) + assert.False(s.t, bytes.Contains(out.Bytes(), []byte("rror"))) + + if doFullIMAPtestLog { + log, err := os.ReadFile(logPath) + assert.NoError(s.t, err) + fmt.Println("LOG", s.name, "\n", string(log), "\nLOG", s.name, "END") + } +} diff --git a/demo/demo.go b/demo/demo.go index 57bad04f..e2a3a150 100644 --- a/demo/demo.go +++ b/demo/demo.go @@ -3,8 +3,10 @@ package main import ( "context" "flag" + "fmt" "net" "os" + "strconv" "time" "github.com/ProtonMail/gluon" @@ -45,25 +47,39 @@ func main() { logrus.SetLevel(level) } - server, err := gluon.New(gluon.WithLogger( - logrus.StandardLogger().WriterLevel(logrus.TraceLevel), - logrus.StandardLogger().WriterLevel(logrus.TraceLevel), - )) + server, err := gluon.New( + gluon.WithLogger( + logrus.StandardLogger().WriterLevel(logrus.TraceLevel), + logrus.StandardLogger().WriterLevel(logrus.TraceLevel), + ), + gluon.WithDataDir(os.Getenv("GLUON_DIR")), + ) if err != nil { logrus.WithError(err).Fatal("Failed to create server") } defer server.Close(ctx) - if err := addUser(ctx, server, []string{"user1@example.com", "alias1@example.com"}, []byte("password1")); err != nil { - logrus.WithError(err).Fatal("Failed to add user") + nUsers := 2 + if envUsers, err := strconv.Atoi(os.Getenv("GLUON_USER_COUNT")); err == nil { + nUsers = envUsers + } + + for i := 1; i <= nUsers; i++ { + if err := addUser(ctx, server, []string{ + fmt.Sprintf("user%d@example.com", i), + fmt.Sprintf("alias%d@example.com", i), + }, []byte("pass")); err != nil { + logrus.WithError(err).Fatal("Failed to add user") + } } - if err := addUser(ctx, server, []string{"user2@example.com", "alias2@example.com"}, []byte("password2")); err != nil { - logrus.WithError(err).Fatal("Failed to add user") + host := "localhost:1143" + if envHost := os.Getenv("GLUON_HOST"); envHost != "" { + host = envHost } - listener, err := net.Listen("tcp", "localhost:1143") + listener, err := net.Listen("tcp", host) if err != nil { logrus.WithError(err).Fatal("Failed to listen") } diff --git a/internal/state/responders.go b/internal/state/responders.go index dd25fa08..f385f671 100644 --- a/internal/state/responders.go +++ b/internal/state/responders.go @@ -144,13 +144,21 @@ func (u *targetedExists) getMessageID() imap.InternalMessageID { func (u *targetedExists) String() string { var originState string + if u.originStateSet { originState = fmt.Sprintf("%v", u.originStateID) } else { originState = "None" } - return fmt.Sprintf("TargetedExists: message = %v remote = %v targetStateID = %v originStateID = %v", u.resp.messageID.InternalID.ShortID(), u.resp.messageID.RemoteID, u.targetStateID, originState) + return fmt.Sprintf( + "TargetedExists: message = %v uid = %v remote = %v targetStateID = %v originStateID = %v", + u.resp.messageID.InternalID.ShortID(), + u.resp.messageUID, + u.resp.messageID.RemoteID, + u.targetStateID, + originState, + ) } // ExistsStateUpdate needs to be a separate update since it has to deal with a Recent flag propagation. If a session @@ -275,7 +283,7 @@ func (u *expunge) getMessageID() imap.InternalMessageID { } func (u *expunge) String() string { - return fmt.Sprintf("Expung: message = %v", + return fmt.Sprintf("Expunge: message = %v", u.messageID.ShortID(), ) }