feat(GODT-2510): Direct SQLite3

DB implementation using SQLite3. This current version is feature complete with the ent implementation. The only noticeable difference is the addition of the db version table. The new database is now the default, but we will still run tests on CI to compare behavior against old implementation until it is completely removed.
ProtonMail · Jun 13, 2023 · 3b6ee4c · 3b6ee4c
1 parent ef9959a
commit 3b6ee4c
Show file tree

Hide file tree

Showing 28 changed files with 2,809 additions and 13 deletions.
diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
@@ -30,6 +30,12 @@ jobs:
       - name: Run tests
         run: go test -timeout 15m -v ./...
 
+      - name: Run tests (Ent)
+        if: runner.os == 'Linux'
+        run: go test -timeout 15m -v ./...
+        env:
+          GLUON_TEST_FORCE_ENT_DB: true
+
       - name: Run tests with race check
         if: runner.os != 'Windows'
         run: go test -race -v ./tests

diff --git a/builder.go b/builder.go
@@ -10,7 +10,7 @@ import (
 	"github.com/ProtonMail/gluon/db"
 	"github.com/ProtonMail/gluon/imap"
 	"github.com/ProtonMail/gluon/internal/backend"
-	"github.com/ProtonMail/gluon/internal/db_impl/ent_db"
+	"github.com/ProtonMail/gluon/internal/db_impl/sqlite3"
 	"github.com/ProtonMail/gluon/internal/session"
 	"github.com/ProtonMail/gluon/limits"
 	"github.com/ProtonMail/gluon/profiling"
@@ -50,7 +50,7 @@ func newBuilder() (*serverBuilder, error) {
 		imapLimits:           limits.DefaultLimits(),
 		uidValidityGenerator: imap.DefaultEpochUIDValidityGenerator(),
 		panicHandler:         async.NoopPanicHandler{},
-		dbCI:                 ent_db.NewEntDBBuilder(),
+		dbCI:                 sqlite3.NewBuilder(),
 	}, nil
 }
 

diff --git a/db/ops.go b/db/ops.go
@@ -7,6 +7,7 @@ type ReadOnly interface {
 }
 
 type Transaction interface {
+	ReadOnly
 	MailboxWriteOps
 	MessageWriteOps
 	SubscriptionWriteOps

diff --git a/db/ops_mailbox.go b/db/ops_mailbox.go
@@ -22,7 +22,7 @@ type MailboxReadOps interface {
 
 	GetMailboxMessageIDPairs(ctx context.Context, mboxID imap.InternalMailboxID) ([]MessageIDPair, error)
 
-	GetAllMailboxes(ctx context.Context) ([]*Mailbox, error)
+	GetAllMailboxesWithAttr(ctx context.Context) ([]*Mailbox, error)
 
 	GetAllMailboxesAsRemoteIDs(ctx context.Context) ([]imap.MailboxID, error)
 

diff --git a/db/ops_message.go b/db/ops_message.go
@@ -13,7 +13,7 @@ type MessageReadOps interface {
 
 	MessageExistsWithRemoteID(ctx context.Context, id imap.MessageID) (bool, error)
 
-	GetMessage(ctx context.Context, id imap.InternalMessageID) (*Message, error)
+	GetMessageNoEdges(ctx context.Context, id imap.InternalMessageID) (*Message, error)
 
 	GetTotalMessageCount(ctx context.Context) (int, error)
 

diff --git a/db/types.go b/db/types.go
@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"github.com/ProtonMail/gluon/imap"
+	"github.com/bradenaw/juniper/xslices"
 )
 
 type MailboxIDPair struct {
@@ -101,6 +102,14 @@ type MessageFlag struct {
 	Value string
 }
 
+func MessageFlagsFromFlagSet(set imap.FlagSet) []*MessageFlag {
+	return xslices.Map(set.ToSlice(), func(t string) *MessageFlag {
+		return &MessageFlag{
+			Value: t,
+		}
+	})
+}
+
 type Message struct {
 	ID            imap.InternalMessageID
 	RemoteID      imap.MessageID

diff --git a/go.mod b/go.mod
@@ -10,7 +10,7 @@ require (
 	github.com/emersion/go-imap-uidplus v0.0.0-20200503180755-e75854c361e9
 	github.com/golang/mock v1.6.0
 	github.com/google/uuid v1.3.0
-	github.com/mattn/go-sqlite3 v1.14.16
+	github.com/mattn/go-sqlite3 v1.14.17
 	github.com/pierrec/lz4/v4 v4.1.17
 	github.com/pkg/profile v1.7.0
 	github.com/sirupsen/logrus v1.9.0

diff --git a/go.sum b/go.sum
@@ -53,6 +53,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4=
 github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y=
 github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
+github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM=
+github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
 github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0=
 github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0=
 github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc=

diff --git a/internal/backend/connector_updates.go b/internal/backend/connector_updates.go
@@ -717,7 +717,7 @@ func (user *user) applyMessageUpdated(ctx context.Context, update *imap.MessageU
 // applyUIDValidityBumped applies a UIDValidityBumped event to the user.
 func (user *user) applyUIDValidityBumped(ctx context.Context, update *imap.UIDValidityBumped) error {
 	if err := user.db.Write(ctx, func(ctx context.Context, tx db.Transaction) error {
-		mailboxes, err := tx.GetAllMailboxes(ctx)
+		mailboxes, err := tx.GetAllMailboxesWithAttr(ctx)
 		if err != nil {
 			return err
 		}

diff --git a/internal/db_impl/db_impl.go b/internal/db_impl/db_impl.go
@@ -3,8 +3,13 @@ package db_impl
 import (
 	"github.com/ProtonMail/gluon/db"
 	"github.com/ProtonMail/gluon/internal/db_impl/ent_db"
+	"github.com/ProtonMail/gluon/internal/db_impl/sqlite3"
 )
 
 func NewEntDB() db.ClientInterface {
 	return ent_db.NewEntDB()
 }
+
+func NewSQLiteDB(options ...sqlite3.Option) db.ClientInterface {
+	return sqlite3.NewBuilder(options...)
+}
diff --git a/internal/db_impl/ent_db/ops_read.go b/internal/db_impl/ent_db/ops_read.go
@@ -68,7 +68,7 @@ func (op *EntOpsRead) GetMailboxMessageIDPairs(ctx context.Context, mboxID imap.
 	})
 }
 
-func (op *EntOpsRead) GetAllMailboxes(ctx context.Context) ([]*db.Mailbox, error) {
+func (op *EntOpsRead) GetAllMailboxesWithAttr(ctx context.Context) ([]*db.Mailbox, error) {
 	return wrapEntErrFnTyped(func() ([]*db.Mailbox, error) {
 		val, err := GetAllMailboxes(ctx, op.client)
 
@@ -226,7 +226,7 @@ func (op *EntOpsRead) MessageExistsWithRemoteID(ctx context.Context, id imap.Mes
 	})
 }
 
-func (op *EntOpsRead) GetMessage(ctx context.Context, id imap.InternalMessageID) (*db.Message, error) {
+func (op *EntOpsRead) GetMessageNoEdges(ctx context.Context, id imap.InternalMessageID) (*db.Message, error) {
 	return wrapEntErrFnTyped(func() (*db.Message, error) {
 		msg, err := GetMessage(ctx, op.client, id)