feat: Allow MessageCreateUpdate ignore missing mailbox IDs

When setting IgnoreUnknownMailboxIDs is set to true the update will no
longer fail and will instead try to create the message and insert it to
only mailboxes Gluon is aware off.

This can be useful during an synchronization with connector state a
startup.

connector/dummy.go

@@ -55,6 +55,8 @@ type Dummy struct {
 
 	// uidValidity holds the global UID validity.
 	uidValidity imap.UID
+
+	allowMessageCreateWithUnknownMailboxID bool
 }
 
 func NewDummy(usernames []string, password []byte, period time.Duration, flags, permFlags, attrs imap.FlagSet) *Dummy {
@@ -162,7 +164,7 @@ func (conn *Dummy) CreateMessage(ctx context.Context, mboxID imap.MailboxID, lit
 		date,
 	)
 
-	update := imap.NewMessagesCreated(&imap.MessageCreated{
+	update := imap.NewMessagesCreated(conn.allowMessageCreateWithUnknownMailboxID, &imap.MessageCreated{
 		Message:       message,
 		Literal:       literal,
 		MailboxIDs:    []imap.MailboxID{mboxID},
@@ -277,7 +279,7 @@ func (conn *Dummy) Sync(ctx context.Context) error {
 		updates = append(updates, update)
 	}
 
-	update := imap.NewMessagesCreated(updates...)
+	update := imap.NewMessagesCreated(conn.allowMessageCreateWithUnknownMailboxID, updates...)
 	defer update.WaitContext(ctx)
 
 	conn.updateCh <- update

connector/dummy_simulate.go

@@ -79,7 +79,7 @@ func (conn *Dummy) MessageCreated(message imap.Message, literal []byte, mboxIDs
 		mboxIDs: mboxIDMap,
 	}
 
-	update := imap.NewMessagesCreated(&imap.MessageCreated{
+	update := imap.NewMessagesCreated(conn.allowMessageCreateWithUnknownMailboxID, &imap.MessageCreated{
 		Message:       message,
 		Literal:       literal,
 		MailboxIDs:    mboxIDs,
@@ -126,7 +126,7 @@ func (conn *Dummy) MessagesCreated(messages []imap.Message, literals [][]byte, m
 		})
 	}
 
-	conn.pushUpdate(imap.NewMessagesCreated(updates...))
+	conn.pushUpdate(imap.NewMessagesCreated(conn.allowMessageCreateWithUnknownMailboxID, updates...))
 
 	return nil
 }
@@ -224,3 +224,7 @@ func (conn *Dummy) UIDValidityBumped() {
 func (conn *Dummy) Flush() {
 	conn.ticker.Poll()
 }
+
+func (conn *Dummy) SetAllowMessageCreateWithUnknownMailboxID(value bool) {
+	conn.allowMessageCreateWithUnknownMailboxID = value
+}

imap/update_message_created.go

@@ -44,6 +44,9 @@ type MessagesCreated struct {
 	*updateWaiter
 
 	Messages []*MessageCreated
+
+	// IgnoreUnknownMailboxIDs will allow message creation when one or more MailboxIDs are not yet known when set to true.
+	IgnoreUnknownMailboxIDs bool
 }
 
 type MessageCreated struct {
@@ -53,10 +56,11 @@ type MessageCreated struct {
 	ParsedMessage *ParsedMessage
 }
 
-func NewMessagesCreated(updates ...*MessageCreated) *MessagesCreated {
+func NewMessagesCreated(ignoreUnknownMailboxIDs bool, updates ...*MessageCreated) *MessagesCreated {
 	return &MessagesCreated{
-		updateWaiter: newUpdateWaiter(),
-		Messages:     updates,
+		updateWaiter:            newUpdateWaiter(),
+		Messages:                updates,
+		IgnoreUnknownMailboxIDs: ignoreUnknownMailboxIDs,
 	}
 }
 

internal/backend/connector_updates.go

@@ -244,6 +244,13 @@ func (user *user) applyMessagesCreated(ctx context.Context, update *imap.Message
 				if !ok {
 					internalMBoxID, err := db.GetMailboxIDWithRemoteID(ctx, client, mboxID)
 					if err != nil {
+						// If a mailbox doesn't exist and we are allowed to skip move to next mailbox.
+						if update.IgnoreUnknownMailboxIDs {
+							logrus.WithField("MailboxID", mboxID.ShortID()).
+								WithField("MessageID", message.Message.ID.ShortID()).
+								Warn("Unknown Mailbox ID, skipping add to mailbox")
+							continue
+						}
 						return err
 					}
 

tests/session_test.go

@@ -33,6 +33,8 @@ type Connector interface {
 	MailboxDeleted(imap.MailboxID) error
 	SetMailboxVisible(imap.MailboxID, bool)
 
+	SetAllowMessageCreateWithUnknownMailboxID(value bool)
+
 	MessageCreated(imap.Message, []byte, []imap.MailboxID) error
 	MessagesCreated([]imap.Message, [][]byte, [][]imap.MailboxID) error
 	MessageUpdated(imap.Message, []byte, []imap.MailboxID) error
@@ -137,6 +139,10 @@ func (s *testSession) mailboxCreated(user string, name []string, withData ...str
 	return s.mailboxCreatedWithAttributes(user, name, defaultAttributes, withData...)
 }
 
+func (s *testSession) setAllowMessageCreateWithUnknownMailboxID(user string, value bool) {
+	s.conns[s.userIDs[user]].SetAllowMessageCreateWithUnknownMailboxID(value)
+}
+
 func (s *testSession) mailboxDeleted(user string, id imap.MailboxID) {
 	require.NoError(s.tb, s.conns[s.userIDs[user]].MailboxDeleted(id))
 }
@@ -199,6 +205,24 @@ func (s *testSession) mailboxCreatedCustom(user string, name []string, flags, pe
 	return mboxID
 }
 
+func (s *testSession) messageCreatedWithMailboxes(user string, mailboxIDs []imap.MailboxID, literal []byte, internalDate time.Time, flags ...string) imap.MessageID {
+	messageID := imap.MessageID(utils.NewRandomMessageID())
+
+	require.NoError(s.tb, s.conns[s.userIDs[user]].MessageCreated(
+		imap.Message{
+			ID:    messageID,
+			Flags: imap.NewFlagSetFromSlice(flags),
+			Date:  internalDate,
+		},
+		literal,
+		mailboxIDs,
+	))
+
+	s.conns[s.userIDs[user]].Flush()
+
+	return messageID
+}
+
 func (s *testSession) messageCreated(user string, mailboxID imap.MailboxID, literal []byte, internalDate time.Time, flags ...string) imap.MessageID {
 	messageID := imap.MessageID(utils.NewRandomMessageID())
 

tests/updates_test.go

@@ -1,7 +1,10 @@
 package tests
 
 import (
+	"github.com/emersion/go-imap/client"
+	"github.com/stretchr/testify/require"
 	"testing"
+	"time"
 
 	"github.com/ProtonMail/gluon/imap"
 	"github.com/ProtonMail/gluon/internal/utils"
@@ -306,3 +309,24 @@ func TestBatchMessageAddedWithMultipleFlags(t *testing.T) {
 		s.flush("user")
 	})
 }
+
+func TestMessageCreatedWithIgnoreMissingMailbox(t *testing.T) {
+	runOneToOneTestClientWithAuth(t, defaultServerOptions(t), func(c *client.Client, s *testSession) {
+		mailboxID := s.mailboxCreated("user", []string{"mbox"})
+		{
+			// First round fails as a missing mailbox is not allowed.
+			s.messageCreatedWithMailboxes("user", []imap.MailboxID{mailboxID, "THIS MAILBOX DOES NOT EXISTS"}, []byte("To: Test"), time.Now())
+			status, err := c.Select("mbox", false)
+			require.NoError(t, err)
+			require.Equal(t, status.Messages, uint32(0))
+		}
+		{
+			// Second round succeeds as we publish an update that is allowed to fail.
+			s.setAllowMessageCreateWithUnknownMailboxID("user", true)
+			s.messageCreatedWithMailboxes("user", []imap.MailboxID{mailboxID, "THIS MAILBOX DOES NOT EXISTS"}, []byte("To: Test"), time.Now())
+			status, err := c.Select("mbox", false)
+			require.NoError(t, err)
+			require.Equal(t, status.Messages, uint32(1))
+		}
+	})
+}