From 7ba466ad6482f3b36eec3ae787c0ee524123cde7 Mon Sep 17 00:00:00 2001 From: Leander Beernaert Date: Wed, 14 Dec 2022 14:48:53 +0100 Subject: [PATCH] feat: Allow MessageCreateUpdate ignore missing mailbox IDs When setting IgnoreUnknownMailboxIDs is set to true the update will no longer fail and will instead try to create the message and insert it to only mailboxes Gluon is aware off. This can be useful during an synchronization with connector state a startup. --- connector/dummy.go | 6 ++++-- connector/dummy_simulate.go | 8 ++++++-- imap/update_message_created.go | 10 +++++++--- internal/backend/connector_updates.go | 7 +++++++ tests/session_test.go | 24 ++++++++++++++++++++++++ tests/updates_test.go | 24 ++++++++++++++++++++++++ 6 files changed, 72 insertions(+), 7 deletions(-) diff --git a/connector/dummy.go b/connector/dummy.go index f9c47989..f280f2c9 100644 --- a/connector/dummy.go +++ b/connector/dummy.go @@ -55,6 +55,8 @@ type Dummy struct { // uidValidity holds the global UID validity. uidValidity imap.UID + + allowMessageCreateWithUnknownMailboxID bool } func NewDummy(usernames []string, password []byte, period time.Duration, flags, permFlags, attrs imap.FlagSet) *Dummy { @@ -162,7 +164,7 @@ func (conn *Dummy) CreateMessage(ctx context.Context, mboxID imap.MailboxID, lit date, ) - update := imap.NewMessagesCreated(&imap.MessageCreated{ + update := imap.NewMessagesCreated(conn.allowMessageCreateWithUnknownMailboxID, &imap.MessageCreated{ Message: message, Literal: literal, MailboxIDs: []imap.MailboxID{mboxID}, @@ -277,7 +279,7 @@ func (conn *Dummy) Sync(ctx context.Context) error { updates = append(updates, update) } - update := imap.NewMessagesCreated(updates...) + update := imap.NewMessagesCreated(conn.allowMessageCreateWithUnknownMailboxID, updates...) defer update.WaitContext(ctx) conn.updateCh <- update diff --git a/connector/dummy_simulate.go b/connector/dummy_simulate.go index db9d2e8e..7c165f33 100644 --- a/connector/dummy_simulate.go +++ b/connector/dummy_simulate.go @@ -79,7 +79,7 @@ func (conn *Dummy) MessageCreated(message imap.Message, literal []byte, mboxIDs mboxIDs: mboxIDMap, } - update := imap.NewMessagesCreated(&imap.MessageCreated{ + update := imap.NewMessagesCreated(conn.allowMessageCreateWithUnknownMailboxID, &imap.MessageCreated{ Message: message, Literal: literal, MailboxIDs: mboxIDs, @@ -126,7 +126,7 @@ func (conn *Dummy) MessagesCreated(messages []imap.Message, literals [][]byte, m }) } - conn.pushUpdate(imap.NewMessagesCreated(updates...)) + conn.pushUpdate(imap.NewMessagesCreated(conn.allowMessageCreateWithUnknownMailboxID, updates...)) return nil } @@ -224,3 +224,7 @@ func (conn *Dummy) UIDValidityBumped() { func (conn *Dummy) Flush() { conn.ticker.Poll() } + +func (conn *Dummy) SetAllowMessageCreateWithUnknownMailboxID(value bool) { + conn.allowMessageCreateWithUnknownMailboxID = value +} diff --git a/imap/update_message_created.go b/imap/update_message_created.go index 3fb8d715..9a3e0506 100644 --- a/imap/update_message_created.go +++ b/imap/update_message_created.go @@ -44,6 +44,9 @@ type MessagesCreated struct { *updateWaiter Messages []*MessageCreated + + // IgnoreUnknownMailboxIDs will allow message creation when one or more MailboxIDs are not yet known when set to true. + IgnoreUnknownMailboxIDs bool } type MessageCreated struct { @@ -53,10 +56,11 @@ type MessageCreated struct { ParsedMessage *ParsedMessage } -func NewMessagesCreated(updates ...*MessageCreated) *MessagesCreated { +func NewMessagesCreated(ignoreUnknownMailboxIDs bool, updates ...*MessageCreated) *MessagesCreated { return &MessagesCreated{ - updateWaiter: newUpdateWaiter(), - Messages: updates, + updateWaiter: newUpdateWaiter(), + Messages: updates, + IgnoreUnknownMailboxIDs: ignoreUnknownMailboxIDs, } } diff --git a/internal/backend/connector_updates.go b/internal/backend/connector_updates.go index 36a7bfd4..fbc2cd70 100644 --- a/internal/backend/connector_updates.go +++ b/internal/backend/connector_updates.go @@ -244,6 +244,13 @@ func (user *user) applyMessagesCreated(ctx context.Context, update *imap.Message if !ok { internalMBoxID, err := db.GetMailboxIDWithRemoteID(ctx, client, mboxID) if err != nil { + // If a mailbox doesn't exist and we are allowed to skip move to next mailbox. + if update.IgnoreUnknownMailboxIDs { + logrus.WithField("MailboxID", mboxID.ShortID()). + WithField("MessageID", message.Message.ID.ShortID()). + Warn("Unknown Mailbox ID, skipping add to mailbox") + continue + } return err } diff --git a/tests/session_test.go b/tests/session_test.go index 73c89895..5443b289 100644 --- a/tests/session_test.go +++ b/tests/session_test.go @@ -33,6 +33,8 @@ type Connector interface { MailboxDeleted(imap.MailboxID) error SetMailboxVisible(imap.MailboxID, bool) + SetAllowMessageCreateWithUnknownMailboxID(value bool) + MessageCreated(imap.Message, []byte, []imap.MailboxID) error MessagesCreated([]imap.Message, [][]byte, [][]imap.MailboxID) error MessageUpdated(imap.Message, []byte, []imap.MailboxID) error @@ -137,6 +139,10 @@ func (s *testSession) mailboxCreated(user string, name []string, withData ...str return s.mailboxCreatedWithAttributes(user, name, defaultAttributes, withData...) } +func (s *testSession) setAllowMessageCreateWithUnknownMailboxID(user string, value bool) { + s.conns[s.userIDs[user]].SetAllowMessageCreateWithUnknownMailboxID(value) +} + func (s *testSession) mailboxDeleted(user string, id imap.MailboxID) { require.NoError(s.tb, s.conns[s.userIDs[user]].MailboxDeleted(id)) } @@ -199,6 +205,24 @@ func (s *testSession) mailboxCreatedCustom(user string, name []string, flags, pe return mboxID } +func (s *testSession) messageCreatedWithMailboxes(user string, mailboxIDs []imap.MailboxID, literal []byte, internalDate time.Time, flags ...string) imap.MessageID { + messageID := imap.MessageID(utils.NewRandomMessageID()) + + require.NoError(s.tb, s.conns[s.userIDs[user]].MessageCreated( + imap.Message{ + ID: messageID, + Flags: imap.NewFlagSetFromSlice(flags), + Date: internalDate, + }, + literal, + mailboxIDs, + )) + + s.conns[s.userIDs[user]].Flush() + + return messageID +} + func (s *testSession) messageCreated(user string, mailboxID imap.MailboxID, literal []byte, internalDate time.Time, flags ...string) imap.MessageID { messageID := imap.MessageID(utils.NewRandomMessageID()) diff --git a/tests/updates_test.go b/tests/updates_test.go index a32f7293..36d05846 100644 --- a/tests/updates_test.go +++ b/tests/updates_test.go @@ -1,7 +1,10 @@ package tests import ( + "github.com/emersion/go-imap/client" + "github.com/stretchr/testify/require" "testing" + "time" "github.com/ProtonMail/gluon/imap" "github.com/ProtonMail/gluon/internal/utils" @@ -306,3 +309,24 @@ func TestBatchMessageAddedWithMultipleFlags(t *testing.T) { s.flush("user") }) } + +func TestMessageCreatedWithIgnoreMissingMailbox(t *testing.T) { + runOneToOneTestClientWithAuth(t, defaultServerOptions(t), func(c *client.Client, s *testSession) { + mailboxID := s.mailboxCreated("user", []string{"mbox"}) + { + // First round fails as a missing mailbox is not allowed. + s.messageCreatedWithMailboxes("user", []imap.MailboxID{mailboxID, "THIS MAILBOX DOES NOT EXISTS"}, []byte("To: Test"), time.Now()) + status, err := c.Select("mbox", false) + require.NoError(t, err) + require.Equal(t, status.Messages, uint32(0)) + } + { + // Second round succeeds as we publish an update that is allowed to fail. + s.setAllowMessageCreateWithUnknownMailboxID("user", true) + s.messageCreatedWithMailboxes("user", []imap.MailboxID{mailboxID, "THIS MAILBOX DOES NOT EXISTS"}, []byte("To: Test"), time.Now()) + status, err := c.Select("mbox", false) + require.NoError(t, err) + require.Equal(t, status.Messages, uint32(1)) + } + }) +}