Skip to content

Commit

Permalink
feat(GODT-2289): Use timestamp for UIDValidity
Browse files Browse the repository at this point in the history
Use a epoch timestamp for uidvalidity starting from 1st of February
2023. This will ensure that the user will always have unique
UIDValidities even if the connector data is erased/lost.
  • Loading branch information
LBeernaertProton committed Feb 1, 2023
1 parent 2c64e59 commit 565039a
Show file tree
Hide file tree
Showing 28 changed files with 343 additions and 185 deletions.
74 changes: 39 additions & 35 deletions builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package gluon

import (
"crypto/tls"
"github.com/ProtonMail/gluon/imap"
"io"
"os"
"time"
Expand All @@ -17,30 +18,32 @@ import (
)

type serverBuilder struct {
dataDir string
databaseDir string
delim string
loginJailTime time.Duration
tlsConfig *tls.Config
idleBulkTime time.Duration
inLogger io.Writer
outLogger io.Writer
versionInfo version.Info
cmdExecProfBuilder profiling.CmdProfilerBuilder
storeBuilder store.Builder
reporter reporter.Reporter
disableParallelism bool
imapLimits limits.IMAP
dataDir string
databaseDir string
delim string
loginJailTime time.Duration
tlsConfig *tls.Config
idleBulkTime time.Duration
inLogger io.Writer
outLogger io.Writer
versionInfo version.Info
cmdExecProfBuilder profiling.CmdProfilerBuilder
storeBuilder store.Builder
reporter reporter.Reporter
disableParallelism bool
imapLimits limits.IMAP
uidValidityGenerator imap.UIDValidityGenerator
}

func newBuilder() (*serverBuilder, error) {
return &serverBuilder{
delim: "/",
cmdExecProfBuilder: &profiling.NullCmdExecProfilerBuilder{},
storeBuilder: &store.OnDiskStoreBuilder{},
reporter: &reporter.NullReporter{},
idleBulkTime: 500 * time.Millisecond,
imapLimits: limits.DefaultLimits(),
delim: "/",
cmdExecProfBuilder: &profiling.NullCmdExecProfilerBuilder{},
storeBuilder: &store.OnDiskStoreBuilder{},
reporter: &reporter.NullReporter{},
idleBulkTime: 500 * time.Millisecond,
imapLimits: limits.DefaultLimits(),
uidValidityGenerator: imap.DefaultEpochUIDValidityGenerator(),
}, nil
}

Expand Down Expand Up @@ -84,20 +87,21 @@ func (builder *serverBuilder) build() (*Server, error) {
}

return &Server{
dataDir: builder.dataDir,
databaseDir: builder.databaseDir,
backend: backend,
sessions: make(map[int]*session.Session),
serveErrCh: queue.NewQueuedChannel[error](1, 1),
serveDoneCh: make(chan struct{}),
inLogger: builder.inLogger,
outLogger: builder.outLogger,
tlsConfig: builder.tlsConfig,
idleBulkTime: builder.idleBulkTime,
storeBuilder: builder.storeBuilder,
cmdExecProfBuilder: builder.cmdExecProfBuilder,
versionInfo: builder.versionInfo,
reporter: builder.reporter,
disableParallelism: builder.disableParallelism,
dataDir: builder.dataDir,
databaseDir: builder.databaseDir,
backend: backend,
sessions: make(map[int]*session.Session),
serveErrCh: queue.NewQueuedChannel[error](1, 1),
serveDoneCh: make(chan struct{}),
inLogger: builder.inLogger,
outLogger: builder.outLogger,
tlsConfig: builder.tlsConfig,
idleBulkTime: builder.idleBulkTime,
storeBuilder: builder.storeBuilder,
cmdExecProfBuilder: builder.cmdExecProfBuilder,
versionInfo: builder.versionInfo,
reporter: builder.reporter,
disableParallelism: builder.disableParallelism,
uidValidityGenerator: builder.uidValidityGenerator,
}, nil
}
6 changes: 0 additions & 6 deletions connector/connector.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,12 +50,6 @@ type Connector interface {
// GetUpdates returns a stream of updates that the gluon server should apply.
GetUpdates() <-chan imap.Update

// GetUIDValidity returns the default UID validity for this user.
GetUIDValidity() imap.UID

// SetUIDValidity sets the default UID validity for this user.
SetUIDValidity(uidValidity imap.UID) error

// Close the connector will no longer be used and all resources should be closed/released.
Close(ctx context.Context) error
}
14 changes: 0 additions & 14 deletions connector/dummy.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,9 +56,6 @@ type Dummy struct {
// hiddenMailboxes holds mailboxes that are hidden from the user.
hiddenMailboxes map[imap.MailboxID]struct{}

// uidValidity holds the global UID validity.
uidValidity imap.UID

allowMessageCreateWithUnknownMailboxID bool

updatesAllowedToFail int32
Expand All @@ -76,7 +73,6 @@ func NewDummy(usernames []string, password []byte, period time.Duration, flags,
updateQuitCh: make(chan struct{}),
ticker: ticker.New(period),
hiddenMailboxes: make(map[imap.MailboxID]struct{}),
uidValidity: 1,
}

go func() {
Expand Down Expand Up @@ -268,16 +264,6 @@ func (conn *Dummy) MarkMessagesFlagged(ctx context.Context, messageIDs []imap.Me
return nil
}

func (conn *Dummy) GetUIDValidity() imap.UID {
return conn.uidValidity
}

func (conn *Dummy) SetUIDValidity(newUIDValidity imap.UID) error {
conn.uidValidity = newUIDValidity

return nil
}

func (conn *Dummy) Sync(ctx context.Context) error {
for _, mailbox := range conn.state.getMailboxes() {
update := imap.NewMailboxCreated(mailbox)
Expand Down
3 changes: 1 addition & 2 deletions connector/dummy_simulate.go
Original file line number Diff line number Diff line change
Expand Up @@ -217,8 +217,7 @@ func (conn *Dummy) MessageDeleted(messageID imap.MessageID) error {
}

func (conn *Dummy) UIDValidityBumped() {
conn.uidValidity += 1
conn.pushUpdate(imap.NewUIDValidityBumped(conn.uidValidity))
conn.pushUpdate(imap.NewUIDValidityBumped())
}

func (conn *Dummy) Flush() {
Expand Down
87 changes: 87 additions & 0 deletions imap/uid_validity_generator.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
package imap

import (
"fmt"
"sync/atomic"
"time"
)

type UIDValidityGenerator interface {
Generate() (UID, error)
}

type EpochUIDValidityGenerator struct {
epochStart time.Time
lastUID uint32
}

func NewEpochUIDValidityGenerator(epochStart time.Time) *EpochUIDValidityGenerator {
return &EpochUIDValidityGenerator{
epochStart: epochStart,
}
}

func DefaultEpochUIDValidityGenerator() *EpochUIDValidityGenerator {
return NewEpochUIDValidityGenerator(time.Date(2023, 2, 1, 0, 0, 0, 0, time.UTC))
}

func (e *EpochUIDValidityGenerator) Generate() (UID, error) {
timeStamp := uint64(time.Now().Sub(e.epochStart).Seconds())
if timeStamp > uint64(0xFFFFFFFF) {
return 0, fmt.Errorf("failed to generate uid validity, interval exceeded maximum capacity")
}

timeStampU32 := uint32(timeStamp)

// This loops is here to ensure that two successive calls to Generate that happen during the same second
// can still generate unique values. To avoid waiting another second until the values are different,
// we keep bumping the last generated value until it is greater than the last generated value.
for {
lastGenerated := atomic.LoadUint32(&e.lastUID)

// Not enough time elapsed between the last time
if lastGenerated >= timeStampU32 {
if timeStampU32 == 0xFFFFFFFF {
return 0, fmt.Errorf("failed to generate uid validity, interval exceeded maximum capacity")
}

timeStampU32 += 1

continue
}

if !atomic.CompareAndSwapUint32(&e.lastUID, lastGenerated, timeStampU32) {
continue
}

return UID(timeStampU32), nil
}
}

type IncrementalUIDValidityGenerator struct {
counter uint32
}

func (i *IncrementalUIDValidityGenerator) Generate() (UID, error) {
return UID(atomic.AddUint32(&i.counter, 1)), nil
}

func (i *IncrementalUIDValidityGenerator) GetValue() UID {
return UID(atomic.LoadUint32(&i.counter))
}

func NewIncrementalUIDValidityGenerator() *IncrementalUIDValidityGenerator {
return &IncrementalUIDValidityGenerator{}
}

type FixedUIDValidityGenerator struct {
Value UID
}

func (f FixedUIDValidityGenerator) Generate() (UID, error) {
return f.Value, nil
}

func NewFixedUIDValidityGenerator(value UID) *FixedUIDValidityGenerator {
return &FixedUIDValidityGenerator{Value: value}
}
56 changes: 56 additions & 0 deletions imap/uid_validity_generator_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
package imap

import (
"github.com/bradenaw/juniper/parallel"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/exp/slices"
"testing"
"time"
)

func TestEpochUIDValidityGenerator_Generate(t *testing.T) {
generator := DefaultEpochUIDValidityGenerator()

const UIDCount = 10

var uids = make([]UID, UIDCount)

for i := 0; i < UIDCount; i++ {
uid, err := generator.Generate()
require.NoError(t, err)

uids[i] = uid
}

time.Sleep(10 * time.Second)

uid, err := generator.Generate()
require.NoError(t, err)

for i := 0; i < UIDCount-1; i++ {
assert.Less(t, uids[i], uids[i+1])
}

assert.Greater(t, uid, uids[UIDCount-1])
}

func TestEpochUIDValidityGenerator_GenerateParallel(t *testing.T) {
generator := DefaultEpochUIDValidityGenerator()

const UIDCount = 1000

var uids = make([]UID, UIDCount)

parallel.Do(0, UIDCount, func(i int) {
uid, err := generator.Generate()
require.NoError(t, err)
uids[i] = uid
})

slices.Sort(uids)

for i := 0; i < UIDCount-1; i++ {
assert.Less(t, uids[i], uids[i+1])
}
}
8 changes: 2 additions & 6 deletions imap/update_uid_validity_bumped.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,14 @@ type UIDValidityBumped struct {
updateBase

*updateWaiter

UIDValidity UID
}

func NewUIDValidityBumped(validity UID) *UIDValidityBumped {
func NewUIDValidityBumped() *UIDValidityBumped {
return &UIDValidityBumped{
updateWaiter: newUpdateWaiter(),

UIDValidity: validity,
}
}

func (u *UIDValidityBumped) String() string {
return fmt.Sprintf("UIDValidityBumped: %v", u.UIDValidity)
return fmt.Sprintf("UIDValidityBumped")
}
4 changes: 2 additions & 2 deletions internal/backend/backend.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ func (b *Backend) GetDelimiter() string {

// AddUser adds a new user to the backend.
// It returns true if the user's database was created, false if it already existed.
func (b *Backend) AddUser(ctx context.Context, userID string, conn connector.Connector, passphrase []byte) (bool, error) {
func (b *Backend) AddUser(ctx context.Context, userID string, conn connector.Connector, passphrase []byte, uidValidityGenerator imap.UIDValidityGenerator) (bool, error) {
b.usersLock.Lock()
defer b.usersLock.Unlock()

Expand All @@ -91,7 +91,7 @@ func (b *Backend) AddUser(ctx context.Context, userID string, conn connector.Con
return false, err
}

user, err := newUser(ctx, userID, db, conn, storeBuilder, b.delim, b.imapLimits)
user, err := newUser(ctx, userID, db, conn, storeBuilder, b.delim, b.imapLimits, uidValidityGenerator)
if err != nil {
return false, err
}
Expand Down
Loading

0 comments on commit 565039a

Please sign in to comment.