#230: pop3_uidl (mdbox)implementation

src/storage-rbox/rbox-mail.cpp

@@ -32,6 +32,7 @@ extern "C" {
 #include "debug-helper.h"
 #include "limits.h"
 #include "macros.h"
+#include "index-pop3-uidl.h"
 }
 
 #include "../librmb/rados-mail.h"
@@ -487,6 +488,7 @@ static int rbox_get_cached_metadata(struct rbox_mail *mail, enum rbox_metadata_k
 
 static int rbox_mail_get_special(struct mail *_mail, enum mail_fetch_field field, const char **value_r) {
   struct rbox_mail *mail = (struct rbox_mail *)_mail;
+  struct rbox_mailbox *mbox = (struct rbox_mailbox *)_mail->box;
   int ret = 0;
 
   *value_r = NULL;
@@ -497,30 +499,32 @@ static int rbox_mail_get_special(struct mail *_mail, enum mail_fetch_field field
     case MAIL_FETCH_GUID:
       return rbox_get_cached_metadata(mail, rbox_metadata_key::RBOX_METADATA_GUID, MAIL_CACHE_GUID, value_r);
     case MAIL_FETCH_UIDL_BACKEND:
-#ifdef DOVECOT_CEPH_PLUGINS_HAVE_INDEX_POP3_UIDL_H
+      if (!rbox_header_have_flag(_mail->box, mbox->hdr_ext_id, offsetof(struct rbox_index_header, flags),
+                                 RBOX_INDEX_HEADER_FLAG_HAVE_POP3_UIDLS)) {
+        *value_r = "";
+        return 0;
+      }
       if (!index_pop3_uidl_can_exist(_mail)) {
         *value_r = "";
         return 0;
       }
-#endif
-
       ret = rbox_get_cached_metadata(mail, rbox_metadata_key::RBOX_METADATA_POP3_UIDL, MAIL_CACHE_POP3_UIDL, value_r);
-
-#ifdef DOVECOT_CEPH_PLUGINS_HAVE_INDEX_POP3_UIDL_H
       if (ret == 0) {
         index_pop3_uidl_update_exists(&mail->imail.mail.mail, (*value_r)[0] != '\0');
       }
-#endif
       return ret;
     case MAIL_FETCH_POP3_ORDER:
-#ifdef DOVECOT_CEPH_PLUGINS_HAVE_INDEX_POP3_UIDL_H
+      if (!rbox_header_have_flag(_mail->box, mbox->hdr_ext_id, offsetof(struct rbox_index_header, flags),
+                                 RBOX_INDEX_HEADER_FLAG_HAVE_POP3_ORDERS)) {
+        *value_r = "";
+        return 0;
+      }
       if (!index_pop3_uidl_can_exist(_mail)) {
         /* we're assuming that if there's a POP3 order, there's
            also a UIDL */
         *value_r = "";
         return 0;
       }
-#endif
       return rbox_get_cached_metadata(mail, rbox_metadata_key::RBOX_METADATA_POP3_ORDER, MAIL_CACHE_POP3_ORDER,
                                       value_r);
 
@@ -578,7 +582,7 @@ static void rbox_index_mail_set_seq(struct mail *_mail, uint32_t seq, bool savin
     rbox_get_index_record(_mail);
   }
 }
-
+/*static void rbox_update_pop3_uidl(struct mail *_mail, const char *uidl) { i_debug("UIDL: %s", uidl); }*/
 /*ebd if old version */
 // rbox_mail_free,
 struct mail_vfuncs rbox_mail_vfuncs = {rbox_mail_close,
@@ -616,7 +620,7 @@ struct mail_vfuncs rbox_mail_vfuncs = {rbox_mail_close,
                                        index_mail_update_keywords,
                                        index_mail_update_modseq,
                                        index_mail_update_pvt_modseq,
-                                       NULL,
+                                       NULL /*rbox_update_pop3_uidl*/,
                                        index_mail_expunge,
                                        index_mail_set_cache_corrupted,
                                        index_mail_opened,

src/storage-rbox/rbox-save.cpp

@@ -30,6 +30,7 @@ extern "C" {
 #include "rbox-sync.h"
 
 #include "debug-helper.h"
+#include "index-pop3-uidl.h"
 }
 
 #include "../librmb/rados-mail.h"
@@ -41,9 +42,9 @@ extern "C" {
 
 using ceph::bufferlist;
 
-using librmb::RadosStorage;
 using librmb::RadosMail;
 using librmb::RadosMetadata;
+using librmb::RadosStorage;
 using librmb::rbox_metadata_key;
 
 using std::string;
@@ -287,12 +288,16 @@ static int rbox_save_mail_set_metadata(struct rbox_save_context *r_ctx, librmb::
     if (mdata->pop3_uidl != NULL) {
       RadosMetadata xattr(rbox_metadata_key::RBOX_METADATA_POP3_UIDL, mdata->pop3_uidl);
       mail_object->add_metadata(xattr);
+      r_ctx->have_pop3_uidls = TRUE;
+      r_ctx->highest_pop3_uidl_seq = I_MAX(r_ctx->highest_pop3_uidl_seq, r_ctx->seq);
     }
   }
   if (r_storage->config->is_mail_attribute(rbox_metadata_key::RBOX_METADATA_POP3_ORDER)) {
     if (mdata->pop3_order != 0) {
       RadosMetadata xattr(rbox_metadata_key::RBOX_METADATA_POP3_ORDER, mdata->pop3_order);
       mail_object->add_metadata(xattr);
+      r_ctx->have_pop3_orders = TRUE;
+      r_ctx->highest_pop3_uidl_seq = I_MAX(r_ctx->highest_pop3_uidl_seq, r_ctx->seq);
     }
   }
   if (r_storage->config->is_mail_attribute(rbox_metadata_key::RBOX_METADATA_FROM_ENVELOPE)) {
@@ -423,7 +428,7 @@ int rbox_save_finish(struct mail_save_context *_ctx) {
       uint32_t t = _ctx->data.save_date;
       index_mail_cache_add((struct index_mail *)_ctx->dest_mail, MAIL_CACHE_SAVE_DATE, &t, sizeof(t));
     }
-
+    
 #if DOVECOT_PREREQ(2, 3)
     int ret = 0;
     if (r_ctx->ctx.data.output != r_ctx->output_stream) {
@@ -522,6 +527,9 @@ static int rbox_save_assign_uids(struct rbox_save_context *r_ctx, const ARRAY_TY
           return -1;
         }
       }
+      if (r_ctx->highest_pop3_uidl_seq == n + 1) {
+        index_pop3_uidl_set_max_uid(&r_ctx->mbox->box, r_ctx->trans, uid);
+      }
     }
     i_assert(!seq_range_array_iter_nth(&iter, n, &uid));
   }
@@ -547,6 +555,10 @@ void rbox_save_update_header_flags(struct rbox_save_context *r_ctx, struct mail_
   }
 
   flags = old_flags;
+  if (r_ctx->have_pop3_uidls)
+    flags |= RBOX_INDEX_HEADER_FLAG_HAVE_POP3_UIDLS;
+  if (r_ctx->have_pop3_orders)
+    flags |= RBOX_INDEX_HEADER_FLAG_HAVE_POP3_ORDERS;
   if (flags != old_flags) {
     /* flags changed, update them */
     mail_index_update_header_ext(r_ctx->trans, ext_id, flags_offset, &flags, 1);

src/storage-rbox/rbox-save.h

@@ -42,7 +42,9 @@ class rbox_save_context {
         failed(1),
         finished(1),
         copying(0),
-        dest_mail_allocated(0) {}
+        dest_mail_allocated(0),
+        highest_pop3_uidl_seq(0),
+        have_pop3_uidls(0) {}
 
   /** dovecot mail save context **/
   struct mail_save_context ctx;
@@ -69,7 +71,9 @@ class rbox_save_context {
   std::vector<librmb::RadosMail *> rados_mails;
   /** current mail in the context **/
   librmb::RadosMail *rados_mail;
-
+  unsigned int highest_pop3_uidl_seq : 1;
+  unsigned int have_pop3_uidls : 1;
+  unsigned int have_pop3_orders : 1;
   unsigned int failed : 1;
   unsigned int finished : 1;
   unsigned int copying : 1;

src/storage-rbox/rbox-storage.cpp

@@ -26,6 +26,7 @@ extern "C" {
 #include "guid.h"
 #include "mailbox-list-fs.h"
 #include "macros.h"
+#include "index-pop3-uidl.h"
 }
 
 #include "rbox-mailbox-list-fs.h"
@@ -422,26 +423,26 @@ int rbox_open_rados_connection(struct mailbox *box, bool alt_storage) {
   // initialize storage with plugin configuration
   read_plugin_configuration(box);
   int ret = 0;
-try{
-  rados_storage->set_ceph_wait_method(rbox->storage->config->is_ceph_aio_wait_for_safe_and_cb()
-                                          ? librmb::WAIT_FOR_SAFE_AND_CB
-                                          : librmb::WAIT_FOR_COMPLETE_AND_CB);
-  /* open connection to primary and alternative storage */
-  ret = rados_storage->open_connection(rbox->storage->config->get_pool_name(),
-                                           rbox->storage->config->get_rados_cluster_name(),
-                                           rbox->storage->config->get_rados_username());
-
-  if (alt_storage) {
-    ret = rbox->storage->alt->open_connection(box->list->set.alt_dir, rbox->storage->config->get_rados_cluster_name(),
-                                              rbox->storage->config->get_rados_username());
-
-    rbox->storage->alt->set_ceph_wait_method(rbox->storage->config->is_ceph_aio_wait_for_safe_and_cb()
-                                                 ? librmb::WAIT_FOR_SAFE_AND_CB
-                                                 : librmb::WAIT_FOR_COMPLETE_AND_CB);
-  }
-}catch(std::exception& e){
-  ret= -1;
-}
+  try {
+    rados_storage->set_ceph_wait_method(rbox->storage->config->is_ceph_aio_wait_for_safe_and_cb()
+                                            ? librmb::WAIT_FOR_SAFE_AND_CB
+                                            : librmb::WAIT_FOR_COMPLETE_AND_CB);
+    /* open connection to primary and alternative storage */
+    ret = rados_storage->open_connection(rbox->storage->config->get_pool_name(),
+                                         rbox->storage->config->get_rados_cluster_name(),
+                                         rbox->storage->config->get_rados_username());
+
+    if (alt_storage) {
+      ret = rbox->storage->alt->open_connection(box->list->set.alt_dir, rbox->storage->config->get_rados_cluster_name(),
+                                                rbox->storage->config->get_rados_username());
+
+      rbox->storage->alt->set_ceph_wait_method(rbox->storage->config->is_ceph_aio_wait_for_safe_and_cb()
+                                                   ? librmb::WAIT_FOR_SAFE_AND_CB
+                                                   : librmb::WAIT_FOR_COMPLETE_AND_CB);
+    }
+  } catch (std::exception &e) {
+    ret = -1;
+  }
 
   if (ret == 1) {
     // already connected nothing to do!
@@ -582,13 +583,11 @@ int rbox_mailbox_create_indexes(struct mailbox *box, const struct mailbox_update
   }
   mail_index_view_close(&view);
 
-#ifdef DOVECOT_CEPH_PLUGINS_HAVE_INDEX_POP3_UIDL_H
   if (box->inbox_user && box->creating) {
     /* initialize pop3-uidl header when creating mailbox
        (not on mailbox_update()) */
     index_pop3_uidl_set_max_uid(box, trans, 0);
   }
-#endif
 
   rbox_update_header((struct rbox_mailbox *)box, trans, update);
   if (trans != NULL) {
@@ -853,8 +852,9 @@ int check_users_mailbox_delete_ns_object(struct mail_user *user, librmb::RadosDo
   for (; ns != NULL; ns = ns->next) {
     struct mailbox_list_iterate_context *iter;
     const struct mailbox_info *info;
-    iter = mailbox_list_iter_init(ns->list, "*", static_cast<enum mailbox_list_iter_flags>(
-                                                     MAILBOX_LIST_ITER_RAW_LIST | MAILBOX_LIST_ITER_RETURN_NO_FLAGS));
+    iter = mailbox_list_iter_init(
+        ns->list, "*",
+        static_cast<enum mailbox_list_iter_flags>(MAILBOX_LIST_ITER_RAW_LIST | MAILBOX_LIST_ITER_RETURN_NO_FLAGS));
 
     int total_mails = 0;
     while ((info = mailbox_list_iter_next(iter)) != NULL) {
@@ -937,6 +937,17 @@ int rbox_storage_mailbox_delete(struct mailbox *box) {
   return ret;
 }
 
+bool rbox_header_have_flag(struct mailbox *box, uint32_t ext_id, unsigned int flags_offset, uint8_t flag) {
+  const void *data;
+  size_t data_size;
+  uint8_t flags = 0;
+
+  mail_index_get_header_ext(box->view, ext_id, &data, &data_size);
+  if (flags_offset < data_size)
+    flags = *((const uint8_t *)data + flags_offset);
+  return (flags & flag) != 0;
+}
+
 struct mailbox_vfuncs rbox_mailbox_vfuncs = {index_storage_is_readonly,
                                              index_storage_mailbox_enable,
                                              index_storage_mailbox_exists,

src/storage-rbox/rbox-storage.h

@@ -63,6 +63,17 @@ extern void read_plugin_configuration(struct mailbox *box);
  */
 extern int rbox_storage_mailbox_delete(struct mailbox *box);
 
+/**
+ * check for flag
+ *
+ * @param box mailbox
+ * @param ext_id header extension id
+ * @param flags_offset flags offset
+ * @param flag flag
+ * @return
+ */
+extern bool rbox_header_have_flag(struct mailbox *box, uint32_t ext_id, unsigned int flags_offset, uint8_t flag);
+
 #ifdef __cplusplus
 }
 #endif
@@ -95,4 +106,11 @@ struct rbox_mailbox {
   ARRAY(struct expunged_item *) moved_items;
 };
 
+enum rbox_index_header_flags {
+  /* messages' metadata contain POP3 UIDLs */
+  RBOX_INDEX_HEADER_FLAG_HAVE_POP3_UIDLS = 0x01,
+  /* messages' metadata contain POP3 orders */
+  RBOX_INDEX_HEADER_FLAG_HAVE_POP3_ORDERS = 0x02
+};
+
 #endif  // SRC_STORAGE_RBOX_RBOX_STORAGE_H_