From eec8ed59bc9ea081cf36c8a17a455f3d6fe8c829 Mon Sep 17 00:00:00 2001 From: Jan Radon Date: Fri, 11 Jan 2019 12:51:46 +0100 Subject: [PATCH] #230: pop3_uidl (mdbox)implementation --- src/storage-rbox/rbox-mail.cpp | 24 +++++++------ src/storage-rbox/rbox-save.cpp | 16 +++++++-- src/storage-rbox/rbox-save.h | 8 +++-- src/storage-rbox/rbox-storage.cpp | 59 ++++++++++++++++++------------- src/storage-rbox/rbox-storage.h | 18 ++++++++++ 5 files changed, 87 insertions(+), 38 deletions(-) diff --git a/src/storage-rbox/rbox-mail.cpp b/src/storage-rbox/rbox-mail.cpp index 5ca58ea3..0d10fec5 100644 --- a/src/storage-rbox/rbox-mail.cpp +++ b/src/storage-rbox/rbox-mail.cpp @@ -32,6 +32,7 @@ extern "C" { #include "debug-helper.h" #include "limits.h" #include "macros.h" +#include "index-pop3-uidl.h" } #include "../librmb/rados-mail.h" @@ -487,6 +488,7 @@ static int rbox_get_cached_metadata(struct rbox_mail *mail, enum rbox_metadata_k static int rbox_mail_get_special(struct mail *_mail, enum mail_fetch_field field, const char **value_r) { struct rbox_mail *mail = (struct rbox_mail *)_mail; + struct rbox_mailbox *mbox = (struct rbox_mailbox *)_mail->box; int ret = 0; *value_r = NULL; @@ -497,30 +499,32 @@ static int rbox_mail_get_special(struct mail *_mail, enum mail_fetch_field field case MAIL_FETCH_GUID: return rbox_get_cached_metadata(mail, rbox_metadata_key::RBOX_METADATA_GUID, MAIL_CACHE_GUID, value_r); case MAIL_FETCH_UIDL_BACKEND: -#ifdef DOVECOT_CEPH_PLUGINS_HAVE_INDEX_POP3_UIDL_H + if (!rbox_header_have_flag(_mail->box, mbox->hdr_ext_id, offsetof(struct rbox_index_header, flags), + RBOX_INDEX_HEADER_FLAG_HAVE_POP3_UIDLS)) { + *value_r = ""; + return 0; + } if (!index_pop3_uidl_can_exist(_mail)) { *value_r = ""; return 0; } -#endif - ret = rbox_get_cached_metadata(mail, rbox_metadata_key::RBOX_METADATA_POP3_UIDL, MAIL_CACHE_POP3_UIDL, value_r); - -#ifdef DOVECOT_CEPH_PLUGINS_HAVE_INDEX_POP3_UIDL_H if (ret == 0) { index_pop3_uidl_update_exists(&mail->imail.mail.mail, (*value_r)[0] != '\0'); } -#endif return ret; case MAIL_FETCH_POP3_ORDER: -#ifdef DOVECOT_CEPH_PLUGINS_HAVE_INDEX_POP3_UIDL_H + if (!rbox_header_have_flag(_mail->box, mbox->hdr_ext_id, offsetof(struct rbox_index_header, flags), + RBOX_INDEX_HEADER_FLAG_HAVE_POP3_ORDERS)) { + *value_r = ""; + return 0; + } if (!index_pop3_uidl_can_exist(_mail)) { /* we're assuming that if there's a POP3 order, there's also a UIDL */ *value_r = ""; return 0; } -#endif return rbox_get_cached_metadata(mail, rbox_metadata_key::RBOX_METADATA_POP3_ORDER, MAIL_CACHE_POP3_ORDER, value_r); @@ -578,7 +582,7 @@ static void rbox_index_mail_set_seq(struct mail *_mail, uint32_t seq, bool savin rbox_get_index_record(_mail); } } - +/*static void rbox_update_pop3_uidl(struct mail *_mail, const char *uidl) { i_debug("UIDL: %s", uidl); }*/ /*ebd if old version */ // rbox_mail_free, struct mail_vfuncs rbox_mail_vfuncs = {rbox_mail_close, @@ -616,7 +620,7 @@ struct mail_vfuncs rbox_mail_vfuncs = {rbox_mail_close, index_mail_update_keywords, index_mail_update_modseq, index_mail_update_pvt_modseq, - NULL, + NULL /*rbox_update_pop3_uidl*/, index_mail_expunge, index_mail_set_cache_corrupted, index_mail_opened, diff --git a/src/storage-rbox/rbox-save.cpp b/src/storage-rbox/rbox-save.cpp index f2aab25b..2dcbb126 100644 --- a/src/storage-rbox/rbox-save.cpp +++ b/src/storage-rbox/rbox-save.cpp @@ -30,6 +30,7 @@ extern "C" { #include "rbox-sync.h" #include "debug-helper.h" +#include "index-pop3-uidl.h" } #include "../librmb/rados-mail.h" @@ -41,9 +42,9 @@ extern "C" { using ceph::bufferlist; -using librmb::RadosStorage; using librmb::RadosMail; using librmb::RadosMetadata; +using librmb::RadosStorage; using librmb::rbox_metadata_key; using std::string; @@ -287,12 +288,16 @@ static int rbox_save_mail_set_metadata(struct rbox_save_context *r_ctx, librmb:: if (mdata->pop3_uidl != NULL) { RadosMetadata xattr(rbox_metadata_key::RBOX_METADATA_POP3_UIDL, mdata->pop3_uidl); mail_object->add_metadata(xattr); + r_ctx->have_pop3_uidls = TRUE; + r_ctx->highest_pop3_uidl_seq = I_MAX(r_ctx->highest_pop3_uidl_seq, r_ctx->seq); } } if (r_storage->config->is_mail_attribute(rbox_metadata_key::RBOX_METADATA_POP3_ORDER)) { if (mdata->pop3_order != 0) { RadosMetadata xattr(rbox_metadata_key::RBOX_METADATA_POP3_ORDER, mdata->pop3_order); mail_object->add_metadata(xattr); + r_ctx->have_pop3_orders = TRUE; + r_ctx->highest_pop3_uidl_seq = I_MAX(r_ctx->highest_pop3_uidl_seq, r_ctx->seq); } } if (r_storage->config->is_mail_attribute(rbox_metadata_key::RBOX_METADATA_FROM_ENVELOPE)) { @@ -423,7 +428,7 @@ int rbox_save_finish(struct mail_save_context *_ctx) { uint32_t t = _ctx->data.save_date; index_mail_cache_add((struct index_mail *)_ctx->dest_mail, MAIL_CACHE_SAVE_DATE, &t, sizeof(t)); } - + #if DOVECOT_PREREQ(2, 3) int ret = 0; if (r_ctx->ctx.data.output != r_ctx->output_stream) { @@ -522,6 +527,9 @@ static int rbox_save_assign_uids(struct rbox_save_context *r_ctx, const ARRAY_TY return -1; } } + if (r_ctx->highest_pop3_uidl_seq == n + 1) { + index_pop3_uidl_set_max_uid(&r_ctx->mbox->box, r_ctx->trans, uid); + } } i_assert(!seq_range_array_iter_nth(&iter, n, &uid)); } @@ -547,6 +555,10 @@ void rbox_save_update_header_flags(struct rbox_save_context *r_ctx, struct mail_ } flags = old_flags; + if (r_ctx->have_pop3_uidls) + flags |= RBOX_INDEX_HEADER_FLAG_HAVE_POP3_UIDLS; + if (r_ctx->have_pop3_orders) + flags |= RBOX_INDEX_HEADER_FLAG_HAVE_POP3_ORDERS; if (flags != old_flags) { /* flags changed, update them */ mail_index_update_header_ext(r_ctx->trans, ext_id, flags_offset, &flags, 1); diff --git a/src/storage-rbox/rbox-save.h b/src/storage-rbox/rbox-save.h index d71b9f21..8c4b8dc2 100644 --- a/src/storage-rbox/rbox-save.h +++ b/src/storage-rbox/rbox-save.h @@ -42,7 +42,9 @@ class rbox_save_context { failed(1), finished(1), copying(0), - dest_mail_allocated(0) {} + dest_mail_allocated(0), + highest_pop3_uidl_seq(0), + have_pop3_uidls(0) {} /** dovecot mail save context **/ struct mail_save_context ctx; @@ -69,7 +71,9 @@ class rbox_save_context { std::vector rados_mails; /** current mail in the context **/ librmb::RadosMail *rados_mail; - + unsigned int highest_pop3_uidl_seq : 1; + unsigned int have_pop3_uidls : 1; + unsigned int have_pop3_orders : 1; unsigned int failed : 1; unsigned int finished : 1; unsigned int copying : 1; diff --git a/src/storage-rbox/rbox-storage.cpp b/src/storage-rbox/rbox-storage.cpp index 6860ec95..a7b238aa 100644 --- a/src/storage-rbox/rbox-storage.cpp +++ b/src/storage-rbox/rbox-storage.cpp @@ -26,6 +26,7 @@ extern "C" { #include "guid.h" #include "mailbox-list-fs.h" #include "macros.h" +#include "index-pop3-uidl.h" } #include "rbox-mailbox-list-fs.h" @@ -422,26 +423,26 @@ int rbox_open_rados_connection(struct mailbox *box, bool alt_storage) { // initialize storage with plugin configuration read_plugin_configuration(box); int ret = 0; -try{ - rados_storage->set_ceph_wait_method(rbox->storage->config->is_ceph_aio_wait_for_safe_and_cb() - ? librmb::WAIT_FOR_SAFE_AND_CB - : librmb::WAIT_FOR_COMPLETE_AND_CB); - /* open connection to primary and alternative storage */ - ret = rados_storage->open_connection(rbox->storage->config->get_pool_name(), - rbox->storage->config->get_rados_cluster_name(), - rbox->storage->config->get_rados_username()); - - if (alt_storage) { - ret = rbox->storage->alt->open_connection(box->list->set.alt_dir, rbox->storage->config->get_rados_cluster_name(), - rbox->storage->config->get_rados_username()); - - rbox->storage->alt->set_ceph_wait_method(rbox->storage->config->is_ceph_aio_wait_for_safe_and_cb() - ? librmb::WAIT_FOR_SAFE_AND_CB - : librmb::WAIT_FOR_COMPLETE_AND_CB); - } -}catch(std::exception& e){ - ret= -1; -} + try { + rados_storage->set_ceph_wait_method(rbox->storage->config->is_ceph_aio_wait_for_safe_and_cb() + ? librmb::WAIT_FOR_SAFE_AND_CB + : librmb::WAIT_FOR_COMPLETE_AND_CB); + /* open connection to primary and alternative storage */ + ret = rados_storage->open_connection(rbox->storage->config->get_pool_name(), + rbox->storage->config->get_rados_cluster_name(), + rbox->storage->config->get_rados_username()); + + if (alt_storage) { + ret = rbox->storage->alt->open_connection(box->list->set.alt_dir, rbox->storage->config->get_rados_cluster_name(), + rbox->storage->config->get_rados_username()); + + rbox->storage->alt->set_ceph_wait_method(rbox->storage->config->is_ceph_aio_wait_for_safe_and_cb() + ? librmb::WAIT_FOR_SAFE_AND_CB + : librmb::WAIT_FOR_COMPLETE_AND_CB); + } + } catch (std::exception &e) { + ret = -1; + } if (ret == 1) { // already connected nothing to do! @@ -582,13 +583,11 @@ int rbox_mailbox_create_indexes(struct mailbox *box, const struct mailbox_update } mail_index_view_close(&view); -#ifdef DOVECOT_CEPH_PLUGINS_HAVE_INDEX_POP3_UIDL_H if (box->inbox_user && box->creating) { /* initialize pop3-uidl header when creating mailbox (not on mailbox_update()) */ index_pop3_uidl_set_max_uid(box, trans, 0); } -#endif rbox_update_header((struct rbox_mailbox *)box, trans, update); if (trans != NULL) { @@ -853,8 +852,9 @@ int check_users_mailbox_delete_ns_object(struct mail_user *user, librmb::RadosDo for (; ns != NULL; ns = ns->next) { struct mailbox_list_iterate_context *iter; const struct mailbox_info *info; - iter = mailbox_list_iter_init(ns->list, "*", static_cast( - MAILBOX_LIST_ITER_RAW_LIST | MAILBOX_LIST_ITER_RETURN_NO_FLAGS)); + iter = mailbox_list_iter_init( + ns->list, "*", + static_cast(MAILBOX_LIST_ITER_RAW_LIST | MAILBOX_LIST_ITER_RETURN_NO_FLAGS)); int total_mails = 0; while ((info = mailbox_list_iter_next(iter)) != NULL) { @@ -937,6 +937,17 @@ int rbox_storage_mailbox_delete(struct mailbox *box) { return ret; } +bool rbox_header_have_flag(struct mailbox *box, uint32_t ext_id, unsigned int flags_offset, uint8_t flag) { + const void *data; + size_t data_size; + uint8_t flags = 0; + + mail_index_get_header_ext(box->view, ext_id, &data, &data_size); + if (flags_offset < data_size) + flags = *((const uint8_t *)data + flags_offset); + return (flags & flag) != 0; +} + struct mailbox_vfuncs rbox_mailbox_vfuncs = {index_storage_is_readonly, index_storage_mailbox_enable, index_storage_mailbox_exists, diff --git a/src/storage-rbox/rbox-storage.h b/src/storage-rbox/rbox-storage.h index 61a1c2db..b44a97de 100644 --- a/src/storage-rbox/rbox-storage.h +++ b/src/storage-rbox/rbox-storage.h @@ -63,6 +63,17 @@ extern void read_plugin_configuration(struct mailbox *box); */ extern int rbox_storage_mailbox_delete(struct mailbox *box); +/** + * check for flag + * + * @param box mailbox + * @param ext_id header extension id + * @param flags_offset flags offset + * @param flag flag + * @return + */ +extern bool rbox_header_have_flag(struct mailbox *box, uint32_t ext_id, unsigned int flags_offset, uint8_t flag); + #ifdef __cplusplus } #endif @@ -95,4 +106,11 @@ struct rbox_mailbox { ARRAY(struct expunged_item *) moved_items; }; +enum rbox_index_header_flags { + /* messages' metadata contain POP3 UIDLs */ + RBOX_INDEX_HEADER_FLAG_HAVE_POP3_UIDLS = 0x01, + /* messages' metadata contain POP3 orders */ + RBOX_INDEX_HEADER_FLAG_HAVE_POP3_ORDERS = 0x02 +}; + #endif // SRC_STORAGE_RBOX_RBOX_STORAGE_H_