From 3eafd0b38353bb27c4f0b7c3f08af7e3a1c8681a Mon Sep 17 00:00:00 2001 From: Jan Radon Date: Mon, 16 Jul 2018 11:38:41 +0200 Subject: [PATCH 1/2] #172: ls now shows unreferenced objects. --- src/librmb/rados-mail-object.cpp | 13 ++-- src/librmb/rados-mail-object.h | 5 +- src/librmb/tools/rmb/rmb.cpp | 3 + src/storage-rbox/doveadm-rbox-plugin.cpp | 85 +++++++++++++++++++++--- 4 files changed, 87 insertions(+), 19 deletions(-) diff --git a/src/librmb/rados-mail-object.cpp b/src/librmb/rados-mail-object.cpp index 36a1a5e2..2054652c 100644 --- a/src/librmb/rados-mail-object.cpp +++ b/src/librmb/rados-mail-object.cpp @@ -26,12 +26,9 @@ using librmb::RadosMailObject; const char RadosMailObject::X_ATTR_VERSION_VALUE[] = "0.1"; const char RadosMailObject::DATA_BUFFER_NAME[] = "RADOS_MAIL_BUFFER"; -RadosMailObject::RadosMailObject() { - this->object_size = -1; - this->active_op = false; - this->save_date_rados = -1; - this->valid = true; -} +RadosMailObject::RadosMailObject() + : object_size(-1), active_op(false), save_date_rados(-1), valid(true), index_ref(false) {} + RadosMailObject::~RadosMailObject() {} void RadosMailObject::set_guid(const uint8_t *_guid) { memcpy(this->guid, _guid, sizeof(this->guid)); } @@ -65,7 +62,9 @@ std::string RadosMailObject::to_string(const string &padding) { if (!valid) { ss << padding << "<<< MAIL OBJECT IS NOT VALID <<<<" << endl; } - + if (!index_ref) { + ss << padding << "<<< MAIL OBJECT HAS NO INDEX REFERENCE <<<<" << endl; + } ss << padding << "MAIL: "; if (!uid.empty()) { ss << static_cast(RBOX_METADATA_MAIL_UID) << "(uid)=" << uid << endl; diff --git a/src/librmb/rados-mail-object.h b/src/librmb/rados-mail-object.h index d0c53446..632977b1 100644 --- a/src/librmb/rados-mail-object.h +++ b/src/librmb/rados-mail-object.h @@ -65,6 +65,8 @@ string get_metadata(const string& key) { return value; } +bool is_index_ref() { return index_ref; } +void set_index_ref(bool ref) { this->index_ref = ref; } bool is_valid() { return valid; } void set_valid(bool valid_) { valid = valid_; } bool has_active_op() { return active_op; } @@ -95,10 +97,11 @@ const string get_extended_metadata(string& key) { map attrset; map extended_attrset; bool valid; - + bool index_ref; public: static const char X_ATTR_VERSION_VALUE[]; static const char DATA_BUFFER_NAME[]; + }; } // namespace librmb diff --git a/src/librmb/tools/rmb/rmb.cpp b/src/librmb/tools/rmb/rmb.cpp index 67b0c29d..73fd5233 100644 --- a/src/librmb/tools/rmb/rmb.cpp +++ b/src/librmb/tools/rmb/rmb.cpp @@ -417,6 +417,9 @@ int main(int argc, const char **argv) { if (opts["ls"].compare("all") == 0 || opts["ls"].compare("-") == 0 || parser.parse_ls_string()) { rmb_commands->load_objects(ms, mail_objects, sort_type); rmb_commands->query_mail_storage(&mail_objects, &parser, false, false); + std::cout << " NOTE: rmb tool does not have access to dovecot index. so all objects are set <<< MAIL OBJECT " + "HAS NO INDEX REFERENCE <<<< use doveadm rmb ls - instead " + << std::endl; } } else if (opts.find("get") != opts.end()) { librmb::CmdLineParser parser(opts["get"]); diff --git a/src/storage-rbox/doveadm-rbox-plugin.cpp b/src/storage-rbox/doveadm-rbox-plugin.cpp index 263b0571..213aa716 100644 --- a/src/storage-rbox/doveadm-rbox-plugin.cpp +++ b/src/storage-rbox/doveadm-rbox-plugin.cpp @@ -31,6 +31,8 @@ extern "C" { #include "istream.h" #include "doveadm-print.h" } +#include + #include "tools/rmb/rmb-commands.h" #include "rados-cluster.h" #include "rados-cluster-impl.h" @@ -43,9 +45,7 @@ extern "C" { #include "rbox-save.h" #include "rbox-storage.hpp" -#include - - +int check_namespace_mailboxes(struct mail_namespace *ns, std::vector &mail_objects); class RboxDoveadmPlugin { public: @@ -169,7 +169,7 @@ static int cmd_rmb_search_run(std::map &opts, struct m int open = open_connection_load_config(&plugin); if (open < 0) { i_error("Error opening rados connection. Errorcode: %d", open); - return 0; + return open; } opts["namespace"] = user->username; @@ -221,7 +221,65 @@ static int cmd_rmb_ls_run(struct doveadm_mail_cmd_context *ctx, struct mail_user if (opts["ls"].compare("all") == 0 || opts["ls"].compare("-") == 0 || parser.parse_ls_string()) { std::vector mail_objects; - ctx->exit_code = cmd_rmb_search_run(opts, user, false, parser, mail_objects, false); + + // open connection + RboxDoveadmPlugin plugin; + int open = open_connection_load_config(&plugin); + if (open < 0) { + i_error("Error opening rados connection. Errorcode: %d", open); + ctx->exit_code = open; + return 0; + } + + opts["namespace"] = user->username; + librmb::RmbCommands rmb_cmds(plugin.storage, plugin.cluster, &opts); + + std::string uid; + librmb::RadosCephConfig *cfg = + (static_cast(plugin.config))->get_rados_ceph_cfg(); + + librmb::RadosStorageMetadataModule *ms = rmb_cmds.init_metadata_storage_module(*cfg, &uid); + if (ms == nullptr) { + i_error(" Error initializing metadata module"); + delete ms; + ctx->exit_code = -1; + return 0; + } + + int ret = rmb_cmds.load_objects(ms, mail_objects, opts["sort"], true); + if (ret < 0) { + i_error("Error loading ceph objects. Errorcode: %d", ret); + delete ms; + ctx->exit_code = ret; + return 0; + } + + // ctx->exit_code = cmd_rmb_search_run(opts, user, false, parser, mail_objects, false); + + if (user->namespaces != NULL) { + struct mail_namespace *ns = mail_namespace_find_inbox(user->namespaces); + for (; ns != NULL; ns = ns->next) { + check_namespace_mailboxes(ns, mail_objects); + } + } + // print objects. + ret = rmb_cmds.query_mail_storage(&mail_objects, &parser, false, false); + if (ret < 0) { + i_error("Error query mail storage. Errorcode: %d", ret); + ctx->exit_code = ret; + delete ms; + return 0; + } + delete ms; + + // TODO: check for mails with + auto it_mail = std::find_if(mail_objects.begin(), mail_objects.end(), + [](librmb::RadosMailObject const *n) -> bool { return !n->is_index_ref(); }); + + if (it_mail != mail_objects.end()) { + std::cout << "There are unreference objects " << std::endl; + } + for (auto mo : mail_objects) { delete mo; } @@ -654,11 +712,12 @@ static int iterate_mailbox(struct mail_namespace *ns, const struct mailbox_info [oid](librmb::RadosMailObject *m) { return m->get_oid().compare(oid) == 0; }); if (it_mail == mail_objects.end()) { - std::cout << " missing mail object: uid=" << mail->uid << " guid=" << guid << " oid : " << oid - << " available: " << (it_mail != mail_objects.end()) << std::endl; + /* std::cout << " missing mail object: uid=" << mail->uid << " guid=" << guid << " oid : " << oid + << " available: " << (it_mail != mail_objects.end()) << std::endl;*/ ++mail_count_missing; } else { - mail_objects.erase(it_mail); // calls destructor of RadosMailObject* + (*it_mail)->set_index_ref(true); + // mail_objects.erase(it_mail); // calls destructor of RadosMailObject* } } if (mailbox_search_deinit(&search_ctx) < 0) { @@ -681,7 +740,7 @@ static int check_namespace_mailboxes(struct mail_namespace *ns, std::vectorlist, "*", static_cast( MAILBOX_LIST_ITER_RAW_LIST | MAILBOX_LIST_ITER_RETURN_NO_FLAGS)); while ((info = mailbox_list_iter_next(iter)) != NULL) { @@ -717,7 +776,11 @@ static int cmd_rmb_check_indices_run(struct doveadm_mail_cmd_context *ctx, struc check_namespace_mailboxes(ns, mail_objects); } - if (mail_objects.size() > 0) { + // TODO: check for mails with + auto it_mail = std::find_if(mail_objects.begin(), mail_objects.end(), + [](librmb::RadosMailObject *m) { return !m->is_index_ref(); }); + + if (it_mail != mail_objects.end()) { std::cout << std::endl << "There are mail objects without a index reference: " << std::endl; std::cout << "NOTE: you can fix(restore) the lost index entries by using doveadm force-resync or delete the " "unrefrenced objects from objectstore " @@ -750,7 +813,7 @@ static int cmd_rmb_check_indices_run(struct doveadm_mail_cmd_context *ctx, struc for (auto mo : mail_objects) { std::cout << mo->to_string(" ") << std::endl; - if (open >= 0 && ctx_->delete_not_referenced_objects) { + if (open >= 0 && ctx_->delete_not_referenced_objects && !mo->is_index_ref()) { std::cout << "mail object: " << mo->get_oid().c_str() << " deleted: " << (plugin.storage->delete_mail(mo) < 0 ? " FALSE " : " TRUE") << std::endl; ctx->exit_code = 2; From 0229d3d17f81cfc5f79468044a7d53fc4ca5b6af Mon Sep 17 00:00:00 2001 From: Jan Radon Date: Mon, 16 Jul 2018 12:17:56 +0200 Subject: [PATCH 2/2] #172: print message in ls mb in case mailbox has unreferenced messages. --- src/storage-rbox/doveadm-rbox-plugin.cpp | 62 +++--------------------- 1 file changed, 8 insertions(+), 54 deletions(-) diff --git a/src/storage-rbox/doveadm-rbox-plugin.cpp b/src/storage-rbox/doveadm-rbox-plugin.cpp index 213aa716..b6f00ed6 100644 --- a/src/storage-rbox/doveadm-rbox-plugin.cpp +++ b/src/storage-rbox/doveadm-rbox-plugin.cpp @@ -191,7 +191,12 @@ static int cmd_rmb_search_run(std::map &opts, struct m delete ms; return ret; } - + if (user->namespaces != NULL) { + struct mail_namespace *ns = mail_namespace_find_inbox(user->namespaces); + for (; ns != NULL; ns = ns->next) { + check_namespace_mailboxes(ns, mail_objects); + } + } if (download) { rmb_cmds.set_output_path(&parser); } @@ -222,57 +227,9 @@ static int cmd_rmb_ls_run(struct doveadm_mail_cmd_context *ctx, struct mail_user if (opts["ls"].compare("all") == 0 || opts["ls"].compare("-") == 0 || parser.parse_ls_string()) { std::vector mail_objects; - // open connection - RboxDoveadmPlugin plugin; - int open = open_connection_load_config(&plugin); - if (open < 0) { - i_error("Error opening rados connection. Errorcode: %d", open); - ctx->exit_code = open; - return 0; - } - - opts["namespace"] = user->username; - librmb::RmbCommands rmb_cmds(plugin.storage, plugin.cluster, &opts); - - std::string uid; - librmb::RadosCephConfig *cfg = - (static_cast(plugin.config))->get_rados_ceph_cfg(); - - librmb::RadosStorageMetadataModule *ms = rmb_cmds.init_metadata_storage_module(*cfg, &uid); - if (ms == nullptr) { - i_error(" Error initializing metadata module"); - delete ms; - ctx->exit_code = -1; - return 0; - } - - int ret = rmb_cmds.load_objects(ms, mail_objects, opts["sort"], true); - if (ret < 0) { - i_error("Error loading ceph objects. Errorcode: %d", ret); - delete ms; - ctx->exit_code = ret; - return 0; - } - - // ctx->exit_code = cmd_rmb_search_run(opts, user, false, parser, mail_objects, false); - - if (user->namespaces != NULL) { - struct mail_namespace *ns = mail_namespace_find_inbox(user->namespaces); - for (; ns != NULL; ns = ns->next) { - check_namespace_mailboxes(ns, mail_objects); - } - } - // print objects. - ret = rmb_cmds.query_mail_storage(&mail_objects, &parser, false, false); - if (ret < 0) { - i_error("Error query mail storage. Errorcode: %d", ret); - ctx->exit_code = ret; - delete ms; - return 0; - } - delete ms; + ctx->exit_code = cmd_rmb_search_run(opts, user, false, parser, mail_objects, false); - // TODO: check for mails with + // TODO: check for mails without reference auto it_mail = std::find_if(mail_objects.begin(), mail_objects.end(), [](librmb::RadosMailObject const *n) -> bool { return !n->is_index_ref(); }); @@ -298,9 +255,6 @@ static int cmd_rmb_ls_mb_run(struct doveadm_mail_cmd_context *ctx, struct mail_u if (opts["ls"].compare("all") == 0 || opts["ls"].compare("-") == 0 || parser.parse_ls_string()) { std::vector mail_objects; ctx->exit_code = cmd_rmb_search_run(opts, user, false, parser, mail_objects, false); - for (auto mo : mail_objects) { - delete mo; - } } else { i_error("invalid ls search query"); ctx->exit_code = -1;