diff --git a/src/librmb/rados-mail-object.cpp b/src/librmb/rados-mail-object.cpp index bd2cd27c..13c79e09 100644 --- a/src/librmb/rados-mail-object.cpp +++ b/src/librmb/rados-mail-object.cpp @@ -62,17 +62,22 @@ std::string RadosMailObject::to_string(const string &padding) { ostringstream ss; ss << endl; - ss << padding << "MAIL: " << static_cast(RBOX_METADATA_MAIL_UID) << "(uid)=" << uid << endl; - ss << padding << " " - << "oid = " << oid << endl; + ss << padding << "MAIL: "; + if (!uid.empty()) { + ss << static_cast(RBOX_METADATA_MAIL_UID) << "(uid)=" << uid << endl; + ss << padding << " "; + } + ss << "oid = " << oid << endl; string recv_time; if (RadosUtils::convert_time_t_to_str(ts, &recv_time) >= 0) { ss << padding << " " << static_cast(RBOX_METADATA_RECEIVED_TIME) << "(receive_time)=" << recv_time << "\n"; } else { - ss << padding << " " << static_cast(RBOX_METADATA_RECEIVED_TIME) << "(receive_time)= INVALID DATE : '" - << recv_time_str << "'" - << "\n"; + if (!recv_time_str.empty()) { + ss << padding << " " << static_cast(RBOX_METADATA_RECEIVED_TIME) + << "(receive_time)= INVALID DATE : '" << recv_time_str << "'" + << "\n"; + } } string save_time; if (RadosUtils::convert_time_t_to_str(save_date_rados, &save_time) >= 0) { @@ -82,18 +87,21 @@ std::string RadosMailObject::to_string(const string &padding) { ss << padding << " " << "save_time= INVALID DATE '" << save_date_rados << "'\n"; } + ss << padding << " " << static_cast(RBOX_METADATA_PHYSICAL_SIZE) << "(phy_size)=" << p_size << " " << static_cast(RBOX_METADATA_VIRTUAL_SIZE) << "(v_size) = " << v_size << " stat_size=" << object_size << endl; - ss << padding << " " << static_cast(RBOX_METADATA_MAILBOX_GUID) << "(mailbox_guid)=" << mailbox_guid - << endl; - + if (!mailbox_guid.empty()) { + ss << padding << " " << static_cast(RBOX_METADATA_MAILBOX_GUID) << "(mailbox_guid)=" << mailbox_guid + << endl; + } if (mb_orig_name.length() > 0) { ss << padding << " " << static_cast(RBOX_METADATA_ORIG_MAILBOX) << "(mailbox_orig_name)=" << mb_orig_name << endl; } - ss << padding << " " << static_cast(RBOX_METADATA_GUID) << "(mail_guid)=" << mail_guid << endl; - + if (!mail_guid.empty()) { + ss << padding << " " << static_cast(RBOX_METADATA_GUID) << "(mail_guid)=" << mail_guid << endl; + } if (rbox_version.length() > 0) { ss << padding << " " << static_cast(RBOX_METADATA_VERSION) << "(rbox_version): " << rbox_version << endl; diff --git a/src/librmb/tools/rmb/rmb-commands.cpp b/src/librmb/tools/rmb/rmb-commands.cpp index 385cf1b0..81d65366 100644 --- a/src/librmb/tools/rmb/rmb-commands.cpp +++ b/src/librmb/tools/rmb/rmb-commands.cpp @@ -12,6 +12,7 @@ #include "rmb-commands.h" #include // std::sort #include +#include #include "../../rados-cluster-impl.h" #include "../../rados-storage-impl.h" @@ -311,69 +312,105 @@ bool RmbCommands::sort_save_date(librmb::RadosMailObject *i, librmb::RadosMailOb return *i->get_rados_save_date() < *j->get_rados_save_date(); } +struct AioStat { + librmb::RadosMailObject *mail; + std::vector *mail_objects; + uint64_t object_size = 0; + time_t save_date_rados; + librados::AioCompletion *completion; +}; + +static void aio_cb(rados_completion_t cb, void *arg) { + AioStat *stat = static_cast(arg); + if (stat->completion->get_return_value() == 0) { + stat->mail->set_mail_size(stat->object_size); + stat->mail->set_rados_save_date(stat->save_date_rados); + stat->mail_objects->push_back(stat->mail); + } else { + std::cout << " object '" << stat->mail->get_oid() << "' is not a valid mail object, size = 0" << std::endl; + delete stat->mail; + } + delete stat; +} int RmbCommands::load_objects(librmb::RadosStorageMetadataModule *ms, - std::vector &mail_objects, std::string &sort_string) { + std::vector &mail_objects, std::string &sort_string, + bool load_metadata) { + time_t begin = time(NULL); + print_debug("entry: load_objects"); if (ms == nullptr || storage == nullptr) { print_debug("end: load_objects"); return -1; } - // get load all objects metadata into memory + std::list completions; + // load all objects metadata into memory librados::NObjectIterator iter(storage->find_mails(nullptr)); while (iter != librados::NObjectIterator::__EndObjectIterator) { librmb::RadosMailObject *mail = new librmb::RadosMailObject(); + AioStat *stat = new AioStat(); + stat->mail = mail; + stat->mail_objects = &mail_objects; + std::string oid = iter->get_oid(); - uint64_t object_size = 0; - time_t save_date_rados; - int ret = storage->stat_mail(oid, &object_size, &save_date_rados); - if (ret != 0 || object_size == 0) { + stat->completion = librados::Rados::aio_create_completion(static_cast(stat), aio_cb, NULL); + int ret = storage->get_io_ctx().aio_stat(oid, stat->completion, &stat->object_size, &stat->save_date_rados); + if (ret != 0) { std::cout << " object '" << oid << "' is not a valid mail object, size = 0" << std::endl; ++iter; delete mail; + delete stat; continue; } + completions.push_back(stat->completion); mail->set_oid(oid); - if (ms->load_metadata(mail) < 0) { - std::cout << " loading metadata of object '" << oid << "' faild " << std::endl; - ++iter; - delete mail; - continue; - } + if (load_metadata) { + if (ms->load_metadata(mail) < 0) { + std::cout << " loading metadata of object '" << oid << "' faild " << std::endl; + ++iter; + delete mail; + continue; + } - if (mail->get_metadata()->empty()) { - std::cout << " pool object " << oid << " is not a mail object" << std::endl; - ++iter; - delete mail; - continue; - } + if (mail->get_metadata()->empty()) { + std::cout << " pool object " << oid << " is not a mail object" << std::endl; + ++iter; + delete mail; + continue; + } - if (!librmb::RadosUtils::validate_metadata(mail->get_metadata())) { - std::cout << "object : " << oid << " metadata is not valid " << std::endl; - ++iter; - delete mail; - continue; + if (!librmb::RadosUtils::validate_metadata(mail->get_metadata())) { + std::cout << "object : " << oid << " metadata is not valid " << std::endl; + ++iter; + delete mail; + continue; + } } - mail->set_mail_size(object_size); - mail->set_rados_save_date(save_date_rados); ++iter; - mail_objects.push_back(mail); if (is_debug) { std::cout << "added: mail " << mail->get_oid() << std::endl; } } - if (sort_string.compare("uid") == 0) { - std::sort(mail_objects.begin(), mail_objects.end(), sort_uid); - } else if (sort_string.compare("recv_date") == 0) { - std::sort(mail_objects.begin(), mail_objects.end(), sort_recv_date); - } else if (sort_string.compare("phy_size") == 0) { - std::sort(mail_objects.begin(), mail_objects.end(), sort_phy_size); - } else { - std::sort(mail_objects.begin(), mail_objects.end(), sort_save_date); + for (std::list::iterator it = completions.begin(); it != completions.end(); ++it) { + (*it)->wait_for_complete_and_cb(); + (*it)->release(); + } + if (load_metadata) { + if (sort_string.compare("uid") == 0) { + std::sort(mail_objects.begin(), mail_objects.end(), sort_uid); + } else if (sort_string.compare("recv_date") == 0) { + std::sort(mail_objects.begin(), mail_objects.end(), sort_recv_date); + } else if (sort_string.compare("phy_size") == 0) { + std::sort(mail_objects.begin(), mail_objects.end(), sort_phy_size); + } else { + std::sort(mail_objects.begin(), mail_objects.end(), sort_save_date); + } } + time_t end = time(NULL); print_debug("end: load_objects"); + std::cout << " time elapsed loading objects: " << (end - begin) << std::endl; return 0; } diff --git a/src/librmb/tools/rmb/rmb-commands.h b/src/librmb/tools/rmb/rmb-commands.h index 34fcec75..cf1ea5e2 100644 --- a/src/librmb/tools/rmb/rmb-commands.h +++ b/src/librmb/tools/rmb/rmb-commands.h @@ -50,7 +50,7 @@ class RmbCommands { int configuration(bool confirmed, librmb::RadosCephConfig &ceph_cfg); int load_objects(librmb::RadosStorageMetadataModule *ms, std::vector &mail_objects, - std::string &sort_string); + std::string &sort_string, bool load_metadata = true); int update_attributes(librmb::RadosStorageMetadataModule *ms, std::map *metadata); int print_mail(std::map *mailbox, std::string &output_dir, bool download); int query_mail_storage(std::vector *mail_objects, librmb::CmdLineParser *parser, diff --git a/src/storage-rbox/doveadm-rbox-plugin.cpp b/src/storage-rbox/doveadm-rbox-plugin.cpp index b2a52be0..80da193a 100644 --- a/src/storage-rbox/doveadm-rbox-plugin.cpp +++ b/src/storage-rbox/doveadm-rbox-plugin.cpp @@ -128,7 +128,7 @@ static int cmd_rmb_config(std::map &opts, struct mail_ static int cmd_rmb_search_run(std::map &opts, struct mail_user *user, bool download, librmb::CmdLineParser &parser, std::vector &mail_objects, - bool silent) { + bool silent, bool load_metadata = true) { RboxDoveadmPlugin plugin; int open = open_connection_load_config(&plugin, user); if (open < 0) { @@ -149,7 +149,7 @@ static int cmd_rmb_search_run(std::map &opts, struct m return 0; } - int ret = rmb_cmds.load_objects(ms, mail_objects, opts["sort"]); + int ret = rmb_cmds.load_objects(ms, mail_objects, opts["sort"], load_metadata); if (ret < 0) { i_error("Error loading ceph objects. Errorcode: %d", ret); delete ms; @@ -673,7 +673,7 @@ static int cmd_rmb_check_indices_run(struct doveadm_mail_cmd_context *ctx, struc librmb::CmdLineParser parser(opts["ls"]); parser.parse_ls_string(); std::vector mail_objects; - ret = cmd_rmb_search_run(opts, user, false, parser, mail_objects, true); + ret = cmd_rmb_search_run(opts, user, false, parser, mail_objects, true, false); if (ret < 0) { return ret; }