From e081b774b2c3e7792900c29e29752b8b5dcf0743 Mon Sep 17 00:00:00 2001 From: Jan Radon Date: Wed, 23 May 2018 11:50:00 +0200 Subject: [PATCH] #142: new rmb unit test - unit test: skip invalid rados object (no metadata and size < 0) - moved metadata module creation to rmb-commands. --- src/librmb/tools/rmb/rmb-commands.cpp | 59 +++++++++++++++++++++++---- src/librmb/tools/rmb/rmb-commands.h | 8 ++-- src/librmb/tools/rmb/rmb.cpp | 41 +++++-------------- src/tests/Makefile.am | 4 +- src/tests/librmb/it_test_librmb.cpp | 55 +++++++++++++++++++++++-- 5 files changed, 120 insertions(+), 47 deletions(-) diff --git a/src/librmb/tools/rmb/rmb-commands.cpp b/src/librmb/tools/rmb/rmb-commands.cpp index 7aed79c4..236fb024 100644 --- a/src/librmb/tools/rmb/rmb-commands.cpp +++ b/src/librmb/tools/rmb/rmb-commands.cpp @@ -15,6 +15,11 @@ #include "../../rados-cluster-impl.h" #include "../../rados-storage-impl.h" #include "rados-util.h" +#include "rados-dovecot-config.h" +#include "rados-dovecot-ceph-cfg-impl.h" +#include "rados-namespace-manager.h" +#include "rados-metadata-storage-ima.h" +#include "rados-metadata-storage-default.h" namespace librmb { @@ -68,8 +73,7 @@ int RmbCommands::delete_mail(bool confirmed) { return ret; } -int RmbCommands::rename_user(librmb::RadosDovecotCephCfg *cfg, bool confirmed, - const std::string &uid) { +int RmbCommands::rename_user(librmb::RadosCephConfig *cfg, bool confirmed, const std::string &uid) { if (!cfg->is_user_mapping()) { std::cout << "Error: The configuration option generate_namespace needs to be active, to be able to rename a user" << std::endl; @@ -117,8 +121,7 @@ int RmbCommands::rename_user(librmb::RadosDovecotCephCfg *cfg, bool confirmed, return ret; } -int RmbCommands::configuration(bool create_config, const std::string &obj_, bool confirmed, - librmb::RadosCephConfig &ceph_cfg) { +int RmbCommands::configuration(bool confirmed, librmb::RadosCephConfig &ceph_cfg) { bool has_update = (*opts).find("update") != (*opts).end(); bool has_ls = (*opts).find("print_cfg") != (*opts).end(); if (has_update && has_ls) { @@ -126,11 +129,6 @@ int RmbCommands::configuration(bool create_config, const std::string &obj_, bool return -1; } - if (create_config) { - std::cout << "Error: there already exists a configuration " << obj_ << std::endl; - return -1; - } - if (has_ls) { std::cout << ceph_cfg.get_config()->to_string() << std::endl; return 0; @@ -216,24 +214,28 @@ int RmbCommands::load_objects(librmb::RadosStorageMetadataModule *ms, if (ret != 0 || object_size <= 0) { std::cout << " object '" << oid << "' is not a valid mail object, size = 0" << std::endl; ++iter; + delete mail; continue; } mail->set_oid(oid); if (ms->load_metadata(mail) < 0) { std::cout << " loading metadata of object '" << oid << "' faild " << std::endl; ++iter; + delete mail; continue; } if (mail->get_metadata()->size() == 0) { std::cout << " pool object " << oid << " is not a mail object" << std::endl; ++iter; + delete mail; continue; } if (!librmb::RadosUtils::validate_metadata(mail->get_metadata())) { std::cout << "object : " << oid << " metadata is not valid " << std::endl; ++iter; + delete mail; continue; } @@ -321,6 +323,45 @@ int RmbCommands::query_mail_storage(std::vector *mail delete it.second; } return ret; +} + +RadosStorageMetadataModule *RmbCommands::init_metadata_storage_module(librmb::RadosCephConfig &ceph_cfg, + std::string *uid) { + librmb::RadosConfig dovecot_cfg; + RadosStorageMetadataModule *ms; + dovecot_cfg.set_config_valid(true); + ceph_cfg.set_config_valid(true); + librmb::RadosDovecotCephCfgImpl cfg(dovecot_cfg, ceph_cfg); + librmb::RadosNamespaceManager mgr(&cfg); + + if (uid == nullptr) { + std::cerr << "please set valid uid ptr" << std::endl; + return nullptr; } + // decide metadata storage! + std::string storage_module_name = ceph_cfg.get_metadata_storage_module(); + if (storage_module_name.compare(librmb::RadosMetadataStorageIma::module_name) == 0) { + ms = new librmb::RadosMetadataStorageIma(&storage->get_io_ctx(), &cfg); + } else { + ms = new librmb::RadosMetadataStorageDefault(&storage->get_io_ctx()); + } + + *uid = (*opts)["namespace"] + cfg.get_user_suffix(); + std::cout << "uidl: " << *uid << std::endl; + std::string ns; + if (mgr.lookup_key(*uid, &ns)) { + storage->set_namespace(ns); + } else { + // use + if (!mgr.lookup_key(*uid, &ns)) { + std::cout << " error unable to determine namespace" << std::endl; + delete ms; + return nullptr; + } + storage->set_namespace(ns); + } + return ms; +} + } /* namespace librmb */ diff --git a/src/librmb/tools/rmb/rmb-commands.h b/src/librmb/tools/rmb/rmb-commands.h index 103d9880..46a0c135 100644 --- a/src/librmb/tools/rmb/rmb-commands.h +++ b/src/librmb/tools/rmb/rmb-commands.h @@ -27,6 +27,8 @@ #include "rados-ceph-config.h" #include "ls_cmd_parser.h" #include "mailbox_tools.h" +#include "rados-metadata-storage-module.h" + namespace librmb { class RmbCommands { @@ -37,9 +39,9 @@ class RmbCommands { static int lspools(); int delete_mail(bool confirmed); - int rename_user(librmb::RadosDovecotCephCfg *cfg, bool confirmed, const std::string &uid); + int rename_user(librmb::RadosCephConfig *cfg, bool confirmed, const std::string &uid); - int configuration(bool create_config, const std::string &obj_, bool confirmed, librmb::RadosCephConfig &ceph_cfg); + int configuration(bool confirmed, librmb::RadosCephConfig &ceph_cfg); int load_objects(librmb::RadosStorageMetadataModule *ms, std::vector &mail_objects, std::string &sort_string); @@ -47,7 +49,7 @@ class RmbCommands { int print_mail(std::map *mailbox, std::string &output_dir, bool download); int query_mail_storage(std::vector *mail_objects, librmb::CmdLineParser *parser, bool download); - + librmb::RadosStorageMetadataModule *init_metadata_storage_module(librmb::RadosCephConfig &ceph_cfg, std::string *uid); static bool sort_uid(librmb::RadosMailObject *i, librmb::RadosMailObject *j); static bool sort_recv_date(librmb::RadosMailObject *i, librmb::RadosMailObject *j); static bool sort_phy_size(librmb::RadosMailObject *i, librmb::RadosMailObject *j); diff --git a/src/librmb/tools/rmb/rmb.cpp b/src/librmb/tools/rmb/rmb.cpp index a56e00a8..c59b01a1 100644 --- a/src/librmb/tools/rmb/rmb.cpp +++ b/src/librmb/tools/rmb/rmb.cpp @@ -350,7 +350,7 @@ int main(int argc, const char **argv) { // connection to rados is established! rmb_commands = new librmb::RmbCommands(&storage, &cluster, &opts); if (is_config_option) { - if (rmb_commands->configuration(create_config, config_obj, confirmed, ceph_cfg) < 0) { + if (rmb_commands->configuration(confirmed, ceph_cfg) < 0) { std::cerr << "error processing config option" << std::endl; } delete rmb_commands; @@ -359,39 +359,19 @@ int main(int argc, const char **argv) { exit(0); } - // load metadata configuration - librmb::RadosConfig dovecot_cfg; - dovecot_cfg.set_config_valid(true); - librmb::RadosDovecotCephCfgImpl cfg(dovecot_cfg, ceph_cfg); - librmb::RadosNamespaceManager mgr(&cfg); - - librmb::RadosStorageMetadataModule *ms; - // decide metadata storage! - std::string storage_module_name = ceph_cfg.get_metadata_storage_module(); - if (storage_module_name.compare(librmb::RadosMetadataStorageIma::module_name) == 0) { - ms = new librmb::RadosMetadataStorageIma(&storage.get_io_ctx(), &cfg); - } else { - ms = new librmb::RadosMetadataStorageDefault(&storage.get_io_ctx()); - } - // namespace (user) needs to be set if (opts.find("namespace") == opts.end()) { usage_exit(); } - std::string uid(opts["namespace"] + cfg.get_user_suffix()); - std::string ns; - if (mgr.lookup_key(uid, &ns)) { - storage.set_namespace(ns); - } else { - // use - if (!mgr.lookup_key(uid, &ns)) { - std::cout << " error unable to determine namespace" << std::endl; - cluster.deinit(); - delete ms; - return -1; - } - storage.set_namespace(ns); + std::string uid; + // load metadata configuration + librmb::RadosStorageMetadataModule *ms = rmb_commands->init_metadata_storage_module(ceph_cfg, &uid); + if (ms == nullptr) { + /// error exit! + std::cerr << " Error initializing metadata module " << std::endl; + delete rmb_commands; + release_exit(&mail_objects, &cluster, false); } if (delete_mail_option) { @@ -399,8 +379,9 @@ int main(int argc, const char **argv) { std::cerr << "error deleting mail" << std::endl; cluster.deinit(); } + } else if (rename_user_option) { - if (rmb_commands->rename_user(&cfg, confirmed, uid) < 0) { + if (rmb_commands->rename_user(&ceph_cfg, confirmed, uid) < 0) { std::cerr << "error renaming user" << std::endl; } } else if (opts.find("ls") != opts.end()) { diff --git a/src/tests/Makefile.am b/src/tests/Makefile.am index 730cb497..99282066 100644 --- a/src/tests/Makefile.am +++ b/src/tests/Makefile.am @@ -60,13 +60,13 @@ test_storage_mock_rbox_LDADD = $(storage_shlibs) $(gtest_shlibs) TESTS += test_librmb_utils test_librmb_utils_SOURCES = librmb/test_librmb_utils.cpp -test_librmb_utils_LDADD = $(rmb_shlibs) $(top_builddir)/src/librmb/tools/rmb/ls_cmd_parser.o $(top_builddir)/src/librmb/tools/rmb/mailbox_tools.o $(gtest_shlibs) +test_librmb_utils_LDADD = $(rmb_shlibs) $(top_builddir)/src/librmb/tools/rmb/ls_cmd_parser.o $(top_builddir)/src/librmb/tools/rmb/mailbox_tools.o $(gtest_shlibs) if BUILD_INTEGRATION_TESTS TESTS += it_test_librmb it_test_librmb_SOURCES = librmb/it_test_librmb.cpp -it_test_librmb_LDADD = $(rmb_shlibs) $(top_builddir)/src/librmb/tools/rmb/ls_cmd_parser.o $(top_builddir)/src/librmb/tools/rmb/mailbox_tools.o $(gtest_shlibs) +it_test_librmb_LDADD = $(rmb_shlibs) $(top_builddir)/src/librmb/tools/rmb/ls_cmd_parser.o $(top_builddir)/src/librmb/tools/rmb/rmb-commands.o $(top_builddir)/src/librmb/tools/rmb/mailbox_tools.o $(gtest_shlibs) TESTS += it_test_dict_rados it_test_dict_rados_SOURCES = dict-rados/it_test_dict_rados.cpp dict-rados/TestCase.cpp dict-rados/TestCase.h diff --git a/src/tests/librmb/it_test_librmb.cpp b/src/tests/librmb/it_test_librmb.cpp index ce9930b7..9575e095 100644 --- a/src/tests/librmb/it_test_librmb.cpp +++ b/src/tests/librmb/it_test_librmb.cpp @@ -21,9 +21,7 @@ #include "../../librmb/rados-metadata-storage-ima.h" #include "../../librmb/rados-dovecot-ceph-cfg-impl.h" #include "../../librmb/rados-util.h" - -//#include "common/Formatter.h" -//#include "common/ceph_json.h" +#include "../../librmb/tools/rmb/rmb-commands.h" using ::testing::AtLeast; using ::testing::Return; @@ -831,6 +829,57 @@ TEST(librmb, increment_sub_from_existing_key) { cluster.deinit(); } +TEST(librmb, rmb_load_objects) { + librados::IoCtx io_ctx; + + librmb::RadosClusterImpl cluster; + librmb::RadosStorageImpl storage(&cluster); + + std::string pool_name("rmb_tool_tests"); + std::string ns("t1"); + + int open_connection = storage.open_connection(pool_name); + + EXPECT_EQ(0, open_connection); + librmb::RadosCephConfig ceph_cfg(&storage.get_io_ctx()); + EXPECT_EQ(0, ceph_cfg.save_cfg()); + + std::map opts; + opts["pool"] = pool_name; + opts["namespace"] = ns; + opts["print_cfg"] = "true"; + opts["cfg_obj"] = ceph_cfg.get_cfg_object_name(); + + librmb::RmbCommands rmb_commands(&storage, &cluster, &opts); + + /* update config + rmb_commands.configuration(false, ceph_cfg); + */ + // load metadata info + std::string uid; + librmb::RadosStorageMetadataModule *ms = rmb_commands.init_metadata_storage_module(ceph_cfg, &uid); + EXPECT_NE(nullptr, ms); + + storage.set_namespace(ns); + std::string key = "my-key"; + librmb::RadosMailObject obj2; + obj2.set_oid("myobject"); + + ceph::bufferlist mail_buf; + storage.save_mail(obj2.get_oid(), mail_buf); + + std::vector mail_objects; + std::string sort_string = "uid"; + + EXPECT_EQ(0, rmb_commands.load_objects(ms, mail_objects, sort_string)); + EXPECT_EQ(0, mail_objects.size()); + + storage.delete_mail(&obj2); + storage.delete_mail(ceph_cfg.get_cfg_object_name()); + delete ms; + // tear down + cluster.deinit(); +} TEST(librmb, mock_obj) {} int main(int argc, char **argv) {