#142: new rmb unit test

- unit test: skip invalid rados object (no metadata and size < 0)
- moved metadata module creation to rmb-commands.

src/librmb/tools/rmb/rmb-commands.cpp

@@ -15,6 +15,11 @@
 #include "../../rados-cluster-impl.h"
 #include "../../rados-storage-impl.h"
 #include "rados-util.h"
+#include "rados-dovecot-config.h"
+#include "rados-dovecot-ceph-cfg-impl.h"
+#include "rados-namespace-manager.h"
+#include "rados-metadata-storage-ima.h"
+#include "rados-metadata-storage-default.h"
 
 namespace librmb {
 
@@ -68,8 +73,7 @@ int RmbCommands::delete_mail(bool confirmed) {
   return ret;
 }
 
-int RmbCommands::rename_user(librmb::RadosDovecotCephCfg *cfg, bool confirmed,
-                             const std::string &uid) {
+int RmbCommands::rename_user(librmb::RadosCephConfig *cfg, bool confirmed, const std::string &uid) {
   if (!cfg->is_user_mapping()) {
     std::cout << "Error: The configuration option generate_namespace needs to be active, to be able to rename a user"
               << std::endl;
@@ -117,20 +121,14 @@ int RmbCommands::rename_user(librmb::RadosDovecotCephCfg *cfg, bool confirmed,
     return ret;
 }
 
-int RmbCommands::configuration(bool create_config, const std::string &obj_, bool confirmed,
-                               librmb::RadosCephConfig &ceph_cfg) {
+int RmbCommands::configuration(bool confirmed, librmb::RadosCephConfig &ceph_cfg) {
   bool has_update = (*opts).find("update") != (*opts).end();
   bool has_ls = (*opts).find("print_cfg") != (*opts).end();
   if (has_update && has_ls) {
     std::cerr << "create and ls is not supported, use separately" << std::endl;
     return -1;
   }
 
-  if (create_config) {
-    std::cout << "Error: there already exists a configuration " << obj_ << std::endl;
-    return -1;
-  }
-
   if (has_ls) {
     std::cout << ceph_cfg.get_config()->to_string() << std::endl;
     return 0;
@@ -216,24 +214,28 @@ int RmbCommands::load_objects(librmb::RadosStorageMetadataModule *ms,
     if (ret != 0 || object_size <= 0) {
       std::cout << " object '" << oid << "' is not a valid mail object, size = 0" << std::endl;
       ++iter;
+      delete mail;
       continue;
     }
     mail->set_oid(oid);
     if (ms->load_metadata(mail) < 0) {
       std::cout << " loading metadata of object '" << oid << "' faild " << std::endl;
       ++iter;
+      delete mail;
       continue;
     }
 
     if (mail->get_metadata()->size() == 0) {
       std::cout << " pool object " << oid << " is not a mail object" << std::endl;
       ++iter;
+      delete mail;
       continue;
     }
 
     if (!librmb::RadosUtils::validate_metadata(mail->get_metadata())) {
       std::cout << "object : " << oid << " metadata is not valid " << std::endl;
       ++iter;
+      delete mail;
       continue;
     }
 
@@ -321,6 +323,45 @@ int RmbCommands::query_mail_storage(std::vector<librmb::RadosMailObject *> *mail
     delete it.second;
   }
   return ret;
+}
+
+RadosStorageMetadataModule *RmbCommands::init_metadata_storage_module(librmb::RadosCephConfig &ceph_cfg,
+                                                                      std::string *uid) {
+  librmb::RadosConfig dovecot_cfg;
+  RadosStorageMetadataModule *ms;
+  dovecot_cfg.set_config_valid(true);
+  ceph_cfg.set_config_valid(true);
+  librmb::RadosDovecotCephCfgImpl cfg(dovecot_cfg, ceph_cfg);
+  librmb::RadosNamespaceManager mgr(&cfg);
+
+  if (uid == nullptr) {
+    std::cerr << "please set valid uid ptr" << std::endl;
+    return nullptr;
   }
 
+  // decide metadata storage!
+  std::string storage_module_name = ceph_cfg.get_metadata_storage_module();
+  if (storage_module_name.compare(librmb::RadosMetadataStorageIma::module_name) == 0) {
+    ms = new librmb::RadosMetadataStorageIma(&storage->get_io_ctx(), &cfg);
+  } else {
+    ms = new librmb::RadosMetadataStorageDefault(&storage->get_io_ctx());
+  }
+
+  *uid = (*opts)["namespace"] + cfg.get_user_suffix();
+  std::cout << "uidl: " << *uid << std::endl;
+  std::string ns;
+  if (mgr.lookup_key(*uid, &ns)) {
+    storage->set_namespace(ns);
+  } else {
+    // use
+    if (!mgr.lookup_key(*uid, &ns)) {
+      std::cout << " error unable to determine namespace" << std::endl;
+      delete ms;
+      return nullptr;
+    }
+    storage->set_namespace(ns);
+  }
+  return ms;
+}
+
 } /* namespace librmb */

src/librmb/tools/rmb/rmb-commands.h

@@ -27,6 +27,8 @@
 #include "rados-ceph-config.h"
 #include "ls_cmd_parser.h"
 #include "mailbox_tools.h"
+#include "rados-metadata-storage-module.h"
+
 namespace librmb {
 
 class RmbCommands {
@@ -37,17 +39,17 @@ class RmbCommands {
   static int lspools();
   int delete_mail(bool confirmed);
 
-  int rename_user(librmb::RadosDovecotCephCfg *cfg, bool confirmed, const std::string &uid);
+  int rename_user(librmb::RadosCephConfig *cfg, bool confirmed, const std::string &uid);
 
-  int configuration(bool create_config, const std::string &obj_, bool confirmed, librmb::RadosCephConfig &ceph_cfg);
+  int configuration(bool confirmed, librmb::RadosCephConfig &ceph_cfg);
 
   int load_objects(librmb::RadosStorageMetadataModule *ms, std::vector<librmb::RadosMailObject *> &mail_objects,
                    std::string &sort_string);
 
   int print_mail(std::map<std::string, librmb::RadosMailBox *> *mailbox, std::string &output_dir, bool download);
   int query_mail_storage(std::vector<librmb::RadosMailObject *> *mail_objects, librmb::CmdLineParser *parser,
                          bool download);
-
+  librmb::RadosStorageMetadataModule *init_metadata_storage_module(librmb::RadosCephConfig &ceph_cfg, std::string *uid);
   static bool sort_uid(librmb::RadosMailObject *i, librmb::RadosMailObject *j);
   static bool sort_recv_date(librmb::RadosMailObject *i, librmb::RadosMailObject *j);
   static bool sort_phy_size(librmb::RadosMailObject *i, librmb::RadosMailObject *j);

src/librmb/tools/rmb/rmb.cpp

@@ -350,7 +350,7 @@ int main(int argc, const char **argv) {
   // connection to rados is established!
   rmb_commands = new librmb::RmbCommands(&storage, &cluster, &opts);
   if (is_config_option) {
-    if (rmb_commands->configuration(create_config, config_obj, confirmed, ceph_cfg) < 0) {
+    if (rmb_commands->configuration(confirmed, ceph_cfg) < 0) {
       std::cerr << "error processing config option" << std::endl;
     }
     delete rmb_commands;
@@ -359,48 +359,29 @@ int main(int argc, const char **argv) {
     exit(0);
   }
 
-  // load metadata configuration
-  librmb::RadosConfig dovecot_cfg;
-  dovecot_cfg.set_config_valid(true);
-  librmb::RadosDovecotCephCfgImpl cfg(dovecot_cfg, ceph_cfg);
-  librmb::RadosNamespaceManager mgr(&cfg);
-
-  librmb::RadosStorageMetadataModule *ms;
-  // decide metadata storage!
-  std::string storage_module_name = ceph_cfg.get_metadata_storage_module();
-  if (storage_module_name.compare(librmb::RadosMetadataStorageIma::module_name) == 0) {
-    ms = new librmb::RadosMetadataStorageIma(&storage.get_io_ctx(), &cfg);
-  } else {
-    ms = new librmb::RadosMetadataStorageDefault(&storage.get_io_ctx());
-  }
-
   // namespace (user) needs to be set
   if (opts.find("namespace") == opts.end()) {
     usage_exit();
   }
 
-  std::string uid(opts["namespace"] + cfg.get_user_suffix());
-  std::string ns;
-  if (mgr.lookup_key(uid, &ns)) {
-    storage.set_namespace(ns);
-  } else {
-    // use
-    if (!mgr.lookup_key(uid, &ns)) {
-      std::cout << " error unable to determine namespace" << std::endl;
-      cluster.deinit();
-      delete ms;
-      return -1;
-    }
-    storage.set_namespace(ns);
+  std::string uid;
+  // load metadata configuration
+  librmb::RadosStorageMetadataModule *ms = rmb_commands->init_metadata_storage_module(ceph_cfg, &uid);
+  if (ms == nullptr) {
+    /// error exit!
+    std::cerr << " Error initializing metadata module " << std::endl;
+    delete rmb_commands;
+    release_exit(&mail_objects, &cluster, false);
   }
 
   if (delete_mail_option) {
     if (rmb_commands->delete_mail(confirmed) < 0) {
       std::cerr << "error deleting mail" << std::endl;
       cluster.deinit();
     }
+
   } else if (rename_user_option) {
-    if (rmb_commands->rename_user(&cfg, confirmed, uid) < 0) {
+    if (rmb_commands->rename_user(&ceph_cfg, confirmed, uid) < 0) {
       std::cerr << "error renaming user" << std::endl;
     }
   } else if (opts.find("ls") != opts.end()) {

src/tests/Makefile.am

@@ -60,13 +60,13 @@ test_storage_mock_rbox_LDADD = $(storage_shlibs) $(gtest_shlibs)
 
 TESTS += test_librmb_utils
 test_librmb_utils_SOURCES = librmb/test_librmb_utils.cpp
-test_librmb_utils_LDADD = $(rmb_shlibs) $(top_builddir)/src/librmb/tools/rmb/ls_cmd_parser.o $(top_builddir)/src/librmb/tools/rmb/mailbox_tools.o $(gtest_shlibs)
+test_librmb_utils_LDADD = $(rmb_shlibs) $(top_builddir)/src/librmb/tools/rmb/ls_cmd_parser.o  $(top_builddir)/src/librmb/tools/rmb/mailbox_tools.o $(gtest_shlibs)
 
 if BUILD_INTEGRATION_TESTS
 
 TESTS += it_test_librmb
 it_test_librmb_SOURCES = librmb/it_test_librmb.cpp 
-it_test_librmb_LDADD = $(rmb_shlibs) $(top_builddir)/src/librmb/tools/rmb/ls_cmd_parser.o $(top_builddir)/src/librmb/tools/rmb/mailbox_tools.o $(gtest_shlibs)
+it_test_librmb_LDADD = $(rmb_shlibs) $(top_builddir)/src/librmb/tools/rmb/ls_cmd_parser.o  $(top_builddir)/src/librmb/tools/rmb/rmb-commands.o $(top_builddir)/src/librmb/tools/rmb/mailbox_tools.o $(gtest_shlibs)
 
 TESTS += it_test_dict_rados
 it_test_dict_rados_SOURCES = dict-rados/it_test_dict_rados.cpp dict-rados/TestCase.cpp dict-rados/TestCase.h  

src/tests/librmb/it_test_librmb.cpp

@@ -21,9 +21,7 @@
 #include "../../librmb/rados-metadata-storage-ima.h"
 #include "../../librmb/rados-dovecot-ceph-cfg-impl.h"
 #include "../../librmb/rados-util.h"
-
-//#include "common/Formatter.h"
-//#include "common/ceph_json.h"
+#include "../../librmb/tools/rmb/rmb-commands.h"
 
 using ::testing::AtLeast;
 using ::testing::Return;
@@ -831,6 +829,57 @@ TEST(librmb, increment_sub_from_existing_key) {
   cluster.deinit();
 }
 
+TEST(librmb, rmb_load_objects) {
+  librados::IoCtx io_ctx;
+
+  librmb::RadosClusterImpl cluster;
+  librmb::RadosStorageImpl storage(&cluster);
+
+  std::string pool_name("rmb_tool_tests");
+  std::string ns("t1");
+
+  int open_connection = storage.open_connection(pool_name);
+
+  EXPECT_EQ(0, open_connection);
+  librmb::RadosCephConfig ceph_cfg(&storage.get_io_ctx());
+  EXPECT_EQ(0, ceph_cfg.save_cfg());
+
+  std::map<std::string, std::string> opts;
+  opts["pool"] = pool_name;
+  opts["namespace"] = ns;
+  opts["print_cfg"] = "true";
+  opts["cfg_obj"] = ceph_cfg.get_cfg_object_name();
+
+  librmb::RmbCommands rmb_commands(&storage, &cluster, &opts);
+
+  /* update config
+  rmb_commands.configuration(false, ceph_cfg);
+  */
+  // load metadata info
+  std::string uid;
+  librmb::RadosStorageMetadataModule *ms = rmb_commands.init_metadata_storage_module(ceph_cfg, &uid);
+  EXPECT_NE(nullptr, ms);
+
+  storage.set_namespace(ns);
+  std::string key = "my-key";
+  librmb::RadosMailObject obj2;
+  obj2.set_oid("myobject");
+
+  ceph::bufferlist mail_buf;
+  storage.save_mail(obj2.get_oid(), mail_buf);
+
+  std::vector<librmb::RadosMailObject *> mail_objects;
+  std::string sort_string = "uid";
+
+  EXPECT_EQ(0, rmb_commands.load_objects(ms, mail_objects, sort_string));
+  EXPECT_EQ(0, mail_objects.size());
+
+  storage.delete_mail(&obj2);
+  storage.delete_mail(ceph_cfg.get_cfg_object_name());
+  delete ms;
+  // tear down
+  cluster.deinit();
+}
 
 TEST(librmb, mock_obj) {}
 int main(int argc, char **argv) {