#148: new rmb tool option -r <logfile>

- new option -r allows the deletion of ALL objects in the given logfile!
- added unit tests to eval logfile read and deletion.

src/librmb/rados-save-log.h

@@ -14,6 +14,8 @@
 
 #include <fstream>  // std::ofstream
 #include <regex>
+#include <cstdio>
+
 namespace librmb {
 
 class RadosSaveLogEntry {
@@ -39,13 +41,13 @@ class RadosSaveLogEntry {
     std::vector<std::string> csv_items{std::sregex_token_iterator(item.begin(), item.end(), re, 1),
                                        std::sregex_token_iterator()};
     // read obj from stream
-    if (csv_items.size() < 4) {
-      is.setstate(std::ios::failbit);
-    } else {
+    if (csv_items.size() == 5) {
       obj.op = csv_items[0];
       obj.pool = csv_items[1];
       obj.ns = csv_items[2];
       obj.oid = csv_items[3];
+    } else {
+      is.setstate(std::ios::failbit);
     }
     return is;
   }

src/librmb/tools/rmb/rmb-commands.cpp

@@ -11,6 +11,7 @@
 
 #include "rmb-commands.h"
 #include <algorithm>  // std::sort
+#include <cstdio>
 
 #include "../../rados-cluster-impl.h"
 #include "../../rados-storage-impl.h"
@@ -20,6 +21,7 @@
 #include "rados-namespace-manager.h"
 #include "rados-metadata-storage-ima.h"
 #include "rados-metadata-storage-default.h"
+#include "rados-save-log.h"
 
 namespace librmb {
 
@@ -34,6 +36,57 @@ RmbCommands::RmbCommands(librmb::RadosStorage *storage_, librmb::RadosCluster *c
 RmbCommands::~RmbCommands() {
 }
 
+int RmbCommands::delete_with_save_log(const std::string &save_log, const std::string &rados_cluster,
+                                      const std::string &rados_user) {
+  librmb::RadosClusterImpl cluster;
+  librmb::RadosStorageImpl storage(&cluster);
+  int count = 0;
+
+  /** check content **/
+  std::ifstream read(save_log);
+  if (!read.is_open()) {
+    std::cerr << " path to log file not valid " << std::endl;
+    return -1;
+  }
+  int line_count = 0;
+  while (true) {
+    line_count++;
+    librmb::RadosSaveLogEntry entry;
+    read >> entry;
+    if (read.eof()) {
+      break;
+    }
+    if (read.fail()) {
+      std::cout << "Objectentry at line '" << line_count << "' is not valid: " << std::endl;
+      break;
+    }
+    std::cout << " size : " << entry.size << std::endl;
+    if (storage.get_pool_name().compare(entry.pool) != 0) {
+      // close connection before open a new one.
+      // TODO: worst case are alternating pool entries e.g. mail_storage ,
+      //       mail_storage_alt.... maybe we should group the entries by pool...
+      storage.close_connection();
+      int open_connection = storage.open_connection(entry.pool, rados_cluster, rados_user);
+      if (open_connection < 0) {
+        std::cerr << " error opening rados connection. Errorcode: " << open_connection << std::endl;
+        cluster.deinit();
+        return -1;
+      }
+    }
+    storage.set_namespace(entry.ns);
+    int ret_delete = storage.delete_mail(entry.oid);
+    if (ret_delete < 0) {
+      std::cout << "Object " << entry.oid << " not deleted: errorcode: " << ret_delete << std::endl;
+    } else {
+      std::cout << "Object " << entry.oid << " successfully deleted" << std::endl;
+      count++;
+    }
+  }
+  read.close();
+  storage.close_connection();
+  cluster.deinit();
+  return count;
+}
 // TODO:: currently untestable with mocks.
 int RmbCommands::lspools() {
   librmb::RadosClusterImpl cluster;

src/librmb/tools/rmb/rmb-commands.h

@@ -36,6 +36,10 @@ class RmbCommands {
   RmbCommands(librmb::RadosStorage *storage_, librmb::RadosCluster *cluster_,
               std::map<std::string, std::string> *opts_);
   virtual ~RmbCommands();
+
+  static int delete_with_save_log(const std::string &save_log, const std::string &rados_cluster,
+                                  const std::string &rados_user);
+
   static int lspools();
   int delete_mail(bool confirmed);
 

src/librmb/tools/rmb/rmb.cpp

@@ -159,6 +159,8 @@ static void usage(std::ostream &out) {
          "   -c    rados cluster name, default: 'ceph'\n"
          "   -u    rados user name, default: 'client.admin' \n"
          "   -D    debug output \n"
+         "   -r    save log with objects to delete => deletes all entries (save,mv,cp) from object store, use with "
+         "care!!!! \n "
          "\n"
          "\nMAIL COMMANDS\n"
          "    ls -    list all mails and mailbox statistic\n"
@@ -232,6 +234,8 @@ static void parse_cmd_line_args(std::map<std::string, std::string> *opts, bool &
       (*opts)["rados_user"] = val;
     } else if (ceph_argparse_witharg(args, &i, &val, "-D", "--debug", static_cast<char>(NULL))) {
       (*opts)["debug"] = val;
+    } else if (ceph_argparse_witharg(args, &i, &val, "-r", "--remove", static_cast<char>(NULL))) {
+      (*opts)["remove_save_log"] = val;
     } else if (ceph_argparse_witharg(args, &i, &val, "ls", "--ls", static_cast<char>(NULL))) {
       (*opts)["ls"] = val;
     } else if (ceph_argparse_witharg(args, &i, &val, "get", "--get", static_cast<char>(NULL))) {
@@ -285,6 +289,7 @@ int main(int argc, const char **argv) {
   bool is_lspools_cmd = false;
   bool delete_mail_option = false;
   bool rename_user_option = false;
+  std::string remove_save_log;
   std::string config_obj = "obj";
   std::string rados_user = "client.admin";
   std::string rados_cluster;
@@ -307,6 +312,20 @@ int main(int argc, const char **argv) {
   rename_user_option = opts.find("to_rename") != opts.end() ? true : false;
   rados_cluster = (opts.find("clustername") != opts.end()) ? opts["clustername"] : "ceph";
   rados_user = (opts.find("rados_user") != opts.end()) ? opts["rados_user"] : "client.admin";
+  remove_save_log = (opts.find("remove_save_log") != opts.end()) ? opts["remove_save_log"] : "";
+
+  if (!remove_save_log.empty()) {
+    if (confirmed) {
+      return librmb::RmbCommands::delete_with_save_log(remove_save_log, rados_cluster, rados_user);
+    } else {
+      std::cout << "WARNING:" << std::endl;
+      std::cout << "Performing this command, will delete all mail objects from ceph object store which are "
+                   "listed in the log file. This operation is Irreversible!!!!!!! Are you sure you want to do this?"
+                << std::endl;
+      std::cout << "To confirm pass --yes-i-really-really-mean-it " << std::endl;
+      return 0;
+    }
+  }
   // set pool to default or given pool name
   std::string pool_name(opts.find("pool") == opts.end() ? "mail_storage" : opts["pool"]);
 

src/tests/librmb/it_test_librmb.cpp

@@ -22,6 +22,7 @@
 #include "../../librmb/rados-dovecot-ceph-cfg-impl.h"
 #include "../../librmb/rados-util.h"
 #include "../../librmb/tools/rmb/rmb-commands.h"
+#include "../../librmb/rados-save-log.h"
 
 using ::testing::AtLeast;
 using ::testing::Return;
@@ -1161,6 +1162,84 @@ TEST(librmb, rmb_load_objects_invalid_metadata) {
   // tear down
   cluster.deinit();
 }
+
+TEST(librmb, delete_objects_via_rmb_tool_and_save_log_file) {
+
+  librmb::RadosClusterImpl cluster;
+  librmb::RadosStorageImpl storage(&cluster);
+
+  std::string pool_name("rmb_tool_tests");
+  std::string ns("t1");
+
+  int open_connection = storage.open_connection(pool_name);
+  EXPECT_EQ(0, open_connection);
+  storage.set_namespace(ns);
+
+  std::string test_file_name = "test1.log";
+  librmb::RadosSaveLog log_file(test_file_name);
+  EXPECT_EQ(true, log_file.open());
+  log_file.append(librmb::RadosSaveLogEntry("abc", "t1", "rmb_tool_tests", "save"));
+  EXPECT_EQ(true, log_file.close());
+  librados::bufferlist bl;
+  EXPECT_EQ(0, storage.save_mail("abc", bl));
+  cluster.deinit();
+
+  EXPECT_EQ(1, librmb::RmbCommands::delete_with_save_log("test1.log", "ceph", "client.admin"));
+  std::remove(test_file_name.c_str());
+}
+TEST(librmb, delete_objects_via_rmb_tool_and_save_log_file_file_not_found) {
+
+  std::string pool_name("rmb_tool_tests");
+  std::string ns("t1");
+
+  std::string test_file_name = "test1.log";
+  librmb::RadosSaveLog log_file(test_file_name);
+  EXPECT_EQ(true, log_file.open());
+  log_file.append(librmb::RadosSaveLogEntry("abc", "t1", "rmb_tool_tests", "save"));
+  EXPECT_EQ(true, log_file.close());
+
+  EXPECT_EQ(0, librmb::RmbCommands::delete_with_save_log("test1.log", "ceph", "client.admin"));
+  std::remove(test_file_name.c_str());
+}
+TEST(librmb, delete_objects_via_rmb_tool_and_save_log_file_invalid_file) {
+  std::string pool_name("rmb_tool_tests");
+  std::string ns("t1");
+
+  std::string test_file_name = "test1.log";
+  librmb::RadosSaveLog log_file(test_file_name);
+  EXPECT_EQ(true, log_file.open());
+  log_file.append(librmb::RadosSaveLogEntry("abc", "t1", "rmb_tool_tests", "save"));
+  EXPECT_EQ(true, log_file.close());
+
+  EXPECT_EQ(-1, librmb::RmbCommands::delete_with_save_log("test12.log", "ceph", "client.admin"));
+  std::remove(test_file_name.c_str());
+}
+TEST(librmb, delete_objects_via_rmb_tool_and_save_log_file_invalid_entry) {
+  librmb::RadosClusterImpl cluster;
+  librmb::RadosStorageImpl storage(&cluster);
+
+  std::string pool_name("rmb_tool_tests");
+  std::string ns("t1");
+
+  int open_connection = storage.open_connection(pool_name);
+  EXPECT_EQ(0, open_connection);
+  storage.set_namespace(ns);
+
+  std::string test_file_name = "test1.log";
+  librmb::RadosSaveLog log_file(test_file_name);
+  EXPECT_EQ(true, log_file.open());
+  log_file.append(
+      librmb::RadosSaveLogEntry("abc", "t1", "2,2,2rmb_tool_tests", "save"));  // -> stop processing (invalid entry)!
+  log_file.append(librmb::RadosSaveLogEntry("abc", "t1", "rmb_tool_tests", "save"));
+  EXPECT_EQ(true, log_file.close());
+  librados::bufferlist bl;
+  EXPECT_EQ(0, storage.save_mail("abc", bl));
+  cluster.deinit();
+
+  EXPECT_EQ(0, librmb::RmbCommands::delete_with_save_log("test1.log", "ceph",
+                                                         "client.admin"));  // -> due to invalid entry in object list
+  std::remove(test_file_name.c_str());
+}
 TEST(librmb, mock_obj) {}
 int main(int argc, char **argv) {
   ::testing::InitGoogleMock(&argc, argv);