From 0a2191591eab16d6f199fe373dea22cb21ca08cf Mon Sep 17 00:00:00 2001 From: Jan Radon Date: Sun, 20 Nov 2022 15:30:56 +0100 Subject: [PATCH] #349 add , to every entry or entry list --- src/librmb/rados-util.cpp | 585 ++++++++++++----------- src/storage-rbox/doveadm-rbox-plugin.cpp | 3 + 2 files changed, 297 insertions(+), 291 deletions(-) diff --git a/src/librmb/rados-util.cpp b/src/librmb/rados-util.cpp index 361678cd..2668bbb9 100644 --- a/src/librmb/rados-util.cpp +++ b/src/librmb/rados-util.cpp @@ -26,353 +26,352 @@ namespace librmb { -RadosUtils::RadosUtils() {} + RadosUtils::RadosUtils() {} -RadosUtils::~RadosUtils() {} + RadosUtils::~RadosUtils() {} -bool RadosUtils::convert_str_to_time_t(const std::string &date, time_t *val) { - struct tm tm = {0}; - if (strptime(date.c_str(), "%Y-%m-%d %H:%M:%S", &tm)) { - tm.tm_isdst = -1; - time_t t = mktime(&tm); - *val = t; - return true; - } - *val = 0; - return false; -} - -bool RadosUtils::convert_string_to_date(const std::string &date_string, std::string *date) { - time_t t; - if (convert_str_to_time_t(date_string, &t)) { - *date = std::to_string(t); - return true; - } - return false; -} - -bool RadosUtils::is_numeric(const char *s) { - if (s == NULL) { + bool RadosUtils::convert_str_to_time_t(const std::string &date, time_t *val) { + struct tm tm = {0}; + if (strptime(date.c_str(), "%Y-%m-%d %H:%M:%S", &tm)) { + tm.tm_isdst = -1; + time_t t = mktime(&tm); + *val = t; + return true; + } + *val = 0; return false; } - bool is_numeric = true; - int len = strlen(s); - for (int i = 0; i < len; i++) { - if (!std::isdigit(s[i])) { - is_numeric = false; - break; + + bool RadosUtils::convert_string_to_date(const std::string &date_string, std::string *date) { + time_t t; + if (convert_str_to_time_t(date_string, &t)) { + *date = std::to_string(t); + return true; } + return false; } - return is_numeric; -} - -bool RadosUtils::is_date_attribute(const rbox_metadata_key &key) { - return (key == RBOX_METADATA_OLDV1_SAVE_TIME || key == RBOX_METADATA_RECEIVED_TIME); -} + bool RadosUtils::is_numeric(const char *s) { + if (s == NULL) { + return false; + } + bool is_numeric = true; + int len = strlen(s); + for (int i = 0; i < len; i++) { + if (!std::isdigit(s[i])) { + is_numeric = false; + break; + } + } -int RadosUtils::convert_time_t_to_str(const time_t &t, std::string *ret_val) { - char buffer[256]; - if (t == -1) { - *ret_val = "invalid date"; - return -1; + return is_numeric; } - struct tm timeinfo; - localtime_r(&t, &timeinfo); - strftime(buffer, sizeof(buffer), "%Y-%m-%d %H:%M:%S", &timeinfo); - *ret_val = std::string(buffer); - return 0; -} -bool RadosUtils::flags_to_string(const uint8_t &flags_, std::string *flags_str) { - std::stringstream sstream; - sstream << std::hex << flags_; - sstream >> *flags_str; - return true; -} - -bool RadosUtils::string_to_flags(const std::string &flags_, uint8_t *flags) { - std::istringstream in(flags_); - - if (in >> std::hex >> *flags) { - return true; + + bool RadosUtils::is_date_attribute(const rbox_metadata_key &key) { + return (key == RBOX_METADATA_OLDV1_SAVE_TIME || key == RBOX_METADATA_RECEIVED_TIME); } - return false; -} -void RadosUtils::find_and_replace(std::string *source, std::string const &find, std::string const &replace) { - for (std::string::size_type i = 0; source != nullptr && (i = source->find(find, i)) != std::string::npos;) { - source->replace(i, find.length(), replace); - i += replace.length(); + int RadosUtils::convert_time_t_to_str(const time_t &t, std::string *ret_val) { + char buffer[256]; + if (t == -1) { + *ret_val = "invalid date"; + return -1; + } + struct tm timeinfo; + localtime_r(&t, &timeinfo); + strftime(buffer, sizeof(buffer), "%Y-%m-%d %H:%M:%S", &timeinfo); + *ret_val = std::string(buffer); + return 0; } -} - -int RadosUtils::get_all_keys_and_values(librados::IoCtx *io_ctx, const std::string &oid, - std::map *kv_map) { - int err = 0; - librados::ObjectReadOperation first_read; - std::set extended_keys; -#ifdef DOVECOT_CEPH_PLUGIN_HAVE_OMAP_GET_KEYS_2 - first_read.omap_get_keys2("", LONG_MAX, &extended_keys, nullptr, &err); -#else - first_read.omap_get_keys("", LONG_MAX, &extended_keys, &err); -#endif - int ret = io_ctx->operate(oid.c_str(), &first_read, NULL); - if (ret < 0) { - return ret; + bool RadosUtils::flags_to_string(const uint8_t &flags_, std::string *flags_str) { + std::stringstream sstream; + sstream << std::hex << flags_; + sstream >> *flags_str; + return true; } - return io_ctx->omap_get_vals_by_keys(oid, extended_keys, kv_map); -} -void RadosUtils::resolve_flags(const uint8_t &flags, std::string *flat) { - std::stringbuf buf; - std::ostream os(&buf); + bool RadosUtils::string_to_flags(const std::string &flags_, uint8_t *flags) { + std::istringstream in(flags_); - if ((flags & 0x01) != 0) { - os << "\\Answered "; - } - if ((flags & 0x02) != 0) { - os << "\\Flagged "; - } - if ((flags & 0x04) != 0) { - os << "\\Deleted "; + if (in >> std::hex >> *flags) { + return true; + } + return false; } - if ((flags & 0x08) != 0) { - os << "\\Seen "; + + void RadosUtils::find_and_replace(std::string *source, std::string const &find, std::string const &replace) { + for (std::string::size_type i = 0; source != nullptr && (i = source->find(find, i)) != std::string::npos;) { + source->replace(i, find.length(), replace); + i += replace.length(); + } } - if ((flags & 0x10) != 0) { - os << "\\Draft "; + + int RadosUtils::get_all_keys_and_values(librados::IoCtx *io_ctx, const std::string &oid, + std::map *kv_map) { + int err = 0; + librados::ObjectReadOperation first_read; + std::set extended_keys; + #ifdef DOVECOT_CEPH_PLUGIN_HAVE_OMAP_GET_KEYS_2 + first_read.omap_get_keys2("", LONG_MAX, &extended_keys, nullptr, &err); + #else + first_read.omap_get_keys("", LONG_MAX, &extended_keys, &err); + #endif + int ret = io_ctx->operate(oid.c_str(), &first_read, NULL); + if (ret < 0) { + return ret; + } + return io_ctx->omap_get_vals_by_keys(oid, extended_keys, kv_map); } - if ((flags & 0x20) != 0) { - os << "\\Recent "; + + void RadosUtils::resolve_flags(const uint8_t &flags, std::string *flat) { + std::stringbuf buf; + std::ostream os(&buf); + + if ((flags & 0x01) != 0) { + os << "\\Answered "; + } + if ((flags & 0x02) != 0) { + os << "\\Flagged "; + } + if ((flags & 0x04) != 0) { + os << "\\Deleted "; + } + if ((flags & 0x08) != 0) { + os << "\\Seen "; + } + if ((flags & 0x10) != 0) { + os << "\\Draft "; + } + if ((flags & 0x20) != 0) { + os << "\\Recent "; + } + *flat = buf.str(); } - *flat = buf.str(); -} -int RadosUtils::osd_add(librados::IoCtx *ioctx, const std::string &oid, const std::string &key, - long long value_to_add) { - librados::bufferlist in, out; - encode(key, in); + int RadosUtils::osd_add(librados::IoCtx *ioctx, const std::string &oid, const std::string &key, + long long value_to_add) { + librados::bufferlist in, out; + encode(key, in); - std::stringstream stream; - stream << value_to_add; + std::stringstream stream; + stream << value_to_add; - encode(stream.str(), in); + encode(stream.str(), in); - return ioctx->exec(oid, "numops", "add", in, out); -} + return ioctx->exec(oid, "numops", "add", in, out); + } -int RadosUtils::osd_sub(librados::IoCtx *ioctx, const std::string &oid, const std::string &key, - long long value_to_subtract) { - return osd_add(ioctx, oid, key, -value_to_subtract); -} + int RadosUtils::osd_sub(librados::IoCtx *ioctx, const std::string &oid, const std::string &key, + long long value_to_subtract) { + return osd_add(ioctx, oid, key, -value_to_subtract); + } -/*! - * @return reference to all write operations related with this object - */ + /*! + * @return reference to all write operations related with this object + */ -void RadosUtils::get_metadata(const std::string &key, std::map *metadata, char **value) { - if (metadata->find(key) != metadata->end()) { - *value = (*metadata)[key].c_str(); - return; + void RadosUtils::get_metadata(const std::string &key, std::map *metadata, char **value) { + if (metadata->find(key) != metadata->end()) { + *value = (*metadata)[key].c_str(); + return; + } + *value = NULL; } - *value = NULL; -} -void RadosUtils::get_metadata(rbox_metadata_key key, std::map *metadata, char **value) { - string str_key(librmb::rbox_metadata_key_to_char(key)); - get_metadata(str_key, metadata, value); -} -bool RadosUtils::is_numeric_optional(const char *text) { - if (text == NULL) { - return true; // optional + void RadosUtils::get_metadata(rbox_metadata_key key, std::map *metadata, char **value) { + string str_key(librmb::rbox_metadata_key_to_char(key)); + get_metadata(str_key, metadata, value); } - return is_numeric(text); -} - -bool RadosUtils::validate_metadata(map *metadata) { - char *uid = NULL; - get_metadata(RBOX_METADATA_MAIL_UID, metadata, &uid); - char *recv_time_str = NULL; - get_metadata(RBOX_METADATA_RECEIVED_TIME, metadata, &recv_time_str); - char *p_size = NULL; - get_metadata(RBOX_METADATA_PHYSICAL_SIZE, metadata, &p_size); - char *v_size = NULL; - get_metadata(RBOX_METADATA_VIRTUAL_SIZE, metadata, &v_size); - - char *rbox_version; - get_metadata(RBOX_METADATA_VERSION, metadata, &rbox_version); - char *mailbox_guid = NULL; - get_metadata(RBOX_METADATA_MAILBOX_GUID, metadata, &mailbox_guid); - char *mail_guid = NULL; - get_metadata(RBOX_METADATA_GUID, metadata, &mail_guid); - char *mb_orig_name = NULL; - get_metadata(RBOX_METADATA_ORIG_MAILBOX, metadata, &mb_orig_name); - - char *flags = NULL; - get_metadata(RBOX_METADATA_OLDV1_FLAGS, metadata, &flags); - char *pvt_flags = NULL; - get_metadata(RBOX_METADATA_PVT_FLAGS, metadata, &pvt_flags); - char *from_envelope = NULL; - get_metadata(RBOX_METADATA_FROM_ENVELOPE, metadata, &from_envelope); - - int test = 0; - test += is_numeric(uid) ? 0 : 1; - test += is_numeric(recv_time_str) ? 0 : 1; - test += is_numeric(p_size) ? 0 : 1; - test += is_numeric(v_size) ? 0 : 1; - - test += is_numeric_optional(flags) ? 0 : 1; - test += is_numeric_optional(pvt_flags) ? 0 : 1; - - test += mailbox_guid == NULL ? 1 : 0; - test += mail_guid == NULL ? 1 : 0; - return test == 0; -} -// assumes that destination is open and initialized with uses namespace -int RadosUtils::move_to_alt(std::string &oid, RadosStorage *primary, RadosStorage *alt_storage, - RadosMetadataStorage *metadata, bool inverse) { - int ret = -1; - ret = copy_to_alt(oid, oid, primary, alt_storage, metadata, inverse); - if (ret > 0) { - if (inverse) { - ret = alt_storage->get_io_ctx().remove(oid); - } else { - ret = primary->get_io_ctx().remove(oid); + bool RadosUtils::is_numeric_optional(const char *text) { + if (text == NULL) { + return true; // optional } - } - return ret; -} -int RadosUtils::copy_to_alt(std::string &src_oid, std::string &dest_oid, RadosStorage *primary, - RadosStorage *alt_storage, RadosMetadataStorage *metadata, bool inverse) { - int ret = 0; - - // TODO(jrse) check that storage is connected and open. - if (primary == nullptr || alt_storage == nullptr) { - return 0; + return is_numeric(text); } - RadosMail mail; - mail.set_oid(src_oid); - - librados::bufferlist *bl = new librados::bufferlist(); - mail.set_mail_buffer(bl); - - if (inverse) { - ret = alt_storage->read_mail(src_oid, mail.get_mail_buffer()); - metadata->get_storage()->set_io_ctx(&alt_storage->get_io_ctx()); - } else { - ret = primary->read_mail(src_oid, mail.get_mail_buffer()); + bool RadosUtils::validate_metadata(map *metadata) { + char *uid = NULL; + get_metadata(RBOX_METADATA_MAIL_UID, metadata, &uid); + char *recv_time_str = NULL; + get_metadata(RBOX_METADATA_RECEIVED_TIME, metadata, &recv_time_str); + char *p_size = NULL; + get_metadata(RBOX_METADATA_PHYSICAL_SIZE, metadata, &p_size); + char *v_size = NULL; + get_metadata(RBOX_METADATA_VIRTUAL_SIZE, metadata, &v_size); + + char *rbox_version; + get_metadata(RBOX_METADATA_VERSION, metadata, &rbox_version); + char *mailbox_guid = NULL; + get_metadata(RBOX_METADATA_MAILBOX_GUID, metadata, &mailbox_guid); + char *mail_guid = NULL; + get_metadata(RBOX_METADATA_GUID, metadata, &mail_guid); + char *mb_orig_name = NULL; + get_metadata(RBOX_METADATA_ORIG_MAILBOX, metadata, &mb_orig_name); + + char *flags = NULL; + get_metadata(RBOX_METADATA_OLDV1_FLAGS, metadata, &flags); + char *pvt_flags = NULL; + get_metadata(RBOX_METADATA_PVT_FLAGS, metadata, &pvt_flags); + char *from_envelope = NULL; + get_metadata(RBOX_METADATA_FROM_ENVELOPE, metadata, &from_envelope); + + int test = 0; + test += is_numeric(uid) ? 0 : 1; + test += is_numeric(recv_time_str) ? 0 : 1; + test += is_numeric(p_size) ? 0 : 1; + test += is_numeric(v_size) ? 0 : 1; + + test += is_numeric_optional(flags) ? 0 : 1; + test += is_numeric_optional(pvt_flags) ? 0 : 1; + + test += mailbox_guid == NULL ? 1 : 0; + test += mail_guid == NULL ? 1 : 0; + return test == 0; } - - if (ret < 0) { - metadata->get_storage()->set_io_ctx(&primary->get_io_ctx()); + // assumes that destination is open and initialized with uses namespace + int RadosUtils::move_to_alt(std::string &oid, RadosStorage *primary, RadosStorage *alt_storage, + RadosMetadataStorage *metadata, bool inverse) { + int ret = -1; + ret = copy_to_alt(oid, oid, primary, alt_storage, metadata, inverse); + if (ret > 0) { + if (inverse) { + ret = alt_storage->get_io_ctx().remove(oid); + } else { + ret = primary->get_io_ctx().remove(oid); + } + } return ret; } - mail.set_mail_size(mail.get_mail_buffer()->length()); + int RadosUtils::copy_to_alt(std::string &src_oid, std::string &dest_oid, RadosStorage *primary, + RadosStorage *alt_storage, RadosMetadataStorage *metadata, bool inverse) { + int ret = 0; - // load the metadata; - ret = metadata->get_storage()->load_metadata(&mail); - if (ret < 0) { - return ret; - } + // TODO(jrse) check that storage is connected and open. + if (primary == nullptr || alt_storage == nullptr) { + return 0; + } - mail.set_oid(dest_oid); + RadosMail mail; + mail.set_oid(src_oid); - librados::ObjectWriteOperation write_op; // = new librados::ObjectWriteOperation(); - metadata->get_storage()->save_metadata(&write_op, &mail); + librados::bufferlist *bl = new librados::bufferlist(); + mail.set_mail_buffer(bl); - bool success; - if (inverse) { - success = primary->save_mail(&write_op, &mail); - } else { - success = alt_storage->save_mail(&write_op, &mail); - } + if (inverse) { + ret = alt_storage->read_mail(src_oid, mail.get_mail_buffer()); + metadata->get_storage()->set_io_ctx(&alt_storage->get_io_ctx()); + } else { + ret = primary->read_mail(src_oid, mail.get_mail_buffer()); + } - if (!success) { - return 0; - } + if (ret < 0) { + metadata->get_storage()->set_io_ctx(&primary->get_io_ctx()); + return ret; + } + mail.set_mail_size(mail.get_mail_buffer()->length()); - return success ? 0 : 1; -} + // load the metadata; + ret = metadata->get_storage()->load_metadata(&mail); + if (ret < 0) { + return ret; + } -static std::vector RadosUtils::extractPgs(const std::string& str) -{ - std::vector tokens; + mail.set_oid(dest_oid); - std::stringstream ss(str); - std::string token; - while (std::getline(ss, token, '\n')) { - std::string pgs = token.substr(0, 10); //take first 10 chars for pgids - // trim the result. - pgs.erase(std::remove_if(pgs.begin(), pgs.end(), ::isspace), pgs.end()); - tokens.push_back(pgs); - } - //skip first line (header) - tokens.erase(tokens.begin()); - //remove last element (footer) - tokens.pop_back(); - - return tokens; -} + librados::ObjectWriteOperation write_op; // = new librados::ObjectWriteOperation(); + metadata->get_storage()->save_metadata(&write_op, &mail); -static std::map> RadosUtils::extractPgAndPrimaryOsd(const std::string& str) -{ - std::map> tokens; + bool success; + if (inverse) { + success = primary->save_mail(&write_op, &mail); + } else { + success = alt_storage->save_mail(&write_op, &mail); + } - std::stringstream ss(str); - std::string token; - bool first_line = true; - while (std::getline(ss, token, '\n')) { - if(first_line){ - first_line = false; - continue; - } - std::vector line = split(token,' '); - if(line.size() < 14){ - continue; - } - std::string tmp_primary_osd = split(line[13],',')[0]; - std::string primary_osd = tmp_primary_osd.erase(0,1); - std::string pgs = line[0]; - - auto it = tokens.find(primary_osd); - if(it!=tokens.end()){ - tokens[primary_osd].push_back(pgs); - }else{ - std::vector t; - t.push_back(pgs); - tokens.insert({primary_osd,t}); - } - + if (!success) { + return 0; } - - return tokens; -} -static std::vector RadosUtils::split(std::string str_to_split, char delimiter) { - std::vector tokens; - std::stringstream stream(str_to_split); - std::string token; + return success ? 0 : 1; + } + + static std::vector RadosUtils::extractPgs(const std::string& str) + { + std::vector tokens; + + std::stringstream ss(str); + std::string token; + while (std::getline(ss, token, '\n')) { + std::string pgs = token.substr(0, 10); //take first 10 chars for pgids + // trim the result. + pgs.erase(std::remove_if(pgs.begin(), pgs.end(), ::isspace), pgs.end()); + tokens.push_back(pgs); + } + //skip first line (header) + tokens.erase(tokens.begin()); + //remove last element (footer) + tokens.pop_back(); + + return tokens; + } - while(getline(stream, token, delimiter)) { - token.erase(std::remove_if(token.begin(), token.end(), ::isspace), token.end()); - if(token.length() > 0) { - tokens.push_back(token); - } - + static std::map> RadosUtils::extractPgAndPrimaryOsd(const std::string& str) + { + std::map> tokens; + + std::stringstream ss(str); + std::string token; + bool first_line = true; + while (std::getline(ss, token, '\n')) { + if(first_line){ + first_line = false; + continue; + } + std::vector line = split(token,' '); + if(line.size() < 14){ + continue; + } + std::string tmp_primary_osd = split(line[13],',')[0]; + std::string primary_osd = tmp_primary_osd.erase(0,1); + std::string pgs = line[0]; + + auto it = tokens.find(primary_osd); + if(it!=tokens.end()){ + tokens[primary_osd].push_back(pgs); + }else{ + std::vector t; + t.push_back(pgs); + tokens.insert({primary_osd,t}); + } + + } + + return tokens; } - return tokens; -} + static std::vector RadosUtils::split(std::string str_to_split, char delimiter) { + std::vector tokens; + std::stringstream stream(str_to_split); + std::string token; + + while(getline(stream, token, delimiter)) { + token.erase(std::remove_if(token.begin(), token.end(), ::isspace), token.end()); + if(token.length() > 0) { + tokens.push_back(token); + } + + } + return tokens; + } static std::string RadosUtils::convert_to_ceph_index(const std::set &list){ std::ostringstream str; std::copy(list.begin(), list.end(), std::ostream_iterator(str, ",")); - return str.str().substr(0,str.str().length() -1); + return str.str(); } static std::string RadosUtils::convert_to_ceph_index(const std::string &str) { - return str; + return str + ","; } static std::set RadosUtils::ceph_index_to_set(const std::string &str) { @@ -382,7 +381,11 @@ static std::vector RadosUtils::split(std::string str_to_split, char while (ss.good()) { std::string substr; getline(ss, substr, ','); - index.insert(substr); + + if(substr.length() != 0){ + index.insert(substr); + } + } return index; } diff --git a/src/storage-rbox/doveadm-rbox-plugin.cpp b/src/storage-rbox/doveadm-rbox-plugin.cpp index 92e531d6..19a56ec3 100644 --- a/src/storage-rbox/doveadm-rbox-plugin.cpp +++ b/src/storage-rbox/doveadm-rbox-plugin.cpp @@ -822,6 +822,9 @@ static int cmd_rmb_create_ceph_index_run(struct doveadm_mail_cmd_context *_ctx, ret = -1; break; } + + //append to index. + } } } // end of for