Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Feature/306 assign unassigned mail objects #309

Merged
merged 1 commit into from
Apr 2, 2022
Merged

Feature/306 assign unassigned mail objects #309

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion src/librmb/tools/rmb/mailbox_tools.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,6 @@ int MailboxTools::save_mail(librmb::RadosMail* mail_obj) {
}

std::string file_path = mailbox_path + "/" + filename;
std::cout << " writing mail to " << file_path << std::endl;
std::ofstream myfile(file_path, std::ofstream::binary | std::ofstream::out);
if (!myfile.is_open()) {
return -1;
Expand Down
157 changes: 130 additions & 27 deletions src/storage-rbox/rbox-sync-rebuild.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,10 @@ extern "C" {
#include "rbox-mail.h"
#include "encoding.h"
#include "../librmb/rados-mail.h"
#include "rados-util.h"
#include "../librmb/rados-util.h"
#include "rados-types.h"


using librmb::RadosMail;
using librmb::rbox_metadata_key;

Expand All @@ -46,8 +47,13 @@ int rbox_sync_add_object(struct index_rebuild_context *ctx, const std::string &o
mail_index_append(ctx->trans, next_uid, &seq);

T_BEGIN {
uint32_t uid = atoi(xattr_mail_uid);
index_rebuild_index_metadata(ctx, seq, uid); }
uint32_t uid = std::stoi(xattr_mail_uid);
// uid = INT32_MAX if a previous force-resync detected, that the mail object has a mailbox guid which
// is no longer valid.
if(uid != INT32_MAX){
// there should be a previous index entry available (if index exist)
index_rebuild_index_metadata(ctx, seq, uid); }
}
T_END;

#ifdef DEBUG
Expand All @@ -59,15 +65,17 @@ int rbox_sync_add_object(struct index_rebuild_context *ctx, const std::string &o
// convert oid and guid to
guid_128_t oid;
if (guid_128_from_string(oi.c_str(), oid) < 0) {
i_error("guid_128 oi.c_str() string (%s), next_uid(%d)", oi.c_str(), next_uid);
FUNC_END();
return -1;
i_error("converting oid failed : guid_128 oi.c_str() string (%s), next_uid(%d)", oi.c_str(), next_uid);
FUNC_END();
return -1;
}
guid_128_t guid;
if (guid_128_from_uuid_string(xattr_guid, guid) < 0) {
i_error("converting guid failed : guid_128 oi.c_str() string (%s), next_uid(%d)", oi.c_str(), next_uid);
FUNC_END();
return -1;
}

memcpy(rec.guid, guid, sizeof(guid));
memcpy(rec.oid, oid, sizeof(oid));

Expand All @@ -76,7 +84,6 @@ int rbox_sync_add_object(struct index_rebuild_context *ctx, const std::string &o
mail_index_update_flags(ctx->trans, seq, MODIFY_ADD, (enum mail_flags)RBOX_INDEX_FLAG_ALT);
}


// update uid.
librmb::RadosMetadata mail_uid(librmb::RBOX_METADATA_MAIL_UID, next_uid);
std::string s_oid = *mail_obj->get_oid();
Expand All @@ -86,7 +93,7 @@ int rbox_sync_add_object(struct index_rebuild_context *ctx, const std::string &o
i_warning("update of MAIL_UID failed: for object: %s , uid: %d", mail_obj->get_oid()->c_str(), next_uid);
}
#ifdef DEBUG
i_debug("rebuilding %s , with oid=%d", oi.c_str(), next_uid);
i_debug("rebuilding %s , with uid=%d", oi.c_str(), next_uid);
#endif
FUNC_END();
return 0;
Expand All @@ -113,7 +120,7 @@ std::map<std::string, std::list<librmb::RadosMail>> load_rados_mail_metadata(
}

if (load_metadata_ret < 0 || !librmb::RadosUtils::validate_metadata(mail_object.get_metadata())) {
i_error("metadata for object : %s is not valid, skipping object ", mail_object.get_oid()->c_str());
i_warning("metadata for object : %s is not valid, skipping object ", mail_object.get_oid()->c_str());
++iter;
continue;
}
Expand Down Expand Up @@ -144,7 +151,6 @@ int rbox_sync_rebuild_entry(struct index_rebuild_context *ctx, std::map<std::str
struct rbox_sync_rebuild_ctx *rebuild_ctx) {
FUNC_START();
struct mail_storage *storage = ctx->box->storage;
struct rbox_storage *r_storage = (struct rbox_storage *)storage;
struct rbox_mailbox *rbox = (struct rbox_mailbox *)ctx->box;

const struct mail_index_header *hdr = mail_index_get_header(ctx->trans->view);
Expand All @@ -153,7 +159,6 @@ int rbox_sync_rebuild_entry(struct index_rebuild_context *ctx, std::map<std::str
rebuild_ctx->next_uid = hdr->next_uid != 0 ? hdr->next_uid : 1;
}

int found = 0;
int sync_add_objects_ret = 0;

std::string mailbox_guid(guid_128_to_string(rbox->mailbox_guid));
Expand All @@ -169,17 +174,17 @@ int rbox_sync_rebuild_entry(struct index_rebuild_context *ctx, std::map<std::str

sync_add_objects_ret =
rbox_sync_add_object(ctx, *it->get_oid(), &(*it), rebuild_ctx->alt_storage, rebuild_ctx->next_uid);
i_debug("re-adding mail : %s to mailbox %s ", it->get_oid()->c_str(), mailbox_guid.c_str() );
i_info("re-adding mail oid:(%s) with uid: %d to mailbox %s (%s) ", it->get_oid()->c_str(), rebuild_ctx->next_uid, mailbox_guid.c_str(), ctx->box->name );
if (sync_add_objects_ret < 0) {
i_error("sync_add_object: oid(%s), alt_storage(%d),uid(%d)", it->get_oid()->c_str(),
rebuild_ctx->alt_storage, rebuild_ctx->next_uid);
break;
}
it->set_restored(true);
it->set_index_ref(true);
i_info("restored rados_mail: %s",it->to_string(" ").c_str());
i_debug("restored rados_mail: %s",it->to_string(" ").c_str());

++rebuild_ctx->next_uid;
rebuild_ctx->next_uid++;
}

if (sync_add_objects_ret < 0) {
Expand Down Expand Up @@ -257,33 +262,131 @@ int rbox_storage_rebuild_in_context(struct rbox_storage *r_storage, bool force)
std::map<std::string, std::list<librmb::RadosMail>> rados_mails;

struct mail_namespace *ns = mail_namespace_find_inbox(user->namespaces);


struct mail_namespace *ns_mailbox = nullptr;
//TODO: save inbox namespace:
// assign unassigned mails to inbox
for (; ns != NULL; ns = ns->next) {
// iter_guid will be initialized in repair_namespace,
// as we first need to call open_mailbox to initialize
// the rados_connection successfully and list objects in
// the user namespace

repair_namespace(ns, force, r_storage, rados_mails);
}
i_info("repair done checking for unassigned mails ");

i_info("Repair done checking for unassigned mails ");
std::map<std::string, std::list<librmb::RadosMail>>::iterator it;
long count_not_assigned = 0;

long count_assigned = 0;
for(it=rados_mails.begin(); it!=rados_mails.end(); ++it){
std::list<librmb::RadosMail>::iterator list_it;
for(list_it=it->second.begin(); list_it!=it->second.end(); ++list_it){
if(!list_it->is_restored()){
i_info("unassigned mail: %s",list_it->to_string(" ").c_str());
}
count_not_assigned += list_it->is_restored() ? 0 : 1;
count_assigned += list_it->is_restored() ? 1 : 0;
}
}
i_info("total unassigned mails %ld", count_not_assigned);

i_info("Total assigned mails %ld", count_assigned);
i_info("Total unassigned mails %ld", count_not_assigned);

if(count_not_assigned > 0)
{
std::string last_known_mailbox_guid;
struct mail_namespace *ns_second = mail_namespace_find_inbox(user->namespaces);

if(find_default_mailbox_guid(ns_second, &last_known_mailbox_guid) < 0)
{
// bad no default mailbox found
i_warning("unable to find inbox guid => unreferenced mails can not automatically be re-assigned ");
return 0;
}
int unassigned_counter = 0;
for(it=rados_mails.begin(); it!=rados_mails.end(); ++it)
{
std::list<librmb::RadosMail>::iterator list_it;
for(list_it=it->second.begin(); list_it!=it->second.end(); ++list_it){
if(list_it->is_restored()){
continue;
}
librmb::RadosMetadata metadata;
metadata.convert(rbox_metadata_key::RBOX_METADATA_MAILBOX_GUID, last_known_mailbox_guid);

librmb::RadosMetadata metadata_uid;
metadata_uid.convert(rbox_metadata_key::RBOX_METADATA_MAIL_UID, INT32_MAX);

librados::ObjectWriteOperation write_mail_uid;
write_mail_uid.setxattr(metadata_uid.key.c_str(), metadata_uid.bl);
write_mail_uid.setxattr(metadata.key.c_str(), metadata.bl);

if (r_storage->s->get_io_ctx().operate(*list_it->get_oid(), &write_mail_uid) < 0) {
i_info("Unable to reset metadata to guid : %s",last_known_mailbox_guid.c_str());
}else {
i_info("(%d) Mailbox guid for mail (oid=%s) restored to %s (INBOX) => re-run force-resync to assign them ",unassigned_counter, list_it->get_oid()->c_str(),last_known_mailbox_guid.c_str());
}
unassigned_counter++;
}
}

}

FUNC_END();
return 0;
}

int find_default_mailbox_guid(struct mail_namespace *ns, std::string *mailbox_guid) {
for (; ns != NULL; ns = ns->next) {
if(find_inbox_mailbox_guid(ns, mailbox_guid) == 0 ){
return 0;
}
}
// none found
return -1;
}

int find_inbox_mailbox_guid(struct mail_namespace *ns, std::string *mailbox_guid) {
FUNC_START();
struct mailbox_list_iterate_context *iter;
const struct mailbox_info *info;
int ret = -1;

iter = mailbox_list_iter_init(ns->list, "*", static_cast<mailbox_list_iter_flags>(MAILBOX_LIST_ITER_RAW_LIST |
MAILBOX_LIST_ITER_RETURN_NO_FLAGS));
while ((info = mailbox_list_iter_next(iter)) != NULL) {
i_debug("checking mailbox %s",info->vname);

if ((info->flags & (MAILBOX_NONEXISTENT | MAILBOX_NOSELECT)) == 0) {

struct mailbox *box = mailbox_alloc(ns->list, info->vname, MAILBOX_FLAG_SAVEONLY);
if (box->virtual_vfuncs != NULL) {
/* the namespace has multiple storages. or is virtual box */
mailbox_free(&box);
return -1;
}

if (mailbox_open(box) < 0) {
FUNC_END();
return -1;
}

if (strcmp(info->vname, "INBOX") == 0){
char* guid = guid_128_to_string(((struct rbox_mailbox *)box)->mailbox_guid);
i_info("found default INBOX mailbox with guid: %s",guid);
*mailbox_guid = guid;
ret = 0;
}
mailbox_free(&box);
}
}

if (mailbox_list_iter_deinit(&iter) < 0) {
ret = -1;
}

FUNC_END();
return ret;
}

int repair_namespace(struct mail_namespace *ns, bool force, struct rbox_storage *r_storage, std::map<std::string, std::list<librmb::RadosMail>> &rados_mails) {
FUNC_START();
struct mailbox_list_iterate_context *iter;
Expand Down Expand Up @@ -317,15 +420,15 @@ int repair_namespace(struct mail_namespace *ns, bool force, struct rbox_storage
FUNC_END();
return -1;
}
i_info("ceph connection established using namespace: %s",r_storage->s->get_namespace().c_str());
i_info("loading mails... ");
i_info("Ceph connection established using namespace: %s",r_storage->s->get_namespace().c_str());
i_info("Loading mails... ");
librados::NObjectIterator *iter_guid = new librados::NObjectIterator(r_storage->s->find_mails(nullptr));
i_info("loading mail metadata...");
i_info("Loading mail metadata...");
rados_mails = load_rados_mail_metadata(false,r_storage,*iter_guid);
i_info("mails completely loaded ");
i_info("Mails completely loaded ");
std::map<std::string, std::list<librmb::RadosMail>>::iterator it;
for(it=rados_mails.begin(); it!=rados_mails.end(); ++it){
i_info("found mails for mailbox_guid: %s: mails : %ld",it->first.c_str(), it->second.size());
i_info("Found mails for mailbox_guid: %s: mails : %ld",it->first.c_str(), it->second.size());
}
}

Expand Down Expand Up @@ -372,7 +475,7 @@ int rbox_sync_index_rebuild(struct rbox_mailbox *rbox, bool force, std::map<std:
i_debug("index could not be opened");
#endif
}
i_warning("rbox %s: Rebuilding index, guid: %s , mailbox_name: %s, alt_storage: %s", mailbox_get_path(&rbox->box),
i_info("rbox %s: Rebuilding index, guid: %s , mailbox_name: %s, alt_storage: %s", mailbox_get_path(&rbox->box),
guid_128_to_string(rbox->mailbox_guid), rbox->box.name, rbox->box.list->set.alt_dir);

view = mail_index_view_open(rbox->box.index);
Expand Down
6 changes: 5 additions & 1 deletion src/storage-rbox/rbox-sync-rebuild.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,5 +44,9 @@ extern int rbox_storage_rebuild_in_context(struct rbox_storage *r_storage, bool
extern int repair_namespace(struct mail_namespace *ns, bool force, struct rbox_storage *r_storage, std::map<std::string, std::list<librmb::RadosMail>> &rados_mails);

extern std::map<std::string, std::list<librmb::RadosMail>> load_rados_mail_metadata(bool alt_storage, struct rbox_storage *r_storage, librados::NObjectIterator &iter);


extern int find_default_mailbox_guid(struct mail_namespace *ns, std::string *mailbox_guid);
extern int find_inbox_mailbox_guid(struct mail_namespace *ns, std::string *mailbox_guid);

#endif // SRC_STORAGE_RBOX_RBOX_SYNC_REBUILD_H_