Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Ls orphaned objects #172 #177

Merged
merged 2 commits into from
Jul 16, 2018
Merged

Ls orphaned objects #172 #177

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3eafd0b
#172: ls now shows unreferenced objects.
jrse Jul 16, 2018
0229d3d
#172: print message in ls mb in case mailbox has unreferenced messages.
jrse Jul 16, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 6 additions & 7 deletions src/librmb/rados-mail-object.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,9 @@ using librmb::RadosMailObject;
const char RadosMailObject::X_ATTR_VERSION_VALUE[] = "0.1";
const char RadosMailObject::DATA_BUFFER_NAME[] = "RADOS_MAIL_BUFFER";

RadosMailObject::RadosMailObject() {
this->object_size = -1;
this->active_op = false;
this->save_date_rados = -1;
this->valid = true;
}
RadosMailObject::RadosMailObject()
: object_size(-1), active_op(false), save_date_rados(-1), valid(true), index_ref(false) {}

RadosMailObject::~RadosMailObject() {}

void RadosMailObject::set_guid(const uint8_t *_guid) { memcpy(this->guid, _guid, sizeof(this->guid)); }
Expand Down Expand Up @@ -65,7 +62,9 @@ std::string RadosMailObject::to_string(const string &padding) {
if (!valid) {
ss << padding << "<<< MAIL OBJECT IS NOT VALID <<<<" << endl;
}

if (!index_ref) {
ss << padding << "<<< MAIL OBJECT HAS NO INDEX REFERENCE <<<<" << endl;
}
ss << padding << "MAIL: ";
if (!uid.empty()) {
ss << static_cast<char>(RBOX_METADATA_MAIL_UID) << "(uid)=" << uid << endl;
Expand Down
5 changes: 4 additions & 1 deletion src/librmb/rados-mail-object.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,8 @@ string get_metadata(const string& key) {
return value;
}

bool is_index_ref() { return index_ref; }
void set_index_ref(bool ref) { this->index_ref = ref; }
bool is_valid() { return valid; }
void set_valid(bool valid_) { valid = valid_; }
bool has_active_op() { return active_op; }
Expand Down Expand Up @@ -95,10 +97,11 @@ const string get_extended_metadata(string& key) {
map<string, ceph::bufferlist> attrset;
map<string, ceph::bufferlist> extended_attrset;
bool valid;

bool index_ref;
public:
static const char X_ATTR_VERSION_VALUE[];
static const char DATA_BUFFER_NAME[];

};

} // namespace librmb
Expand Down
3 changes: 3 additions & 0 deletions src/librmb/tools/rmb/rmb.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -417,6 +417,9 @@ int main(int argc, const char **argv) {
if (opts["ls"].compare("all") == 0 || opts["ls"].compare("-") == 0 || parser.parse_ls_string()) {
rmb_commands->load_objects(ms, mail_objects, sort_type);
rmb_commands->query_mail_storage(&mail_objects, &parser, false, false);
std::cout << " NOTE: rmb tool does not have access to dovecot index. so all objects are set <<< MAIL OBJECT "
"HAS NO INDEX REFERENCE <<<< use doveadm rmb ls - instead "
<< std::endl;
}
} else if (opts.find("get") != opts.end()) {
librmb::CmdLineParser parser(opts["get"]);
Expand Down
45 changes: 31 additions & 14 deletions src/storage-rbox/doveadm-rbox-plugin.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ extern "C" {
#include "istream.h"
#include "doveadm-print.h"
}
#include <algorithm>

#include "tools/rmb/rmb-commands.h"
#include "rados-cluster.h"
#include "rados-cluster-impl.h"
Expand All @@ -43,9 +45,7 @@ extern "C" {
#include "rbox-save.h"
#include "rbox-storage.hpp"

#include <algorithm>


int check_namespace_mailboxes(struct mail_namespace *ns, std::vector<librmb::RadosMailObject *> &mail_objects);

class RboxDoveadmPlugin {
public:
Expand Down Expand Up @@ -169,7 +169,7 @@ static int cmd_rmb_search_run(std::map<std::string, std::string> &opts, struct m
int open = open_connection_load_config(&plugin);
if (open < 0) {
i_error("Error opening rados connection. Errorcode: %d", open);
return 0;
return open;
}

opts["namespace"] = user->username;
Expand All @@ -191,7 +191,12 @@ static int cmd_rmb_search_run(std::map<std::string, std::string> &opts, struct m
delete ms;
return ret;
}

if (user->namespaces != NULL) {
struct mail_namespace *ns = mail_namespace_find_inbox(user->namespaces);
for (; ns != NULL; ns = ns->next) {
check_namespace_mailboxes(ns, mail_objects);
}
}
if (download) {
rmb_cmds.set_output_path(&parser);
}
Expand Down Expand Up @@ -221,7 +226,17 @@ static int cmd_rmb_ls_run(struct doveadm_mail_cmd_context *ctx, struct mail_user

if (opts["ls"].compare("all") == 0 || opts["ls"].compare("-") == 0 || parser.parse_ls_string()) {
std::vector<librmb::RadosMailObject *> mail_objects;

ctx->exit_code = cmd_rmb_search_run(opts, user, false, parser, mail_objects, false);

// TODO: check for mails without reference
auto it_mail = std::find_if(mail_objects.begin(), mail_objects.end(),
[](librmb::RadosMailObject const *n) -> bool { return !n->is_index_ref(); });

if (it_mail != mail_objects.end()) {
std::cout << "There are unreference objects " << std::endl;
}

for (auto mo : mail_objects) {
delete mo;
}
Expand All @@ -240,9 +255,6 @@ static int cmd_rmb_ls_mb_run(struct doveadm_mail_cmd_context *ctx, struct mail_u
if (opts["ls"].compare("all") == 0 || opts["ls"].compare("-") == 0 || parser.parse_ls_string()) {
std::vector<librmb::RadosMailObject *> mail_objects;
ctx->exit_code = cmd_rmb_search_run(opts, user, false, parser, mail_objects, false);
for (auto mo : mail_objects) {
delete mo;
}
} else {
i_error("invalid ls search query");
ctx->exit_code = -1;
Expand Down Expand Up @@ -654,11 +666,12 @@ static int iterate_mailbox(struct mail_namespace *ns, const struct mailbox_info
[oid](librmb::RadosMailObject *m) { return m->get_oid().compare(oid) == 0; });

if (it_mail == mail_objects.end()) {
std::cout << " missing mail object: uid=" << mail->uid << " guid=" << guid << " oid : " << oid
<< " available: " << (it_mail != mail_objects.end()) << std::endl;
/* std::cout << " missing mail object: uid=" << mail->uid << " guid=" << guid << " oid : " << oid
<< " available: " << (it_mail != mail_objects.end()) << std::endl;*/
++mail_count_missing;
} else {
mail_objects.erase(it_mail); // calls destructor of RadosMailObject*
(*it_mail)->set_index_ref(true);
// mail_objects.erase(it_mail); // calls destructor of RadosMailObject*
}
}
if (mailbox_search_deinit(&search_ctx) < 0) {
Expand All @@ -681,7 +694,7 @@ static int check_namespace_mailboxes(struct mail_namespace *ns, std::vector<libr
struct mailbox_list_iterate_context *iter;
const struct mailbox_info *info;
int ret = 0;

std::cout << "INDEX: Check" << std::endl;
iter = mailbox_list_iter_init(ns->list, "*", static_cast<enum mailbox_list_iter_flags>(
MAILBOX_LIST_ITER_RAW_LIST | MAILBOX_LIST_ITER_RETURN_NO_FLAGS));
while ((info = mailbox_list_iter_next(iter)) != NULL) {
Expand Down Expand Up @@ -717,7 +730,11 @@ static int cmd_rmb_check_indices_run(struct doveadm_mail_cmd_context *ctx, struc
check_namespace_mailboxes(ns, mail_objects);
}

if (mail_objects.size() > 0) {
// TODO: check for mails with
auto it_mail = std::find_if(mail_objects.begin(), mail_objects.end(),
[](librmb::RadosMailObject *m) { return !m->is_index_ref(); });

if (it_mail != mail_objects.end()) {
std::cout << std::endl << "There are mail objects without a index reference: " << std::endl;
std::cout << "NOTE: you can fix(restore) the lost index entries by using doveadm force-resync or delete the "
"unrefrenced objects from objectstore "
Expand Down Expand Up @@ -750,7 +767,7 @@ static int cmd_rmb_check_indices_run(struct doveadm_mail_cmd_context *ctx, struc

for (auto mo : mail_objects) {
std::cout << mo->to_string(" ") << std::endl;
if (open >= 0 && ctx_->delete_not_referenced_objects) {
if (open >= 0 && ctx_->delete_not_referenced_objects && !mo->is_index_ref()) {
std::cout << "mail object: " << mo->get_oid().c_str()
<< " deleted: " << (plugin.storage->delete_mail(mo) < 0 ? " FALSE " : " TRUE") << std::endl;
ctx->exit_code = 2;
Expand Down