From 8cf80edb833d08d133c408062c9badf99f049fd3 Mon Sep 17 00:00:00 2001 From: Jan Radon Date: Mon, 16 Oct 2017 17:45:48 +0200 Subject: [PATCH] #93 storage interface - added save_mail --- src/librmb/rados-mail-object.h | 9 +- src/librmb/rados-storage-impl.cpp | 50 +++++- src/librmb/rados-storage-impl.h | 4 + src/librmb/rados-storage.h | 3 + src/librmb/tools/rmb/mailbox_tools.cpp | 2 +- src/librmb/tools/rmb/rmb.cpp | 8 +- src/storage-rbox/rbox-mail.cpp | 2 +- src/storage-rbox/rbox-save.cpp | 154 ++++++++---------- src/storage-rbox/rbox-save.h | 2 + src/tests/Makefile.am | 4 +- src/tests/mocks/mock_test.h | 3 + src/tests/rmb/test_rmb.cpp | 2 +- .../test_storage_mock_rbox.cpp | 20 +-- src/tests/test-utils/it_utils.cpp | 40 ++++- src/tests/test-utils/it_utils.h | 5 + 15 files changed, 190 insertions(+), 118 deletions(-) diff --git a/src/librmb/rados-mail-object.h b/src/librmb/rados-mail-object.h index 51f48f08..96906cf7 100644 --- a/src/librmb/rados-mail-object.h +++ b/src/librmb/rados-mail-object.h @@ -35,20 +35,21 @@ class RadosMailObject { void set_state(const string& _state) { this->state = _state; } void set_version(const string& _version) { this->version = _version; } void set_guid(const uint8_t* guid); - void set_object_size(const uint64_t& _size) { object_size = _size; } + void set_mail_size(const uint64_t& _size) { object_size = _size; } void set_mail_buffer(char* _mail_buffer) { this->mail_buffer = _mail_buffer; } void set_active_op(bool _active) { this->active_op = _active; } void set_rados_save_date(const time_t& _save_date) { this->save_date_rados = _save_date; } const string get_oid() { return this->oid; } const string get_version() { return this->version; } - const uint64_t& get_object_size() { return this->object_size; } + const uint64_t& get_mail_size() { return this->object_size; } time_t* get_rados_save_date() { return &this->save_date_rados; } uint8_t* get_guid_ref() { return guid; } char* get_mail_buffer() { return this->mail_buffer; } map* get_metadata() { return &this->attrset; } - + void set_mail_buffer_content_ptr(const void* start) { mail_buffer_start = start; } + const void* get_mail_buffer_content_ptr() { return mail_buffer_start; } map* get_completion_op_map() { return &completion_op; } string get_metadata(rbox_metadata_key key) { string str_key(1, static_cast(key)); @@ -64,6 +65,7 @@ class RadosMailObject { bool has_active_op() { return active_op; } string to_string(const string& padding); + void add_metadata(RadosMetadata& metadata) { attrset[metadata.key] = metadata.bl; } private: string oid; @@ -74,6 +76,7 @@ class RadosMailObject { uint8_t guid[GUID_128_SIZE]; uint64_t object_size; // byte map completion_op; + const void* mail_buffer_start; bool active_op; // used as pointer to a buffer_t (to avoid using dovecot datatypes in library) diff --git a/src/librmb/rados-storage-impl.cpp b/src/librmb/rados-storage-impl.cpp index 5b3f6524..50ad9110 100644 --- a/src/librmb/rados-storage-impl.cpp +++ b/src/librmb/rados-storage-impl.cpp @@ -26,7 +26,7 @@ const char *RadosStorageImpl::CFG_OSD_MAX_WRITE_SIZE = "osd_max_write_size"; RadosStorageImpl::RadosStorageImpl(RadosCluster *_cluster) { cluster = _cluster; - max_write_size = 0; + max_write_size = 10; } RadosStorageImpl::~RadosStorageImpl() {} @@ -178,6 +178,25 @@ bool RadosStorageImpl::wait_for_write_operations_complete( return failed; } +bool RadosStorageImpl::wait_for_rados_operations(const std::vector &object_list) { + bool ctx_failed = false; + // wait for all writes to finish! + // imaptest shows it's possible that begin -> continue -> finish cycle is invoked several times before + // rbox_transaction_save_commit_pre is called. + for (std::vector::const_iterator it_cur_obj = object_list.begin(); + it_cur_obj != object_list.end(); ++it_cur_obj) { + // if we come from copy mail, there is no operation to wait for. + if ((*it_cur_obj)->has_active_op()) { + bool op_failed = wait_for_write_operations_complete((*it_cur_obj)->get_completion_op_map()); + + ctx_failed = ctx_failed ? ctx_failed : op_failed; + (*it_cur_obj)->get_completion_op_map()->clear(); + (*it_cur_obj)->set_active_op(false); + } + } + return ctx_failed; +} + bool RadosStorageImpl::update_metadata(std::string oid, std::list &to_update) { librados::ObjectWriteOperation write_op; librados::AioCompletion *completion = librados::Rados::aio_create_completion(); @@ -276,5 +295,32 @@ bool RadosStorageImpl::copy(std::string &src_oid, const char *src_ns, std::strin return ret == 0; } +// if save_async = true, don't forget to call wait_for_rados_operations e.g. wait_for_write_operations_complete +// to wait for completion and free resources. +bool RadosStorageImpl::save_mail(RadosMailObject *mail, bool &save_async) { + // delete write_op_xattr is called after operation completes (wait_for_rados_operations) + librados::ObjectWriteOperation *write_op_xattr = new librados::ObjectWriteOperation(); + int ret = -1; + // set metadata + for (std::map::iterator it = mail->get_metadata()->begin(); + it != mail->get_metadata()->end(); ++it) { + write_op_xattr->setxattr(it->first.c_str(), it->second); + } + write_op_xattr->mtime(mail->get_rados_save_date()); + + ret = split_buffer_and_exec_op(reinterpret_cast(mail->get_mail_buffer_content_ptr()), + mail->get_mail_size(), mail, write_op_xattr, get_max_write_size_bytes()); + mail->set_active_op(true); + if (!save_async) { + std::vector objects; + objects.push_back(mail); + return wait_for_rados_operations(objects); + } + if (ret != 0) { + write_op_xattr->remove(); + delete write_op_xattr; + mail->set_active_op(false); + } - + return ret == 0; +} diff --git a/src/librmb/rados-storage-impl.h b/src/librmb/rados-storage-impl.h index 536a686f..a6da6661 100644 --- a/src/librmb/rados-storage-impl.h +++ b/src/librmb/rados-storage-impl.h @@ -53,6 +53,8 @@ class RadosStorageImpl : public RadosStorage { bool wait_for_write_operations_complete( std::map *completion_op_map); + bool wait_for_rados_operations(const std::vector &object_list); + int read_mail(librados::bufferlist *buffer, const std::string &oid); bool update_metadata(std::string oid, std::list &to_update); bool move(std::string &src_oid, const char *src_ns, std::string &dest_oid, const char *dest_ns, @@ -60,6 +62,8 @@ class RadosStorageImpl : public RadosStorage { bool copy(std::string &src_oid, const char *src_ns, std::string &dest_oid, const char *dest_ns, std::list &to_update); + bool save_mail(RadosMailObject *mail, bool &save_async); + private: RadosCluster *cluster; int max_write_size; diff --git a/src/librmb/rados-storage.h b/src/librmb/rados-storage.h index 1b1840c0..ccd27ed2 100644 --- a/src/librmb/rados-storage.h +++ b/src/librmb/rados-storage.h @@ -47,12 +47,15 @@ class RadosStorage { virtual int open_connection(const std::string &poolname, const std::string &ns) = 0; virtual bool wait_for_write_operations_complete( std::map* completion_op_map) = 0; + virtual bool wait_for_rados_operations(const std::vector &object_list) = 0; + virtual int read_mail(librados::bufferlist *buffer, const std::string &oid) = 0; virtual bool update_metadata(std::string oid, std::list &to_update) = 0; virtual bool move(std::string &src_oid, const char *src_ns, std::string &dest_oid, const char *dest_ns, std::list &to_update, bool delete_source) = 0; virtual bool copy(std::string &src_oid, const char *src_ns, std::string &dest_oid, const char *dest_ns, std::list &to_update) = 0; + virtual bool save_mail(RadosMailObject *mail, bool &save_async) = 0; }; } // namespace librmb diff --git a/src/librmb/tools/rmb/mailbox_tools.cpp b/src/librmb/tools/rmb/mailbox_tools.cpp index 25857f56..e7808929 100644 --- a/src/librmb/tools/rmb/mailbox_tools.cpp +++ b/src/librmb/tools/rmb/mailbox_tools.cpp @@ -89,7 +89,7 @@ int MailboxTools::save_mail(librmb::RadosMailObject* mail_obj) { if (!myfile.is_open()) { return -1; } - myfile.write(mail_obj->get_mail_buffer(), mail_obj->get_object_size()); + myfile.write(mail_obj->get_mail_buffer(), mail_obj->get_mail_size()); myfile.close(); return 0; } diff --git a/src/librmb/tools/rmb/rmb.cpp b/src/librmb/tools/rmb/rmb.cpp index e2d9d9c2..0a1e1d8c 100644 --- a/src/librmb/tools/rmb/rmb.cpp +++ b/src/librmb/tools/rmb/rmb.cpp @@ -170,12 +170,12 @@ static void query_mail_storage(std::vector *mail_obje } if (mailbox.count(mailbox_guid) > 0) { mailbox[mailbox_guid]->add_mail((*it)); - mailbox[mailbox_guid]->add_to_mailbox_size((*it)->get_object_size()); + mailbox[mailbox_guid]->add_to_mailbox_size((*it)->get_mail_size()); } else { mailbox[mailbox_guid] = new librmb::RadosMailBox(mailbox_guid, 1, mailbox_orig_name); mailbox[mailbox_guid]->set_xattr_filter(parser); mailbox[mailbox_guid]->add_mail((*it)); - mailbox[mailbox_guid]->add_to_mailbox_size((*it)->get_object_size()); + mailbox[mailbox_guid]->add_to_mailbox_size((*it)->get_mail_size()); } } std::cout << "mailbox_count: " << mailbox.size() << std::endl; @@ -207,7 +207,7 @@ static void query_mail_storage(std::vector *mail_obje char *mail_buffer = new char[size_r + 1]; (*it_mail)->set_mail_buffer(mail_buffer); - (*it_mail)->set_object_size(size_r); + (*it_mail)->set_mail_size(size_r); int read = storage->read_mail(&buffer, oid); if (read > 0) { memcpy(mail_buffer, buffer.to_str().c_str(), read + 1); @@ -285,7 +285,7 @@ static void load_objects(librmb::RadosStorageImpl &storage, std::vectorget_oid(), &object_size, &save_date_rados); - mail->set_object_size(object_size); + mail->set_mail_size(object_size); mail->set_rados_save_date(save_date_rados); ++iter; mail_objects.push_back(mail); diff --git a/src/storage-rbox/rbox-mail.cpp b/src/storage-rbox/rbox-mail.cpp index 5572cbf1..12c075e0 100644 --- a/src/storage-rbox/rbox-mail.cpp +++ b/src/storage-rbox/rbox-mail.cpp @@ -90,7 +90,7 @@ int rbox_get_index_record(struct mail *_mail) { rmail->mail_object->get_metadata()->clear(); } uint64_t obj_size = -1; - rmail->mail_object->set_object_size(obj_size); + rmail->mail_object->set_mail_size(obj_size); FUNC_END(); return 0; } diff --git a/src/storage-rbox/rbox-save.cpp b/src/storage-rbox/rbox-save.cpp index 420a6c27..97c7af89 100644 --- a/src/storage-rbox/rbox-save.cpp +++ b/src/storage-rbox/rbox-save.cpp @@ -156,6 +156,34 @@ void rbox_move_index(struct mail_save_context *_ctx, struct mail *src_mail) { mail_set_seq_saving(_ctx->dest_mail, r_ctx->seq); } } +void init_output_stream(mail_save_context *_ctx) { + rbox_save_context *r_ctx = (struct rbox_save_context *)_ctx; + if (_ctx->data.output != NULL) { + o_stream_unref(&_ctx->data.output); + } + + r_ctx->output_stream = o_stream_create_buffer(reinterpret_cast(r_ctx->current_object->get_mail_buffer())); + o_stream_cork(r_ctx->output_stream); + _ctx->data.output = r_ctx->output_stream; +} + +int allocate_mail_buffer(mail_save_context *_ctx, int &initial_mail_buffer_size) { + rbox_save_context *r_ctx = (struct rbox_save_context *)_ctx; + librmb::RadosMailObject *mail = r_ctx->current_object; + char *current_mail_buffer = mail->get_mail_buffer(); + + if (current_mail_buffer != NULL) { + buffer_t *buffer = reinterpret_cast(current_mail_buffer); + // make 100% sure, buffer is empty! + buffer_free(&buffer); + } + mail->set_mail_buffer(reinterpret_cast(buffer_create_dynamic(default_pool, initial_mail_buffer_size))); + + if (mail->get_mail_buffer() == NULL) { + return -1; + } + return 0; +} int rbox_save_begin(struct mail_save_context *_ctx, struct istream *input) { FUNC_START(); @@ -170,38 +198,25 @@ int rbox_save_begin(struct mail_save_context *_ctx, struct istream *input) { } rbox_add_to_index(_ctx); - i_debug("SAVE OID: %s, %d uid, seq=%d", guid_128_to_string(r_ctx->mail_oid), _ctx->dest_mail->uid, r_ctx->seq); - // currently it is required to have a input stream, also when we copy or move - // a mail. (this is due to plugins like zlib, which manipulate input and output stream). mail_set_seq_saving(_ctx->dest_mail, r_ctx->seq); crlf_input = i_stream_create_crlf(input); r_ctx->input = index_mail_cache_parse_init(_ctx->dest_mail, crlf_input); i_stream_unref(&crlf_input); - if (r_ctx->current_object->get_mail_buffer() != NULL) { - buffer_t *buffer = reinterpret_cast(r_ctx->current_object->get_mail_buffer()); - // make 100% sure, buffer is empty! - buffer_free(&buffer); - } - r_ctx->current_object->set_mail_buffer( - reinterpret_cast(buffer_create_dynamic(default_pool, initial_mail_buffer_size))); - - if (r_ctx->current_object->get_mail_buffer() == NULL) { + int ret = allocate_mail_buffer(_ctx, initial_mail_buffer_size); + if (ret < 0) { FUNC_END_RET("ret == -1"); return -1; } - if (_ctx->data.output != NULL) { - o_stream_unref(&_ctx->data.output); - } - r_ctx->output_stream = o_stream_create_buffer(reinterpret_cast(r_ctx->current_object->get_mail_buffer())); - o_stream_cork(r_ctx->output_stream); - _ctx->data.output = r_ctx->output_stream; + init_output_stream(_ctx); if (_ctx->data.received_date == (time_t)-1) _ctx->data.received_date = ioloop_time; + i_debug("SAVE OID: %s, %d uid, seq=%d", guid_128_to_string(r_ctx->mail_oid), _ctx->dest_mail->uid, r_ctx->seq); + FUNC_END(); return 0; } @@ -236,45 +251,43 @@ int rbox_save_continue(struct mail_save_context *_ctx) { return 0; } -static int rbox_save_mail_write_metadata(struct rbox_save_context *ctx, librados::ObjectWriteOperation *write_op_xattr, - const uoff_t &output_msg_size) { +static int rbox_save_mail_set_metadata(struct rbox_save_context *ctx, librmb::RadosMailObject *mail_object) { FUNC_START(); struct mail_save_data *mdata = &ctx->ctx.data; + uoff_t &output_msg_size = ctx->output_stream->offset; { - RadosMetadata xattr(rbox_metadata_key::RBOX_METADATA_VERSION, RadosMailObject::X_ATTR_VERSION_VALUE); - write_op_xattr->setxattr(xattr.key.c_str(), xattr.bl); + mail_object->add_metadata(xattr); } { RadosMetadata xattr(rbox_metadata_key::RBOX_METADATA_MAILBOX_GUID, guid_128_to_string(ctx->mbox->mailbox_guid)); - - write_op_xattr->setxattr(xattr.key.c_str(), xattr.bl); + mail_object->add_metadata(xattr); } { RadosMetadata xattr(rbox_metadata_key::RBOX_METADATA_GUID, guid_128_to_string(ctx->mail_guid)); - write_op_xattr->setxattr(xattr.key.c_str(), xattr.bl); + mail_object->add_metadata(xattr); } { RadosMetadata xattr(rbox_metadata_key::RBOX_METADATA_RECEIVED_TIME, mdata->received_date); - write_op_xattr->setxattr(xattr.key.c_str(), xattr.bl); + mail_object->add_metadata(xattr); } { if (mdata->pop3_uidl != NULL) { RadosMetadata xattr(rbox_metadata_key::RBOX_METADATA_POP3_UIDL, mdata->pop3_uidl); - write_op_xattr->setxattr(xattr.key.c_str(), xattr.bl); + mail_object->add_metadata(xattr); } } { if (mdata->pop3_order != 0) { RadosMetadata xattr(rbox_metadata_key::RBOX_METADATA_POP3_ORDER, mdata->pop3_order); - write_op_xattr->setxattr(xattr.key.c_str(), xattr.bl); + mail_object->add_metadata(xattr); } } { if (mdata->from_envelope != NULL) { RadosMetadata xattr(rbox_metadata_key::RBOX_METADATA_FROM_ENVELOPE, mdata->from_envelope); - write_op_xattr->setxattr(xattr.key.c_str(), xattr.bl); + mail_object->add_metadata(xattr); } } { @@ -284,64 +297,40 @@ static int rbox_save_mail_write_metadata(struct rbox_save_context *ctx, librados vsize = ctx->input->v_offset; } librmb::RadosMetadata xattr(rbox_metadata_key::RBOX_METADATA_VIRTUAL_SIZE, vsize); - write_op_xattr->setxattr(xattr.key.c_str(), xattr.bl); + mail_object->add_metadata(xattr); } { i_debug("ctx->input->v_offset != output_msg_size %ld vs %ld ", ctx->input->v_offset, output_msg_size); librmb::RadosMetadata xattr(rbox_metadata_key::RBOX_METADATA_PHYSICAL_SIZE, ctx->input->v_offset); - write_op_xattr->setxattr(xattr.key.c_str(), xattr.bl); + mail_object->add_metadata(xattr); } { std::string flags = std::to_string(mdata->flags); RadosMetadata xattr(rbox_metadata_key::RBOX_METADATA_OLDV1_FLAGS, flags); - write_op_xattr->setxattr(xattr.key.c_str(), xattr.bl); + mail_object->add_metadata(xattr); } { std::string pvt_flags = std::to_string(mdata->pvt_flags); RadosMetadata xattr(rbox_metadata_key::RBOX_METADATA_PVT_FLAGS, pvt_flags); - write_op_xattr->setxattr(xattr.key.c_str(), xattr.bl); + mail_object->add_metadata(xattr); } { RadosMetadata xattr(rbox_metadata_key::RBOX_METADATA_ORIG_MAILBOX, ctx->mbox->box.name); - write_op_xattr->setxattr(xattr.key.c_str(), xattr.bl); + mail_object->add_metadata(xattr); } i_debug("save_date %s", std::ctime(&mdata->save_date)); - write_op_xattr->mtime(&mdata->save_date); + mail_object->set_rados_save_date(mdata->save_date); FUNC_END(); return 0; } -static bool wait_for_rados_operations(struct rbox_storage *r_storage, - const std::vector &object_list) { - bool ctx_failed = false; - // wait for all writes to finish! - // imaptest shows it's possible that begin -> continue -> finish cycle is invoked several times before - // rbox_transaction_save_commit_pre is called. - for (std::vector::const_iterator it_cur_obj = object_list.begin(); - it_cur_obj != object_list.end(); ++it_cur_obj) { - // if we come from copy mail, there is no operation to wait for. - if ((*it_cur_obj)->has_active_op()) { - bool op_failed = r_storage->s->wait_for_write_operations_complete((*it_cur_obj)->get_completion_op_map()); - - ctx_failed = ctx_failed ? ctx_failed : op_failed; - - // ctx_failed = (*it_cur_obj)->wait_for_write_operations_complete(); - i_debug("OID %s, SAVED success=%s", (*it_cur_obj)->get_oid().c_str(), - ctx_failed ? "false" : "true"); //, file_size); - (*it_cur_obj)->get_completion_op_map()->clear(); - (*it_cur_obj)->set_active_op(false); - } - } - return ctx_failed; -} - static void clean_up_failed(struct rbox_save_context *r_ctx) { struct rbox_storage *r_storage = (struct rbox_storage *)&r_ctx->mbox->storage->storage; - wait_for_rados_operations(r_storage, r_ctx->objects); + r_storage->s->wait_for_rados_operations(r_ctx->objects); for (std::vector::iterator it_cur_obj = r_ctx->objects.begin(); it_cur_obj != r_ctx->objects.end(); ++it_cur_obj) { @@ -372,8 +361,6 @@ int rbox_save_finish(struct mail_save_context *_ctx) { FUNC_START(); struct rbox_save_context *r_ctx = (struct rbox_save_context *)_ctx; struct rbox_storage *r_storage = (struct rbox_storage *)&r_ctx->mbox->storage->storage; - int ret = 0; - int max_write_size = -1; r_ctx->finished = TRUE; if (!r_ctx->failed) { @@ -392,39 +379,28 @@ int rbox_save_finish(struct mail_save_context *_ctx) { // reset virtual size index_mail_cache_parse_deinit(_ctx->dest_mail, r_ctx->ctx.data.received_date, !r_ctx->failed); - // delete write_op_xattr is called after operation completes (wait_for_rados_operations) - librados::ObjectWriteOperation *write_op_xattr = new librados::ObjectWriteOperation(); - - buffer_t *mail_buffer = reinterpret_cast(r_ctx->current_object->get_mail_buffer()); - size_t total_size = buffer_get_size(mail_buffer); - - size_t write_buffer_size = buffer_get_used_size(mail_buffer); - - i_debug("oid: %s, save_date: %s, mail_size %lu, total_buf size %lu", r_ctx->current_object->get_oid().c_str(), - std::ctime(&_ctx->data.save_date), write_buffer_size, total_size); - - rbox_save_mail_write_metadata(r_ctx, write_op_xattr, r_ctx->output_stream->offset); - if (rbox_open_rados_connection(_ctx->transaction->box) < 0) { i_debug("ERROR, cannot open rados connection (rbox_save_finish)"); r_ctx->failed = true; } else { - max_write_size = r_storage->s->get_max_write_size_bytes(); - if (max_write_size <= 0) { - r_ctx->failed = true; + bool async_write = true; + // build rbox_mail_object + buffer_t *mail_buffer = reinterpret_cast(r_ctx->current_object->get_mail_buffer()); + r_ctx->current_object->set_mail_buffer_content_ptr(mail_buffer->data); + r_ctx->current_object->set_mail_size(buffer_get_used_size(mail_buffer)); + + rbox_save_mail_set_metadata(r_ctx, r_ctx->current_object); + // save mail + r_ctx->failed = !r_storage->s->save_mail(r_ctx->current_object, async_write); + + if (r_ctx->failed) { + i_debug("saved mail: %s failed metadata_count %lu", r_ctx->current_object->get_oid().c_str(), + r_ctx->current_object->get_metadata()->size()); } else { - ret = - r_storage->s->split_buffer_and_exec_op(reinterpret_cast(mail_buffer->data), write_buffer_size, - r_ctx->current_object, write_op_xattr, max_write_size); - r_ctx->current_object->set_active_op(true); - r_ctx->failed = ret < 0; + i_debug("saved mail: %s successful metadata_count %lu", r_ctx->current_object->get_oid().c_str(), + r_ctx->current_object->get_metadata()->size()); } } - - if (r_ctx->failed == true) { - write_op_xattr->remove(); - delete write_op_xattr; - } } clean_up_write_finish(_ctx); @@ -477,7 +453,7 @@ int rbox_transaction_save_commit_pre(struct mail_save_context *_ctx) { i_assert(r_ctx->finished); - r_ctx->failed = wait_for_rados_operations(r_storage, r_ctx->objects); + r_ctx->failed = r_storage->s->wait_for_rados_operations(r_ctx->objects); // if one write fails! all writes will be reverted and r_ctx->failed is true! if (r_ctx->failed) { diff --git a/src/storage-rbox/rbox-save.h b/src/storage-rbox/rbox-save.h index 9879d84b..4e23ebbd 100644 --- a/src/storage-rbox/rbox-save.h +++ b/src/storage-rbox/rbox-save.h @@ -67,5 +67,7 @@ class rbox_save_context { void rbox_add_to_index(struct mail_save_context *_ctx); void rbox_move_index(struct mail_save_context *_ctx, struct mail *src_mail); +void init_output_stream(mail_save_context *_ctx); +int allocate_mail_buffer(mail_save_context *_ctx, int &initial_mail_buffer_size); #endif // SRC_STORAGE_RBOX_RBOX_SAVE_H_ diff --git a/src/tests/Makefile.am b/src/tests/Makefile.am index be60a57a..a3dbc0bf 100644 --- a/src/tests/Makefile.am +++ b/src/tests/Makefile.am @@ -53,9 +53,9 @@ test_rmb_SOURCES = rmb/test_rmb.cpp mocks/mock_test.h test_rmb_LDADD = $(rmb_shlibs) $(top_builddir)/src/librmb/tools/rmb/ls_cmd_parser.o $(top_builddir)/src/librmb/tools/rmb/mailbox_tools.o $(gtest_shlibs) TESTS += test_storage_mock_rbox -test_storage_mock_rbox_SOURCES = storage-mock-rbox/test_storage_mock_rbox.cpp storage-mock-rbox/TestCase.cpp storage-mock-rbox/TestCase.h mocks/mock_test.h +test_storage_mock_rbox_SOURCES = storage-mock-rbox/test_storage_mock_rbox.cpp storage-mock-rbox/TestCase.cpp storage-mock-rbox/TestCase.h mocks/mock_test.h test-utils/it_utils.cpp test_storage_mock_rbox_CPPFLAGS = $(AM_CPPFLAGS) $(LIBDOVECOT_INCLUDE) -test_storage_mock_rbox_LDADD = $(storage_shlibs) $(gtest_shlibs) +test_storage_mock_rbox_LDADD = $(storage_shlibs) $(gtest_shlibs) libtest_utils.a TESTS += test_librmb_utils test_librmb_utils_SOURCES = librmb/test_librmb_utils.cpp diff --git a/src/tests/mocks/mock_test.h b/src/tests/mocks/mock_test.h index a1c71d59..91169993 100644 --- a/src/tests/mocks/mock_test.h +++ b/src/tests/mocks/mock_test.h @@ -43,12 +43,15 @@ class RadosStorageMock : public RadosStorage { MOCK_METHOD2(open_connection, int(const std::string &poolname, const std::string &ns)); MOCK_METHOD1(wait_for_write_operations_complete, bool(std::map *completion_op_map)); + MOCK_METHOD1(wait_for_rados_operations, bool(const std::vector &object_list)); + MOCK_METHOD2(read_mail, int(librados::bufferlist *buffer, const std::string &oid)); MOCK_METHOD2(update_metadata, bool(std::string oid, std::list &to_update)); MOCK_METHOD6(move, bool(std::string &src_oid, const char *src_ns, std::string &dest_oid, const char *dest_ns, std::list &to_update, bool delete_source)); MOCK_METHOD5(copy, bool(std::string &src_oid, const char *src_ns, std::string &dest_oid, const char *dest_ns, std::list &to_update)); + MOCK_METHOD2(save_mail, bool(RadosMailObject *mail, bool &save_async)); }; using librmb::RadosDictionary; diff --git a/src/tests/rmb/test_rmb.cpp b/src/tests/rmb/test_rmb.cpp index 573e07f6..d1f966a7 100644 --- a/src/tests/rmb/test_rmb.cpp +++ b/src/tests/rmb/test_rmb.cpp @@ -94,7 +94,7 @@ TEST(rmb1, save_mail) { mail.set_oid(mail_guid); mail.set_mail_buffer(&mail_content[0u]); uint64_t size = mail_content.length(); - mail.set_object_size(size); + mail.set_mail_size(size); int save = tools.save_mail(&mail); EXPECT_EQ(0, save); diff --git a/src/tests/storage-mock-rbox/test_storage_mock_rbox.cpp b/src/tests/storage-mock-rbox/test_storage_mock_rbox.cpp index af78c571..9520b0ac 100644 --- a/src/tests/storage-mock-rbox/test_storage_mock_rbox.cpp +++ b/src/tests/storage-mock-rbox/test_storage_mock_rbox.cpp @@ -30,10 +30,12 @@ extern "C" { #include "mailbox-list.h" #include "ioloop.h" #include "istream.h" +#include "mail-search-build.h" #include "libdict-rados-plugin.h" } #include "dovecot-ceph-plugin-config.h" +#include "../test-utils/it_utils.h" #include "rbox-storage.hpp" #include "../mocks/mock_test.h" @@ -54,6 +56,7 @@ using ::testing::_; #define i_zero(p) memset(p, 0, sizeof(*(p))) #endif + TEST_F(StorageTest, init) {} TEST_F(StorageTest, mail_save_to_inbox_storage_mock_no_rados_available) { @@ -118,7 +121,6 @@ TEST_F(StorageTest, mail_save_to_inbox_storage_mock_no_rados_available) { FAIL() << "Save transaction commit failed: " << mailbox_get_last_internal_error(box, NULL); } else { ret = 0; - // FAIL() << "connection is not open: " << mailbox_get_last_internal_error(box, NULL); } EXPECT_EQ(save_ctx, nullptr); @@ -165,11 +167,8 @@ TEST_F(StorageTest, exec_write_op_fails) { struct rbox_storage *storage = (struct rbox_storage *)box->storage; delete storage->s; librmbtest::RadosStorageMock *storage_mock = new librmbtest::RadosStorageMock(); - // first call to open_connection will fail! EXPECT_CALL(*storage_mock, open_connection("mail_storage", "user-rbox-test")).Times(AtLeast(1)).WillOnce(Return(0)); - - EXPECT_CALL(*storage_mock, get_max_write_size_bytes()).Times(AtLeast(1)).WillOnce(Return(10)); - EXPECT_CALL(*storage_mock, split_buffer_and_exec_op(_, _, _, _, _)).Times(AtLeast(1)).WillRepeatedly(Return(-1)); + EXPECT_CALL(*storage_mock, save_mail(_, _)).Times(1).WillOnce(Return(false)); storage->s = storage_mock; ssize_t ret; @@ -201,7 +200,6 @@ TEST_F(StorageTest, exec_write_op_fails) { FAIL() << "Save transaction commit failed: " << mailbox_get_last_internal_error(box, NULL); } else { ret = 0; - // FAIL() << "connection is not open: " << mailbox_get_last_internal_error(box, NULL); } EXPECT_EQ(save_ctx, nullptr); @@ -248,12 +246,10 @@ TEST_F(StorageTest, write_op_fails) { struct rbox_storage *storage = (struct rbox_storage *)box->storage; delete storage->s; librmbtest::RadosStorageMock *storage_mock = new librmbtest::RadosStorageMock(); - // first call to open_connection will fail! - EXPECT_CALL(*storage_mock, open_connection("mail_storage", "user-rbox-test")).Times(AtLeast(1)).WillOnce(Return(0)); - - EXPECT_CALL(*storage_mock, get_max_write_size_bytes()).Times(AtLeast(1)).WillOnce(Return(10)); - EXPECT_CALL(*storage_mock, wait_for_write_operations_complete(_)).Times(AtLeast(1)).WillOnce(Return(true)); + EXPECT_CALL(*storage_mock, open_connection("mail_storage", "user-rbox-test")).Times(AtLeast(1)).WillOnce(Return(0)); + EXPECT_CALL(*storage_mock, save_mail(_, _)).Times(1).WillOnce(Return(true)); + EXPECT_CALL(*storage_mock, wait_for_rados_operations(_)).Times(AtLeast(1)).WillOnce(Return(false)); storage->s = storage_mock; ssize_t ret; @@ -285,7 +281,6 @@ TEST_F(StorageTest, write_op_fails) { SUCCEED() << "should fail here"; } else { ret = 0; - // FAIL() << "connection is not open: " << mailbox_get_last_internal_error(box, NULL); } EXPECT_EQ(save_ctx, nullptr); @@ -303,6 +298,7 @@ TEST_F(StorageTest, write_op_fails) { mailbox_free(&box); } + TEST_F(StorageTest, deinit) {} int main(int argc, char **argv) { diff --git a/src/tests/test-utils/it_utils.cpp b/src/tests/test-utils/it_utils.cpp index 9d3ca7bc..6f09e646 100644 --- a/src/tests/test-utils/it_utils.cpp +++ b/src/tests/test-utils/it_utils.cpp @@ -31,7 +31,33 @@ ItUtils::ItUtils() { ItUtils::~ItUtils() { } +void ItUtils::add_mail(const char *message, const char *mailbox, struct mail_namespace *_ns, + librmb::RadosStorage *storage_impl) { + struct mail_namespace *ns = mail_namespace_find_inbox(_ns); + ASSERT_NE(ns, nullptr); + struct mailbox *box = mailbox_alloc(ns->list, mailbox, (mailbox_flags)0); + ASSERT_NE(box, nullptr); + ASSERT_GE(mailbox_open(box), 0); + + struct istream *input = i_stream_create_from_data(message, strlen(message)); + +#ifdef DOVECOT_CEPH_PLUGIN_HAVE_MAIL_STORAGE_TRANSACTION_OLD_SIGNATURE + struct mailbox_transaction_context *trans = mailbox_transaction_begin(box, MAILBOX_TRANSACTION_FLAG_EXTERNAL); +#else + char reason[256]; + struct mailbox_transaction_context *trans = mailbox_transaction_begin(box, MAILBOX_TRANSACTION_FLAG_EXTERNAL, reason); +#endif + struct mail_save_context *save_ctx = mailbox_save_alloc(trans); + struct rbox_storage *storage = (struct rbox_storage *)box->storage; + delete storage->s; + storage->s = storage_impl; + + ItUtils::add_mail(save_ctx, input, box, trans); + + i_stream_unref(&input); + mailbox_free(&box); +} void ItUtils::add_mail(const char *message, const char *mailbox, struct mail_namespace *_ns) { struct mail_namespace *ns = mail_namespace_find_inbox(_ns); ASSERT_NE(ns, nullptr); @@ -48,6 +74,16 @@ void ItUtils::add_mail(const char *message, const char *mailbox, struct mail_nam struct mailbox_transaction_context *trans = mailbox_transaction_begin(box, MAILBOX_TRANSACTION_FLAG_EXTERNAL, reason); #endif struct mail_save_context *save_ctx = mailbox_save_alloc(trans); + + + ItUtils::add_mail(save_ctx, input, box, trans); + + i_stream_unref(&input); + mailbox_free(&box); +} + +void ItUtils::add_mail(struct mail_save_context *save_ctx, struct istream *input, struct mailbox *box, + struct mailbox_transaction_context *trans) { ssize_t ret; bool save_failed = FALSE; @@ -88,9 +124,7 @@ void ItUtils::add_mail(const char *message, const char *mailbox, struct mail_nam EXPECT_TRUE(input->eof); EXPECT_GE(ret, 0); - } - i_stream_unref(&input); - mailbox_free(&box); + } } } /* namespace testutils */ diff --git a/src/tests/test-utils/it_utils.h b/src/tests/test-utils/it_utils.h index 49583238..477605c5 100644 --- a/src/tests/test-utils/it_utils.h +++ b/src/tests/test-utils/it_utils.h @@ -31,6 +31,7 @@ extern "C" { #include "mail-search-parser-private.h" #include "mail-search.h" } +#include "rbox-storage.hpp" #pragma GCC diagnostic pop #if DOVECOT_PREREQ(2, 3) @@ -50,6 +51,10 @@ class ItUtils { ItUtils(); virtual ~ItUtils(); static void add_mail(const char *message, const char *mailbox, struct mail_namespace *_ns); + static void add_mail(const char *message, const char *mailbox, struct mail_namespace *_ns, + librmb::RadosStorage *storage_impl); + static void add_mail(struct mail_save_context *save_ctx, struct istream *input, struct mailbox *box, + struct mailbox_transaction_context *trans); }; } /* namespace testutils */