From fa61d654037aba723d2a43178275cf3eb26227e5 Mon Sep 17 00:00:00 2001 From: Jan Fabian Radon Date: Thu, 2 Feb 2023 21:27:39 +0100 Subject: [PATCH] Develop (#370) * Loglevel Debug is sufficient as it gets retried (#337) * Bugfix/339 rados config timeout (#340) * 339 retry 10 times with random wait. retry read for simple read e.g. rados_config. fail with assert in case we can't find rados_config * Feature/342 object search (#343) * Develop (#331) * #328: fix copy move (#330) * Develop (#334) * 217: fix unused code. * 217: unused code and warning * disabled man pages for rmb * Bugfix/283 virtual mailbox fetch metadata (#284) * Develop (#282) * 217: fix unused code. * 217: unused code and warning * disabled man pages for rmb * #283: fetch metadata for mails in virtual mailbox * #283: fix metadata date.saved. date.received virtual mailbox * #283: version * Feature/286 use guid from UUID string (#287) * #256: use guid_128_from_uuid_string instead of guid_128_from_string, to support older uuid formats like RECORD or MICROSOFT, always use compact for printing * version 0.0.25 preparations * #286: build issue * merge * version * #286: prefere cached mail guid. * #286: determine if uuid has - hyphon, if true preseve it. * fix non void return * Feature/289 GitHub actions (#290) * #289: build * #289: build plugin. * #289: submodules * submodules * upgrade git * clean outdated repos * Bugfix/UUID record format (#293) * bugfix initialisaction rados_mail->deprecated_uid * rados_mail creation and default value check * #295: update index after full object has been moved (metadata) (#296) * Bugfix/298 295 bugfixes (#300) * #289: fix force-resync (always use INBOX) #295: rbox_mail_set_expunge (play save) * #298: move ceph objects the standard way. * clean up log. * #298: log warning if client connection times out. * Feature/302 retry expunge connection timeout (#303) * #302: retry in case of ceph connection timeout * rpm * #302: max_retry = 10 * max 10 retry, in case object not available, fail with error * #304: use last know uid to restore email flags (#305) * #306: re-assign unreferenced mail objects to inbox (#309) * #310: read osd_max_object_size and fail if mail.size > osd_max_object_size (#311) * version: 0.0.31 * #313: fix imap append crash (#314) * version 0.0.32 * Bugfix/316 mailbox save (#317) * #316: imap append (remove index entry twice in case of error) * #316: changed storage for big attachments. (sync write) * #316: operate api * #316: fix operate call * #316: release preparations * #316: fixed some legacy unit tests * 316: review results * #316: fix * Bugfix/test output stream check (#323) * init output stream and free mailbuffer * build * extracted write chunks * mailsize -1; * fix * debug messages * init stream any case * added deprecation comment, writing chunks now default. new config param rbox_chunk_size default 10240 bytes * missing mock class * version * disabled (temporarily storage tests) * disabled storage tests( api change) * Feature/alternative save methods (#326) * #322: alternative save methods * version * Feature/319 force resync immediatelly assign bugfix/328 segmentation fault copy from virtual mailbox (#321) * #319: automatically add lost objects to inbox (if no mailbox guid exist) * #319: version 0.0.34 * catch error in case mailbox does not have obox header: e.v. Virtual Mailbox * check for virtual mailbox when moving or copying, (virtual mailbox not allowed as origin) * version * #328: fix copy move (#330) * Bugfix/332 quota and move (#333) * 291: failed save mock test * 291: fixed most of the tests * 332 fix notify messages number and type * Develop (#336) * 217: fix unused code. * 217: unused code and warning * disabled man pages for rmb * Bugfix/283 virtual mailbox fetch metadata (#284) * Develop (#282) * 217: fix unused code. * 217: unused code and warning * disabled man pages for rmb * #283: fetch metadata for mails in virtual mailbox * #283: fix metadata date.saved. date.received virtual mailbox * #283: version * Feature/286 use guid from UUID string (#287) * #256: use guid_128_from_uuid_string instead of guid_128_from_string, to support older uuid formats like RECORD or MICROSOFT, always use compact for printing * version 0.0.25 preparations * #286: build issue * merge * version * #286: prefere cached mail guid. * #286: determine if uuid has - hyphon, if true preseve it. * fix non void return * Feature/289 GitHub actions (#290) * #289: build * #289: build plugin. * #289: submodules * submodules * upgrade git * clean outdated repos * Bugfix/UUID record format (#293) * bugfix initialisaction rados_mail->deprecated_uid * rados_mail creation and default value check * #295: update index after full object has been moved (metadata) (#296) * Bugfix/298 295 bugfixes (#300) * #289: fix force-resync (always use INBOX) #295: rbox_mail_set_expunge (play save) * #298: move ceph objects the standard way. * clean up log. * #298: log warning if client connection times out. * Feature/302 retry expunge connection timeout (#303) * #302: retry in case of ceph connection timeout * rpm * #302: max_retry = 10 * max 10 retry, in case object not available, fail with error * #304: use last know uid to restore email flags (#305) * #306: re-assign unreferenced mail objects to inbox (#309) * #310: read osd_max_object_size and fail if mail.size > osd_max_object_size (#311) * version: 0.0.31 * #313: fix imap append crash (#314) * version 0.0.32 * Bugfix/316 mailbox save (#317) * #316: imap append (remove index entry twice in case of error) * #316: changed storage for big attachments. (sync write) * #316: operate api * #316: fix operate call * #316: release preparations * #316: fixed some legacy unit tests * 316: review results * #316: fix * Bugfix/test output stream check (#323) * init output stream and free mailbuffer * build * extracted write chunks * mailsize -1; * fix * debug messages * init stream any case * added deprecation comment, writing chunks now default. new config param rbox_chunk_size default 10240 bytes * missing mock class * version * disabled (temporarily storage tests) * disabled storage tests( api change) * Feature/alternative save methods (#326) * #322: alternative save methods * version * Feature/319 force resync immediatelly assign bugfix/328 segmentation fault copy from virtual mailbox (#321) * #319: automatically add lost objects to inbox (if no mailbox guid exist) * #319: version 0.0.34 * catch error in case mailbox does not have obox header: e.v. Virtual Mailbox * check for virtual mailbox when moving or copying, (virtual mailbox not allowed as origin) * version * #328: fix copy move (#330) * Bugfix/332 quota and move (#333) * 291: failed save mock test * 291: fixed most of the tests * 332 fix notify messages number and type * #332: fix append additional error mssage in case connection can't be opend (#335) * Develop (#341) * Loglevel Debug is sufficient as it gets retried (#337) * Bugfix/339 rados config timeout (#340) * 339 retry 10 times with random wait. retry read for simple read e.g. rados_config. fail with assert in case we can't find rados_config * #342: simple multithreading object search * #342: version 0.0.40 Co-authored-by: Ewald Dieterich * Bugfix/342 logging multithreading (#344) * #342 fix threading and additional logs * version * Bugfix/rbox copy context (#347) * 346; fix rbox-copy seg fault always init rbox_mail * #346 fix segmentation fault rbox_copy * #364 and code cleanup * Feature/349 ceph object index (#350) * #349: initial storage api * #349 doveadm rmb create ceph index force resync strategy * allow refresh of ceph object index via -r param * #349 save ceph index objects to separate pool * #349 poc impl ceph index done * #349 new config setting rbox_index_pool_name * #349 force reload of all objects, config setting defines if append of oid to ceph index happens * #349 add , to every entry or entry list * #349 add , to every entry or entry list, and test * repair invalid object not debug level log * #349 changed the remove function, initial index creation: append after each mailbox * #349 fix unit tests * #349 version * #349 bugfix return code doveadm rmb (#351) * #349 validate object (#353) * Bugfix/355 fix gzip trailer for empty stream (#357) * #355 fix write gzip trailer even the stream is empty. * #355 minor improvements for quota messaging * #355 fix gzip trailer for empty stream * #349 fix threshold calc for ceph index file * #355 fix buffersize for write method 1 and 2 (#360) * Bugfix 355 fix buffersize write method (#363) * #355 fix buffersize for write method 1 and 2 * 355 only use append write, clear read buffer in case of read timeout * #355 check for gzip and fix invalid trailer * #355 fix unit tests. * #355 fix call to operate * 355 stream size for uncompressed mails * #355 additional tests * delete ceph index if doveadm mailbox delete -u <> INBOX * Fix/ceph index (#367) * fix ceph index size calculation * delete ceph-index in case its activated * version string * fix ceph-index-file-size calculation * added unit test for object calculation --------- Co-authored-by: Ewald Dieterich --- src/tests/librmb/test_librmb_utils.cpp | 86 ++++++++------------------ 1 file changed, 25 insertions(+), 61 deletions(-) diff --git a/src/tests/librmb/test_librmb_utils.cpp b/src/tests/librmb/test_librmb_utils.cpp index 2f780c4a..85d498b0 100644 --- a/src/tests/librmb/test_librmb_utils.cpp +++ b/src/tests/librmb/test_librmb_utils.cpp @@ -290,57 +290,6 @@ __attribute__((noreturn)) static void *write_to_save_file(void *threadid) { pthread_exit(NULL); } -TEST(librmb, append_to_existing_file_multi_threading) { - std::string test_file_name = "test1.log"; - int rc; - void *status; - pthread_attr_t attr; - pthread_t threads[5]; - // Initialize and set thread joinable - pthread_attr_init(&attr); - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); - - for (uintptr_t i = 0; i < 5; i++) { - rc = pthread_create(&threads[i], NULL, write_to_save_file, (void *)i); - } - sleep(1); - std::cout << " threads created " << std::endl; - // free attribute and wait for the other threads - pthread_attr_destroy(&attr); - for (int i = 0; i < 5; i++) { - rc = pthread_join(threads[i], &status); - if (rc) { - std::cout << "Error:unable to join," << rc << std::endl; - exit(-1); - } - - std::cout << "Main: completed thread id :" << i; - std::cout << " exiting with status :" << status << std::endl; - } - sleep(1); - int line_count = 0; - /** check content **/ - std::ifstream read(test_file_name); - while (true) { - librmb::RadosSaveLogEntry entry; - read >> entry; - - if (read.eof()) { - break; - } - EXPECT_EQ(entry.oid, "abc"); - EXPECT_EQ(entry.ns, "ns_1"); - EXPECT_EQ(entry.pool, "mail_storage"); - EXPECT_EQ(entry.op, "save"); - line_count++; - } - EXPECT_EQ(25, line_count); - read.close(); - std::remove(test_file_name.c_str()); - - std::cout << " exiting main " << std::endl; - // pthread_exit(NULL); -} TEST(librmb, test_mvn_option) { std::list metadata; @@ -377,18 +326,33 @@ TEST(librmb, test_mvn_option) { std::remove(test_file_name.c_str()); } -/*TEST(librmb, test_if) { - int arr[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; +TEST(librmb, test_max_object_size_calculation) { + + bool result = librmb::RadosUtils::object_size_close_to_reach_max(10.0,100.0); + EXPECT_EQ(result, false); + + result = librmb::RadosUtils::object_size_close_to_reach_max(81.0,100.0); + EXPECT_EQ(result, true); + + result = librmb::RadosUtils::object_size_close_to_reach_max(112820433.0 + ,134217728.0); + EXPECT_EQ(result, true); + + result = librmb::RadosUtils::object_size_close_to_reach_max(13646193.0 + ,134217728.0); + EXPECT_EQ(result, false); + + result = librmb::RadosUtils::object_size_close_to_reach_max(13646226.0 + ,134217728.0); + EXPECT_EQ(result, false); + + result = librmb::RadosUtils::object_size_close_to_reach_max(13646259.0 + ,134217728.0); + EXPECT_EQ(result, false); + +} - for (int i = 0; i < 10; ++i) { - std::cout << " value: " << arr[i] << std::endl; - } - for (int i = 0; i < 10; i++) { - std::cout << " value: " << arr[i] << std::endl; - } - EXPECT_EQ(1, 2); -}*/ TEST(librmb, mock_obj) {} int main(int argc, char **argv) { ::testing::InitGoogleMock(&argc, argv);