Develop (#370)

* Loglevel Debug is sufficient as it gets retried (#337)

* Bugfix/339 rados config timeout (#340)

* 339 retry 10 times with random wait. retry read for simple read e.g. rados_config. fail with assert in case we can't find rados_config

* Feature/342 object search (#343)

* Develop (#331)

* #328: fix copy move (#330)

* Develop (#334)

* 217: fix unused code.

* 217: unused code and warning

* disabled man pages for rmb

* Bugfix/283 virtual mailbox fetch metadata (#284)

* Develop (#282)

* 217: fix unused code.

* 217: unused code and warning

* disabled man pages for rmb

* #283: fetch metadata for mails in virtual mailbox

* #283: fix metadata date.saved. date.received virtual mailbox

* #283: version

* Feature/286 use guid from UUID string (#287)

* #256: use guid_128_from_uuid_string instead of guid_128_from_string, to support older uuid formats like RECORD or MICROSOFT, always use compact for printing

* version 0.0.25 preparations

* #286: build issue

* merge

* version

* #286: prefere cached mail guid.

* #286: determine if uuid has - hyphon, if true preseve it.

* fix non void return

* Feature/289 GitHub actions (#290)

* #289: build

* #289: build plugin.

* #289: submodules

* submodules

* upgrade git

* clean outdated repos

* Bugfix/UUID record format (#293)

* bugfix initialisaction rados_mail->deprecated_uid

* rados_mail creation and default value check

* #295: update index after full object has been moved (metadata) (#296)

* Bugfix/298 295 bugfixes (#300)

* #289: fix force-resync (always use INBOX)
#295: rbox_mail_set_expunge (play save)

* #298: move ceph objects the standard way.

* clean up log.

* #298: log warning if client connection times out.

* Feature/302 retry expunge connection timeout (#303)

* #302: retry in case of ceph connection timeout

* rpm

* #302: max_retry = 10

* max 10 retry, in case object not available, fail with error

* #304: use last know uid to restore email flags (#305)

* #306: re-assign unreferenced mail objects to inbox (#309)

* #310: read osd_max_object_size and fail if mail.size > osd_max_object_size (#311)

* version: 0.0.31

* #313: fix imap append crash (#314)

* version 0.0.32

* Bugfix/316 mailbox save (#317)

* #316: imap append (remove index entry twice in case of error)

* #316: changed storage for big attachments. (sync write)

* #316: operate api

* #316:  fix operate call

* #316: release preparations

* #316: fixed some legacy unit tests

* 316: review results

* #316: fix

* Bugfix/test output stream check (#323)

* init output stream and free mailbuffer

* build

* extracted write chunks

* mailsize -1;

* fix

* debug messages

* init stream any case

* added deprecation comment, writing chunks now default. new config param rbox_chunk_size default 10240 bytes

* missing mock class

* version

* disabled (temporarily storage tests)

* disabled storage tests( api change)

* Feature/alternative save methods (#326)

* #322: alternative save methods

* version

* Feature/319 force resync immediatelly assign bugfix/328 segmentation fault copy from virtual mailbox (#321)

* #319: automatically add lost objects to inbox (if no mailbox guid exist)

* #319: version 0.0.34

* catch error in case mailbox does not have obox header: e.v. Virtual Mailbox

* check for virtual mailbox when moving or copying, (virtual mailbox not allowed as origin)

* version

* #328: fix copy move (#330)

* Bugfix/332 quota and move (#333)

* 291: failed save mock test

* 291: fixed most of the tests

* 332 fix notify messages number and type

* Develop (#336)

* 217: fix unused code.

* 217: unused code and warning

* disabled man pages for rmb

* Bugfix/283 virtual mailbox fetch metadata (#284)

* Develop (#282)

* 217: fix unused code.

* 217: unused code and warning

* disabled man pages for rmb

* #283: fetch metadata for mails in virtual mailbox

* #283: fix metadata date.saved. date.received virtual mailbox

* #283: version

* Feature/286 use guid from UUID string (#287)

* #256: use guid_128_from_uuid_string instead of guid_128_from_string, to support older uuid formats like RECORD or MICROSOFT, always use compact for printing

* version 0.0.25 preparations

* #286: build issue

* merge

* version

* #286: prefere cached mail guid.

* #286: determine if uuid has - hyphon, if true preseve it.

* fix non void return

* Feature/289 GitHub actions (#290)

* #289: build

* #289: build plugin.

* #289: submodules

* submodules

* upgrade git

* clean outdated repos

* Bugfix/UUID record format (#293)

* bugfix initialisaction rados_mail->deprecated_uid

* rados_mail creation and default value check

* #295: update index after full object has been moved (metadata) (#296)

* Bugfix/298 295 bugfixes (#300)

* #289: fix force-resync (always use INBOX)
#295: rbox_mail_set_expunge (play save)

* #298: move ceph objects the standard way.

* clean up log.

* #298: log warning if client connection times out.

* Feature/302 retry expunge connection timeout (#303)

* #302: retry in case of ceph connection timeout

* rpm

* #302: max_retry = 10

* max 10 retry, in case object not available, fail with error

* #304: use last know uid to restore email flags (#305)

* #306: re-assign unreferenced mail objects to inbox (#309)

* #310: read osd_max_object_size and fail if mail.size > osd_max_object_size (#311)

* version: 0.0.31

* #313: fix imap append crash (#314)

* version 0.0.32

* Bugfix/316 mailbox save (#317)

* #316: imap append (remove index entry twice in case of error)

* #316: changed storage for big attachments. (sync write)

* #316: operate api

* #316:  fix operate call

* #316: release preparations

* #316: fixed some legacy unit tests

* 316: review results

* #316: fix

* Bugfix/test output stream check (#323)

* init output stream and free mailbuffer

* build

* extracted write chunks

* mailsize -1;

* fix

* debug messages

* init stream any case

* added deprecation comment, writing chunks now default. new config param rbox_chunk_size default 10240 bytes

* missing mock class

* version

* disabled (temporarily storage tests)

* disabled storage tests( api change)

* Feature/alternative save methods (#326)

* #322: alternative save methods

* version

* Feature/319 force resync immediatelly assign bugfix/328 segmentation fault copy from virtual mailbox (#321)

* #319: automatically add lost objects to inbox (if no mailbox guid exist)

* #319: version 0.0.34

* catch error in case mailbox does not have obox header: e.v. Virtual Mailbox

* check for virtual mailbox when moving or copying, (virtual mailbox not allowed as origin)

* version

* #328: fix copy move (#330)

* Bugfix/332 quota and move (#333)

* 291: failed save mock test

* 291: fixed most of the tests

* 332 fix notify messages number and type

* #332: fix append additional error mssage in case connection can't be opend (#335)

* Develop (#341)

* Loglevel Debug is sufficient as it gets retried (#337)

* Bugfix/339 rados config timeout (#340)

* 339 retry 10 times with random wait. retry read for simple read e.g. rados_config. fail with assert in case we can't find rados_config

* #342: simple multithreading object search

* #342: version 0.0.40

Co-authored-by: Ewald Dieterich <[email protected]>

* Bugfix/342 logging multithreading (#344)

* #342 fix threading and additional logs

* version

* Bugfix/rbox copy context (#347)

* 346; fix rbox-copy seg fault always init rbox_mail

* #346 fix segmentation fault rbox_copy

* #364 and code cleanup

* Feature/349 ceph object index (#350)

* #349: initial storage api

* #349 doveadm rmb create ceph index force resync strategy

* allow refresh of ceph object index via -r param

* #349 save ceph index objects to separate pool

* #349 poc impl ceph index done

* #349 new config setting rbox_index_pool_name

* #349 force reload of all objects, config setting defines if append of oid to ceph index happens

* #349 add , to every entry or entry list

* #349 add , to every entry or entry list, and test

* repair invalid object not debug level log

* #349 changed the remove function, initial index creation: append after each mailbox

* #349 fix unit tests

* #349 version

* #349 bugfix return code doveadm rmb (#351)

* #349 validate object (#353)

* Bugfix/355 fix gzip trailer for empty stream (#357)

* #355 fix write gzip trailer even the stream is empty.

* #355 minor improvements for quota messaging

* #355 fix gzip trailer for empty stream

* #349 fix threshold calc for ceph index file

* #355 fix buffersize for write method 1 and 2 (#360)

* Bugfix 355 fix buffersize write method (#363)

* #355 fix buffersize for write method 1 and 2

* 355 only use append write, clear read buffer in case of read timeout

* #355 check for gzip and fix invalid trailer

* #355 fix unit tests.

* #355 fix call to operate

* 355 stream size for uncompressed mails

* #355 additional tests

* delete ceph index if doveadm mailbox delete -u <> INBOX

* Fix/ceph index (#367)

* fix ceph index size calculation

* delete ceph-index in case its activated

* version string

* fix ceph-index-file-size calculation

* added unit test for object calculation

---------

Co-authored-by: Ewald Dieterich <[email protected]>

src/tests/librmb/test_librmb_utils.cpp

@@ -290,57 +290,6 @@ __attribute__((noreturn)) static void *write_to_save_file(void *threadid) {
 
   pthread_exit(NULL);
 }
-TEST(librmb, append_to_existing_file_multi_threading) {
-  std::string test_file_name = "test1.log";
-  int rc;
-  void *status;
-  pthread_attr_t attr;
-  pthread_t threads[5];
-  // Initialize and set thread joinable
-  pthread_attr_init(&attr);
-  pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
-
-  for (uintptr_t i = 0; i < 5; i++) {
-    rc = pthread_create(&threads[i], NULL, write_to_save_file, (void *)i);
-  }
-  sleep(1);
-  std::cout << " threads created " << std::endl;
-  // free attribute and wait for the other threads
-  pthread_attr_destroy(&attr);
-  for (int i = 0; i < 5; i++) {
-    rc = pthread_join(threads[i], &status);
-    if (rc) {
-      std::cout << "Error:unable to join," << rc << std::endl;
-      exit(-1);
-    }
-
-    std::cout << "Main: completed thread id :" << i;
-    std::cout << "  exiting with status :" << status << std::endl;
-  }
-  sleep(1);
-  int line_count = 0;
-  /** check content **/
-  std::ifstream read(test_file_name);
-  while (true) {
-    librmb::RadosSaveLogEntry entry;
-    read >> entry;
-
-    if (read.eof()) {
-      break;
-    }
-    EXPECT_EQ(entry.oid, "abc");
-    EXPECT_EQ(entry.ns, "ns_1");
-    EXPECT_EQ(entry.pool, "mail_storage");
-    EXPECT_EQ(entry.op, "save");
-    line_count++;
-  }
-  EXPECT_EQ(25, line_count);
-  read.close();
-  std::remove(test_file_name.c_str());
-
-  std::cout << " exiting main " << std::endl;
-  // pthread_exit(NULL);
-}
 
 TEST(librmb, test_mvn_option) {
   std::list<librmb::RadosMetadata *> metadata;
@@ -377,18 +326,33 @@ TEST(librmb, test_mvn_option) {
   std::remove(test_file_name.c_str());
 }
 
-/*TEST(librmb, test_if) {
-  int arr[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10};
+TEST(librmb, test_max_object_size_calculation) {
+
+    bool result = librmb::RadosUtils::object_size_close_to_reach_max(10.0,100.0);
+    EXPECT_EQ(result, false);
+
+    result = librmb::RadosUtils::object_size_close_to_reach_max(81.0,100.0);
+    EXPECT_EQ(result, true);
+
+    result = librmb::RadosUtils::object_size_close_to_reach_max(112820433.0
+                                                               ,134217728.0);
+    EXPECT_EQ(result, true);
+
+    result = librmb::RadosUtils::object_size_close_to_reach_max(13646193.0
+                                                               ,134217728.0);
+    EXPECT_EQ(result, false);
+
+    result = librmb::RadosUtils::object_size_close_to_reach_max(13646226.0
+                                                               ,134217728.0);
+    EXPECT_EQ(result, false);
+
+    result = librmb::RadosUtils::object_size_close_to_reach_max(13646259.0
+                                                               ,134217728.0);
+    EXPECT_EQ(result, false);
+
+}
 
-  for (int i = 0; i < 10; ++i) {
-    std::cout << " value: " << arr[i] << std::endl;
-  }
-  for (int i = 0; i < 10; i++) {
-    std::cout << " value: " << arr[i] << std::endl;
-  }
 
-  EXPECT_EQ(1, 2);
-}*/
 TEST(librmb, mock_obj) {}
 int main(int argc, char **argv) {
   ::testing::InitGoogleMock(&argc, argv);