utils: periodically memory release of tcmalloc

include/dsn/dist/replication/replication.codes.h

@@ -66,6 +66,7 @@ MAKE_EVENT_CODE_AIO(LPC_WRITE_REPLICATION_LOG_COMMON, TASK_PRIORITY_HIGH)
 MAKE_EVENT_CODE_AIO(LPC_WRITE_REPLICATION_LOG_PRIVATE, TASK_PRIORITY_HIGH)
 MAKE_EVENT_CODE_AIO(LPC_WRITE_REPLICATION_LOG_SHARED, TASK_PRIORITY_HIGH)
 MAKE_EVENT_CODE(LPC_QUERY_CONFIGURATION_ALL, TASK_PRIORITY_HIGH)
+MAKE_EVENT_CODE(LPC_MEM_RELEASE, TASK_PRIORITY_COMMON)
 #undef CURRENT_THREAD_POOL
 
 // THREAD_POOL_META_SERVER

src/core/core/service_api_c.cpp

@@ -31,6 +31,9 @@
 #include <dsn/utility/process_utils.h>
 #include <dsn/tool-api/command_manager.h>
 #include <fstream>
+#ifdef DSN_ENABLE_GPERF
+#include <gperftools/malloc_extension.h>
+#endif
 
 #include "service_engine.h"
 #include "rpc_engine.h"
@@ -381,6 +384,16 @@ bool run(const char *config_file,
         "thread local transient memory buffer size (KB), default is 1024");
     ::dsn::tls_trans_mem_init(tls_trans_memory_KB * 1024);
 
+#ifdef DSN_ENABLE_GPERF
+    double_t tcmalloc_release_rate =
+        (double_t)dsn_config_get_value_double("core",
+                                              "tcmalloc_release_rate",
+                                              1., // [0, 10]
+                                              "the memory releasing rate of tcmalloc, default is "
+                                              "1.0 in gperftools, value range is 0.0~10.0");
+    ::MallocExtension::instance()->SetMemoryReleaseRate(tcmalloc_release_rate);
+#endif
+
     // prepare minimum necessary
     ::dsn::service_engine::instance().init_before_toollets(spec);
 

src/dist/http/pprof_http_service.cpp

@@ -28,6 +28,7 @@
 #include <dsn/utility/timer.h>
 #include <dsn/utility/string_splitter.h>
 #include <gperftools/malloc_extension.h>
+#include <gperftools/profiler.h>
 
 namespace dsn {
 
@@ -334,6 +335,7 @@ static bool get_heap_profile(std::string &result)
 void pprof_http_service::heap_handler(const http_request &req, http_response &resp)
 {
     resp.status_code = http_status_code::ok;
+
     get_heap_profile(resp.body);
 }
 

src/dist/replication/common/replication_common.cpp

@@ -99,6 +99,9 @@ replication_options::replication_options()
     config_sync_disabled = false;
     config_sync_interval_ms = 30000;
 
+    mem_release_enabled = true;
+    mem_release_interval_ms = 86400000;
+
     lb_interval_ms = 10000;
 
     learn_app_max_concurrent_count = 5;
@@ -471,6 +474,17 @@ void replication_options::initialize()
         config_sync_interval_ms,
         "every this period(ms) the replica syncs replica configuration with the meta server");
 
+    mem_release_enabled = dsn_config_get_value_bool("replication",
+                                                    "mem_release_enabled",
+                                                    mem_release_enabled,
+                                                    "whether to enable periodic memory release");
+
+    mem_release_interval_ms = (int)dsn_config_get_value_uint64(
+        "replication",
+        "mem_release_interval_ms",
+        mem_release_interval_ms,
+        "the replica releases its idle memory to the system every this period of time(ms)");
+
     lb_interval_ms = (int)dsn_config_get_value_uint64(
         "replication",
         "lb_interval_ms",

src/dist/replication/common/replication_common.h

@@ -103,6 +103,9 @@ class replication_options
     bool config_sync_disabled;
     int32_t config_sync_interval_ms;
 
+    bool mem_release_enabled;
+    int32_t mem_release_interval_ms;
+
     int32_t lb_interval_ms;
 
     int32_t learn_app_max_concurrent_count;

src/dist/replication/lib/replica_stub.cpp

@@ -45,6 +45,9 @@
 #include <dsn/dist/replication/replication_app_base.h>
 #include <vector>
 #include <deque>
+#ifdef DSN_ENABLE_GPERF
+#include <gperftools/malloc_extension.h>
+#endif
 
 namespace dsn {
 namespace replication {
@@ -632,6 +635,22 @@ void replica_stub::initialize_start()
                                    std::chrono::milliseconds(_options.config_sync_interval_ms));
     }
 
+#ifdef DSN_ENABLE_GPERF
+    if (_options.mem_release_enabled) {
+        _mem_release_timer_task =
+            tasking::enqueue_timer(LPC_MEM_RELEASE,
+                                   &_tracker,
+                                   []() {
+                                       ddebug("Memory release has started...");
+                                       ::MallocExtension::instance()->ReleaseFreeMemory();
+                                       ddebug("Memory release has ended...");
+                                   },
+                                   std::chrono::milliseconds(_options.mem_release_interval_ms),
+                                   0,
+                                   std::chrono::milliseconds(_options.mem_release_interval_ms));
+    }
+#endif
+
     // init liveness monitor
     dassert(NS_Disconnected == _state, "");
     if (_options.fd_disabled == false) {
@@ -2185,6 +2204,11 @@ void replica_stub::close()
         _gc_timer_task = nullptr;
     }
 
+    if (_mem_release_timer_task != nullptr) {
+        _mem_release_timer_task->cancel(true);
+        _mem_release_timer_task = nullptr;
+    }
+
     {
         zauto_write_lock l(_replicas_lock);
         while (!_closing_replicas.empty()) {
@@ -2254,5 +2278,5 @@ std::string replica_stub::get_replica_dir(const char *app_type, gpid id, bool cr
     }
     return ret_dir;
 }
-}
-} // namespace
+} // namespace replication
+} // namespace dsn

src/dist/replication/lib/replica_stub.h

@@ -243,6 +243,7 @@ class replica_stub : public serverlet<replica_stub>, public ref_counter
     ::dsn::task_ptr _config_sync_timer_task;
     ::dsn::task_ptr _gc_timer_task;
     ::dsn::task_ptr _disk_stat_timer_task;
+    ::dsn::task_ptr _mem_release_timer_task;
 
     // command_handlers
     dsn_handle_t _kill_partition_command;

src/tests/dsn/failure_detector.cpp

@@ -290,6 +290,7 @@ void clear(test_worker *worker, std::vector<test_master *> masters)
 
 void finish(test_worker *worker, test_master *master, int master_index)
 {
+    dwarn("start to finish");
     std::atomic_int wait_count;
     wait_count.store(2);
     worker->fd()->when_disconnected(