From 0a34c6e32a9c0a89f03957f8447a2c298ee98adb Mon Sep 17 00:00:00 2001 From: Mixficsol <838844609@qq.com> Date: Wed, 10 May 2023 14:06:34 +0800 Subject: [PATCH] delete tcmalloc in Pika (#1472) --- CMakeLists.txt | 12 +--- detect_environment | 20 ------ docs/ops/bestPractice.md | 4 +- docs/ops/install.md | 3 - docs/ops/memoryUsage.md | 7 -- include/pika_admin.h | 16 ----- include/pika_command.h | 3 - pika-tools/pika-port/pika_port_2/Makefile | 2 - .../pika2.3.6/include/pika_command.h | 3 - .../pika-port/pika_port_3/pika_command.h | 3 - src/pika.cc | 6 -- src/pika_admin.cc | 64 ------------------- src/pika_command.cc | 4 -- 13 files changed, 4 insertions(+), 143 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7aed054660..0758a25cf1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -320,9 +320,9 @@ set(ZSTD_INCLUDE_DIR ${INSTALL_INCLUDEDIR}) ExternalProject_Add(fmt DEPENDS URL - https://github.com/fmtlib/fmt/archive/refs/heads/master.tar.gz + https://github.com/fmtlib/fmt/archive/refs/tags/7.1.0.tar.gz URL_HASH - MD5=27e54411fce14de11624994b3ebf3c48 + MD5=32af902636d373641f4ef9032fc65b3a DOWNLOAD_NO_PROGRESS 1 UPDATE_COMMAND @@ -451,14 +451,6 @@ if(CMAKE_SYSTEM_NAME MATCHES "Linux") set(LIBGPERF_NAME gperftools) endif() -if(${OS_VERSION} MATCHES "CentOS") - set(TCMALLOC_LIBRARY ${INSTALL_LIBDIR_64}/liblibtcmalloclz4.a) -else() - set(TCMALLOC_LIBRARY ${INSTALL_LIBDIR}/libtcmalloc.a) -endif() - -set(TCMALLOC_INCLUDE_DIR ${INSTALL_INCLUDEDIR}) - if(CMAKE_SYSTEM_NAME MATCHES "Linux") ExternalProject_Add(jemalloc DEPENDS diff --git a/detect_environment b/detect_environment index a316ec02da..e002020726 100755 --- a/detect_environment +++ b/detect_environment @@ -14,24 +14,6 @@ if test -z "$CXX"; then CXX=g++ fi -# Test whether tcmalloc is available -if echo 'int main() {}' | $CXX $CFLAGS -x c++ - -o /dev/null \ - -ltcmalloc 2>/dev/null; then - TCMALLOC_LDFLAGS=" -ltcmalloc" -fi - -# Test whether malloc_extension is available -$CXX $CFLAGS -x c++ - -o /dev/null -ltcmalloc 2>/dev/null < - int main() { - MallocExtension::instance()->Initialize();; - return 0; - } -EOF -if [ "$?" = 0 ]; then - TCMALLOC_EXTENSION_FLAGS=" -DTCMALLOC_EXTENSION" -fi - # Test whether Snappy library is installed # http://code.google.com/p/snappy/ $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <> "$OUTPUT" -echo "TCMALLOC_EXTENSION_FLAGS=$TCMALLOC_EXTENSION_FLAGS" >> "$OUTPUT" -echo "TCMALLOC_LDFLAGS=$TCMALLOC_LDFLAGS" >> "$OUTPUT" echo "PROCESSOR_NUMS=$PROCESSOR_NUMS" >> "$OUTPUT" diff --git a/docs/ops/bestPractice.md b/docs/ops/bestPractice.md index fd0154a612..5385f40ec3 100644 --- a/docs/ops/bestPractice.md +++ b/docs/ops/bestPractice.md @@ -35,7 +35,7 @@ pika的性能和IO性能息息相关，我们不建议在机械盘上部署耗 **pika最佳实践之八：** -pika的内存占用主要集中在sst文件的cache和连接申请内存，而通常连接申请内存会比sst的cache要大很多，pika目前已支持连接申请内存的动态调整、回收，因此连接占用的总内存大小是可以粗略估算的，如果你的pika内存占用远超预估或大于10g，那么可能是内存泄漏了，尝试依次执行命令`client kill all`和`tcmalloc free`来对连接内存进行强制回收，如果效果不好请升级到最新版本 +pika的内存占用主要集中在sst文件的cache和连接申请内存，而通常连接申请内存会比sst的cache要大很多，pika目前已支持连接申请内存的动态调整、回收，因此连接占用的总内存大小是可以粗略估算的，如果你的pika内存占用远超预估或大于10g，那么可能是内存泄漏了，尝试依次执行命令`client kill all`和`tcmalloc free`(tcmalloc目前已移除)来对连接内存进行强制回收，如果效果不好请升级到最新版本 **pika最佳实践之九：** @@ -149,4 +149,4 @@ Pika从3.0.5开始提供了更细致的compact策略，该策略允许对key的 在建立主从的时候发现从库反复的进行全同步很可能是由于主库写入量太大，在全同步完毕期间清理掉了原先dumpdb时的binlog点位，导致从库在替换了db之后重新trysync发现点位又找不到，然后重新触发全同步，陷入了循环，这种场景下可以动态将主库的expire-logs-nums调大，让其保留尽量多的binlog, 建立主从成功之后再调节回来即可 --- -_不断更新_ \ No newline at end of file +_不断更新_ diff --git a/docs/ops/install.md b/docs/ops/install.md index 9a35fdc645..f046e6b4c4 100644 --- a/docs/ops/install.md +++ b/docs/ops/install.md @@ -100,6 +100,3 @@ $ make ``` $ ./output/bin/pika -c ./conf/pika.conf ``` - -## 注意 -启动出现 Attempt to free invalid pointer 问题请尝试升级tcmalloc 版本，建议使用gperftools 2.7 版本包含的tcmalloc。 \ No newline at end of file diff --git a/docs/ops/memoryUsage.md b/docs/ops/memoryUsage.md index a0c1ae423d..34f276836e 100644 --- a/docs/ops/memoryUsage.md +++ b/docs/ops/memoryUsage.md @@ -1,6 +1,5 @@ ### Pika内存占用 1. rocksdb 内存占用 -2. pika 内存占用(tcmalloc 占用) #### 1. rocksdb 内存占用 命令行命令 info data @@ -21,9 +20,3 @@ https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning https://github.com/facebook/rocksdb/wiki/Memory-usage-in-RocksDB -#### 2. pika 内存占用 -如果使用tcmalloc，绝大多数情况下是tcmalloc暂时占用内存。 - -命令行命令：tcmalloc stats - -命令行命令：tcmalloc free 释放tcmalloc 占用内存 \ No newline at end of file diff --git a/include/pika_admin.h b/include/pika_admin.h index 9282f1082f..da0776e59e 100644 --- a/include/pika_admin.h +++ b/include/pika_admin.h @@ -388,22 +388,6 @@ class PaddingCmd : public Cmd { uint64_t offset) override; }; -#ifdef TCMALLOC_EXTENSION -class TcmallocCmd : public Cmd { - public: - TcmallocCmd(const std::string& name, int arity, uint16_t flag) : Cmd(name, arity, flag) {} - virtual void Do(std::shared_ptr partition = nullptr); - virtual void Split(std::shared_ptr partition, const HintKeys& hint_keys){}; - virtual void Merge(){}; - virtual Cmd* Clone() override { return new TcmallocCmd(*this); } - - private: - int64_t type_; - int64_t rate_; - virtual void DoInitial() override; -}; -#endif - class PKPatternMatchDelCmd : public Cmd { public: PKPatternMatchDelCmd(const std::string& name, int arity, uint16_t flag) : Cmd(name, arity, flag) {} diff --git a/include/pika_command.h b/include/pika_command.h index e20779322b..9b37feb123 100644 --- a/include/pika_command.h +++ b/include/pika_command.h @@ -41,9 +41,6 @@ const std::string kCmdNameEcho = "echo"; const std::string kCmdNameScandb = "scandb"; const std::string kCmdNameSlowlog = "slowlog"; const std::string kCmdNamePadding = "padding"; -#ifdef TCMALLOC_EXTENSION -const std::string kCmdNameTcmalloc = "tcmalloc"; -#endif const std::string kCmdNamePKPatternMatchDel = "pkpatternmatchdel"; const std::string kCmdDummy = "dummy"; const std::string kCmdNameQuit = "quit"; diff --git a/pika-tools/pika-port/pika_port_2/Makefile b/pika-tools/pika-port/pika_port_2/Makefile index 02bfe8250d..644a596ae4 100644 --- a/pika-tools/pika-port/pika_port_2/Makefile +++ b/pika-tools/pika-port/pika_port_2/Makefile @@ -16,8 +16,6 @@ THIRD_PATH = ../../third # ROCKSDB_LDFLAGS= -lsnappy -lz -lbz2 -llz4 -lzstd ROCKSDB_LDFLAGS= -lsnappy -lgflags -lz -lbz2 -TCMALLOC_EXTENSION_FLAGS= -TCMALLOC_LDFLAGS= PROCESSOR_NUMS=2 NEMO_PATH=$(THIRD_PATH)/nemo diff --git a/pika-tools/pika-port/pika_port_2/pika2.3.6/include/pika_command.h b/pika-tools/pika-port/pika_port_2/pika2.3.6/include/pika_command.h index b6ac00fb67..07a8b21fa6 100644 --- a/pika-tools/pika-port/pika_port_2/pika2.3.6/include/pika_command.h +++ b/pika-tools/pika-port/pika_port_2/pika2.3.6/include/pika_command.h @@ -40,9 +40,6 @@ const std::string kCmdNameDbsize = "dbsize"; const std::string kCmdNameTime = "time"; const std::string kCmdNameDelbackup = "delbackup"; const std::string kCmdNameEcho = "echo"; -#ifdef TCMALLOC_EXTENSION -const std::string kCmdNameTcmalloc = "tcmalloc"; -#endif // Migrate slot const std::string kCmdNameSlotsMgrtSlot = "slotsmgrtslot"; diff --git a/pika-tools/pika-port/pika_port_3/pika_command.h b/pika-tools/pika-port/pika_port_3/pika_command.h index 0a9fe603cb..84f5c3ef14 100644 --- a/pika-tools/pika-port/pika_port_3/pika_command.h +++ b/pika-tools/pika-port/pika_port_3/pika_command.h @@ -43,9 +43,6 @@ const std::string kCmdNameDelbackup = "delbackup"; const std::string kCmdNameEcho = "echo"; const std::string kCmdNameScandb = "scandb"; const std::string kCmdNameSlowlog = "slowlog"; -#ifdef TCMALLOC_EXTENSION -const std::string kCmdNameTcmalloc = "tcmalloc"; -#endif // Migrate slot const std::string kCmdNameSlotsMgrtSlot = "slotsmgrtslot"; diff --git a/src/pika.cc b/src/pika.cc index b06c02e53d..043c869a59 100644 --- a/src/pika.cc +++ b/src/pika.cc @@ -17,9 +17,6 @@ #include "include/pika_version.h" #include "pstd/include/env.h" -#ifdef TCMALLOC_EXTENSION -# include -#endif PikaConf* g_pika_conf; PikaServer* g_pika_server; @@ -156,9 +153,6 @@ int main(int argc, char* argv[]) { usage(); exit(-1); } -#ifdef TCMALLOC_EXTENSION - MallocExtension::instance()->Initialize(); -#endif PikaConfInit(path); rlimit limit; diff --git a/src/pika_admin.cc b/src/pika_admin.cc index b24c4a2787..37b5773b5c 100644 --- a/src/pika_admin.cc +++ b/src/pika_admin.cc @@ -19,10 +19,6 @@ #include "include/pika_version.h" #include "pstd/include/rsync.h" -#ifdef TCMALLOC_EXTENSION -# include -#endif - extern PikaServer* g_pika_server; extern PikaConf* g_pika_conf; extern PikaReplicaManager* g_pika_rm; @@ -2226,66 +2222,6 @@ std::string PaddingCmd::ToBinlog(uint32_t exec_time, uint32_t term_id, uint64_t argv_[1].size() + BINLOG_ITEM_HEADER_SIZE + PADDING_BINLOG_PROTOCOL_SIZE + SPACE_STROE_PARAMETER_LENGTH); } -#ifdef TCMALLOC_EXTENSION -void TcmallocCmd::DoInitial() { - if (argv_.size() != 2 && argv_.size() != 3) { - res_.SetRes(CmdRes::kWrongNum, kCmdNameTcmalloc); - return; - } - rate_ = 0; - std::string type = argv_[1]; - if (!strcasecmp(type.data(), "stats")) { - type_ = 0; - } else if (!strcasecmp(type.data(), "rate")) { - type_ = 1; - if (argv_.size() == 3) { - if (!pstd::string2int(argv_[2].data(), argv_[2].size(), &rate_)) { - res_.SetRes(CmdRes::kSyntaxErr, kCmdNameTcmalloc); - } - } - } else if (!strcasecmp(type.data(), "list")) { - type_ = 2; - } else if (!strcasecmp(type.data(), "free")) { - type_ = 3; - } else { - res_.SetRes(CmdRes::kInvalidParameter, kCmdNameTcmalloc); - return; - } -} - -void TcmallocCmd::Do(std::shared_ptr partition) { - std::vector fli; - std::vector elems; - switch (type_) { - case 0: - char stats[1024]; - MallocExtension::instance()->GetStats(stats, 1024); - pstd::StringSplit(stats, '\n', elems); - res_.AppendArrayLen(elems.size()); - for (auto& i : elems) { - res_.AppendString(i); - } - break; - case 1: - if (rate_) { - MallocExtension::instance()->SetMemoryReleaseRate(rate_); - } - res_.AppendInteger(MallocExtension::instance()->GetMemoryReleaseRate()); - break; - case 2: - MallocExtension::instance()->GetFreeListSizes(&fli); - res_.AppendArrayLen(fli.size()); - for (auto& i : fli) { - res_.AppendString("type: " + std::string(i.type) + ", min: " + std::to_string(i.min_object_size) + ", max: " + - std::to_string(i.max_object_size) + ", total: " + std::to_string(i.total_bytes_free)); - } - break; - case 3: - MallocExtension::instance()->ReleaseFreeMemory(); - res_.SetRes(CmdRes::kOk); - } -} -#endif void PKPatternMatchDelCmd::DoInitial() { if (!CheckArg(argv_.size())) { diff --git a/src/pika_command.cc b/src/pika_command.cc index 4c9a72f133..c21b15123a 100644 --- a/src/pika_command.cc +++ b/src/pika_command.cc @@ -116,10 +116,6 @@ void InitCmdTable(std::unordered_map* cmd_table) { // Cluster related Cmd* pkclusterinfoptr = new PkClusterInfoCmd(kCmdNamePkClusterInfo, -3, kCmdFlagsRead | kCmdFlagsAdmin); cmd_table->insert(std::pair(kCmdNamePkClusterInfo, pkclusterinfoptr)); -#ifdef TCMALLOC_EXTENSION - Cmd* tcmallocptr = new TcmallocCmd(kCmdNameTcmalloc, -2, kCmdFlagsRead | kCmdFlagsAdmin); - cmd_table->insert(std::pair(kCmdNameTcmalloc, tcmallocptr)); -#endif // Kv ////SetCmd