Merge branch 'change-thread-shedule' into unstable

.github/pr-title-checker-config.json

@@ -4,7 +4,7 @@
     "color": "B60205"
   },
   "CHECKS": {
-    "regexp": "^(feat|fix|test|refactor|chore|upgrade|style|docs|perf|build|ci|revert)(\\(.*\\))?:.*",
+    "regexp": "^(feat|fix|test|refactor|chore|upgrade|style|docs|perf|build|ci|revert)(\\(.*\\))?:[^\u4e00-\u9fa5]+$",
     "ignoreLabels": [
       "ignore-title"
     ]

.github/workflows/codeql.yml

@@ -13,10 +13,10 @@ name: "CodeQL"
 
 on:
   push:
-    branches: [ "unstable", "3.5" ]
+    branches: [ "unstable", "3.5" , "4.0"]
   pull_request:
     # The branches below must be a subset of the branches above
-    branches: [ "unstable", "3.5" ]
+    branches: [ "unstable", "3.5" , "4.0"]
   schedule:
     - cron: '25 19 * * 6'
 

.github/workflows/codis.yml

@@ -5,9 +5,9 @@ name: Codis
 
 on:
   push:
-    branches: [ "unstable", "3.5" ]
+    branches: [ "unstable", "3.5" , "4.0" ]
   pull_request:
-    branches: [ "unstable", "3.5" ]
+    branches: [ "unstable", "3.5" , "4.0"]
 
 jobs:
 

.github/workflows/pika.yml

@@ -2,9 +2,9 @@ name: Pika
 
 on:
   push:
-    branches: [ "unstable", "3.5" ]
+    branches: [ "unstable", "3.5" , "4.0"]
   pull_request:
-    branches: [ "unstable", "3.5" ]
+    branches: [ "unstable", "3.5" , "4.0"]
 
 env:
   # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)
@@ -26,25 +26,20 @@ jobs:
         with:
           go-version: 1.19
 
+      - name: ccache
+        uses: hendrikmuhs/[email protected]
+        with:
+          key: ubuntu-latest
+
       - name: Install Deps
-        if: ${{ steps.cache.output.cache-hit != 'true' }}
         run: |
           sudo apt-get install -y autoconf libprotobuf-dev protobuf-compiler
           sudo apt-get install -y clang-tidy-12
 
       - name: Configure CMake
         # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
         # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
-        run: cmake -B build -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} -DUSE_PIKA_TOOLS=ON -DCMAKE_CXX_FLAGS_DEBUG=-fsanitize=address
-
-      - name: Cache Build
-        uses: actions/cache@v3
-        id: cache-ubuntu
-        with:
-          key: ${{ runner.os }}-build-ubuntu-${{ hashFiles('**/CMakeLists.txt') }}
-          path: |
-            ${{ github.workspace }}/buildtrees
-            ${{ github.workspace }}/deps
+        run: cmake -B build -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} -DUSE_PIKA_TOOLS=ON -DCMAKE_CXX_FLAGS_DEBUG=-fsanitize=address -D CMAKE_C_COMPILER_LAUNCHER=ccache -D CMAKE_CXX_COMPILER_LAUNCHER=ccache
 
       - name: Build
         # Build your program with the given configuration
@@ -166,25 +161,21 @@ jobs:
         with:
           go-version: 1.19
 
+      - name: ccache
+        uses: hendrikmuhs/[email protected]
+        with:
+          key: macos-12
+
       - name: Install Deps
         run: |
           brew update
           brew install --overwrite [email protected] autoconf protobuf llvm wget git
           brew install gcc@10 automake cmake make binutils
+
       - name: Configure CMake
         run: |
           export CC=/usr/local/opt/gcc@10/bin/gcc-10 
-          cmake -B build -DCMAKE_C_COMPILER=/usr/local/opt/gcc@10/bin/gcc-10 -DUSE_PIKA_TOOLS=ON -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} -DCMAKE_CXX_FLAGS_DEBUG=-fsanitize=address
-
-      - uses: actions/cache@v3
-        with:
-          path: ${{ github.workspace }}/deps
-          key: ${{ runner.os }}-deps-${{ hashFiles('**/CMakeLists.txt') }}
-
-      - uses: actions/cache@v3
-        with:
-          path: ${{ github.workspace }}/buildtrees
-          key: ${{ runner.os }}-buildtrees-${{ hashFiles('**/CMakeLists.txt') }}
+          cmake -B build -DCMAKE_C_COMPILER=/usr/local/opt/gcc@10/bin/gcc-10 -DUSE_PIKA_TOOLS=ON -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} -DCMAKE_CXX_FLAGS_DEBUG=-fsanitize=address -D CMAKE_C_COMPILER_LAUNCHER=ccache -D CMAKE_CXX_COMPILER_LAUNCHER=ccache
 
       - name: Build
         run: |

.github/workflows/pr-title-checker.yaml

@@ -12,7 +12,7 @@ jobs:
   check:
     runs-on: ubuntu-latest
     steps:
-      - uses: thehanimo/[email protected].1
+      - uses: thehanimo/[email protected].2
         with:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           pass_on_octokit_error: false

.github/workflows/release.yml

@@ -16,7 +16,7 @@ jobs:
         include:
           - os: ubuntu-latest
             name: ${{ github.event.repository.name }}-${{ github.ref_name }}-ubuntu-amd64.tar.gz
-          - os: macos-latest
+          - os: macos-12
             name: ${{ github.event.repository.name }}-${{ github.ref_name }}-macos-amd64.tar.gz
 
     runs-on: ${{ matrix.os }}

.github/workflows/tools_go.yml

@@ -2,11 +2,11 @@ name: Tools_go_build
 
 on:
   push:
-    branches: [ "unstable", "3.5" ]
+    branches: [ "unstable", "3.5" , "4.0"]
     paths:
       - 'tools/**'
   pull_request:
-    branches: [ "unstable", "3.5" ]
+    branches: [ "unstable", "3.5" , "4.0"]
     paths:
       - 'tools/**'
 

CMakeLists.txt

@@ -56,6 +56,9 @@ endif()
 if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
    set(CMAKE_CXX_FLAGS "-pthread")
    add_definitions(-DOS_MACOSX)
+elseif (CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
+    set(CMAKE_CXX_FLAGS "-pthread")
+    add_definitions(-DOS_FREEBSD)
 elseif(CMAKE_SYSTEM_NAME MATCHES "Linux")
    if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
      set(CMAKE_EXE_LINKER_FLAGS "-stdlib=libc++ -fuse-ld=lld -lc++ -lc++abi ${CMAKE_EXE_LINKER_FLAGS}")
@@ -66,7 +69,7 @@ elseif(CMAKE_SYSTEM_NAME MATCHES "Linux")
    endif()
    add_definitions(-DOS_LINUX)
 else()
-   message(FATAL_ERROR "only support linux or macOs")
+   message(FATAL_ERROR "only support linux or macOs or FreeBSD")
 endif()
 
 if(HOST_ARCH MATCHES "x86_64" OR HOST_ARCH MATCHES "i386")

README.md

@@ -252,14 +252,21 @@ Users can directly download the latest binary version package from [releases](ht
 
 * #### 3.1 Running with Docker
 
-  ```bash
+  Modify the following configuration items of conf/pika.conf file:
+ ```
+  log-path : /data/log/
+  db-path : /data/db/
+  db-sync-path : /data/dbsync/
+  dump-path : /data/dump/
+ ```
+
+  And then execute the following statement to start pika in docker:
+ ```bash
   docker run -d \
     --restart=always \
     -p 9221:9221 \
-    -v <log_dir>:/pika/log \
-    -v <db_dir>:/pika/db \
-    -v <dump_dir>:/pika/dump \
-    -v <dbsync_dir>:/pika/dbsync \
+    -v "$(pwd)/conf":"/pika/conf" \
+    -v "/tmp/pika-data":"/data" \
     pikadb/pika:v3.3.6
 
   redis-cli -p 9221 "info"

conf/pika.conf

@@ -27,6 +27,11 @@ thread-num : 1
 # are dedicated to handling user requests.
 thread-pool-size : 12
 
+# This parameter is used to control whether to separate fast and slow commands.
+# When slow-cmd-pool is set to yes, fast and slow commands are separated.
+# When set to no, they are not separated.
+slow-cmd-pool : no
+
 # Size of the low level thread pool, The threads within this pool
 # are dedicated to handling slow user requests.
 slow-cmd-thread-pool-size : 1
@@ -240,7 +245,8 @@ slave-priority : 100
 # The disable_auto_compactions option is [true | false]
 disable_auto_compactions : false
 
-# Rocksdb max_subcompactions
+# Rocksdb max_subcompactions, increasing this value can accelerate the exec speed of a single compaction task
+# it's recommended to increase it's value if large compaction is found in you instance
 max-subcompactions : 1
 # The minimum disk usage ratio for checking resume.
 # If the disk usage ratio is lower than min-check-resume-ratio, it will not check resume, only higher will check resume.
@@ -317,6 +323,11 @@ max-write-buffer-num : 2
 # whether the key exists. Setting this value too high may hurt performance.
 min-write-buffer-number-to-merge : 1
 
+# The total size of wal files, when reaches this limit, rocksdb will force the flush of column-families
+# whose memtables are backed by the oldest live WAL file. Also used to control the rocksdb open time when
+# process restart.
+max-total-wal-size : 1073741824
+
 # rocksdb level0_stop_writes_trigger
 level0-stop-writes-trigger : 36
 
@@ -347,17 +358,42 @@ compression : snappy
 # https://github.com/facebook/rocksdb/wiki/Compression
 #compression_per_level : [none:none:snappy:lz4:lz4]
 
+# The number of rocksdb background threads(sum of max-background-compactions and max-background-flushes)
+# If max-background-jobs has a valid value AND both 'max-background-flushs' and 'max-background-compactions' is set to -1,
+# then max-background-flushs' and 'max-background-compactions will be auto config by rocksdb, specifically:
+# 1/4 of max-background-jobs will be given to max-background-flushs' and the rest(3/4) will be given to  'max-background-compactions'.
+# 'max-background-jobs' default value is 3 and the value range is [2, 12].
+max-background-jobs : 3
+
 # The number of background flushing threads.
-# max-background-flushes default value is 1 and the value range is [1, 4].
-max-background-flushes : 1
+# max-background-flushes default value is -1 and the value range is [1, 4] or -1.
+# if 'max-background-flushes' is set to -1, the 'max-background-compactions' should also be set to -1,
+# which means let rocksdb to auto config them based on the value of 'max-background-jobs'
+max-background-flushes : -1
+
+# [NOTICE] you MUST NOT set one of the max-background-flushes or max-background-compactions to -1 while setting another one to other values(not -1).
+# They SHOULD both be -1 or both not(if you want to config them manually).
 
 # The number of background compacting threads.
-# max-background-compactions default value is 2 and the value range is [1, 8].
-max-background-compactions : 2
+# max-background-compactions default value is -1 and the value range is [1, 8] or -1.
+# if 'max-background-compactions' is set to -1, the 'max-background-flushes' should also be set to -1,
+# which means let rocksdb to auto config them based on the value of 'max-background-jobs'
+max-background-compactions : -1
+
+# RocksDB delayed-write-rate, default is 0(infer from rate-limiter by RocksDB)
+# Ref from rocksdb: Whenever stall conditions are triggered, RocksDB will reduce write rate to delayed_write_rate,
+# and could possibly reduce write rate to even lower than delayed_write_rate if estimated pending compaction bytes accumulates.
+# If the value is 0, RcoksDB will infer a value from `rater_limiter` value if it is not empty, or 16MB if `rater_limiter` is empty.
+# Note that if users change the rate in `rate_limiter` after DB is opened, delayed_write_rate won't be adjusted.
+# [Support Dynamically changeable] send 'config set delayed-write-rate' to a running pika can change it's value dynamically
+delayed-write-rate : 0
+
+
+# RocksDB will try to limit number of bytes in one compaction to be lower than this max-compaction-bytes.
+# But it's NOT guaranteed.
+# default value is -1, means let it be 25 * target-file-size-base (Which is RocksDB's default value)
+max-compaction-bytes : -1
 
-# The number of background threads.
-# max-background-jobs default value is 3 and the value range is [2, 12].
-max-background-jobs : 3
 
 # maximum value of RocksDB cached open file descriptors
 max-cache-files : 5000
@@ -423,14 +459,17 @@ default-slot-num : 1024
 # 0: Read 1: Write 2: ReadAndWrite
 # rate-limiter-mode : default 1
 
-# rate limiter bandwidth, default 2000MB/s
-#rate-limiter-bandwidth : 2097152000
+# rate limiter bandwidth, units in bytes, default 1024GB/s (No limit)
+# [Support Dynamically changeable] send 'rate-limiter-bandwidth' to a running pika can change it's value dynamically
+#rate-limiter-bandwidth : 1099511627776
 
 #rate-limiter-refill-period-us : 100000
 #
 #rate-limiter-fairness: 10
 
-# rate limiter auto tune https://rocksdb.org/blog/2017/12/18/17-auto-tuned-rate-limiter.html. the default value is false.
+# if auto_tuned is true: Enables dynamic adjustment of rate limit within the range
+#`[rate-limiter-bandwidth / 20, rate-limiter-bandwidth]`, according to the recent demand for background I/O.
+# rate limiter auto tune https://rocksdb.org/blog/2017/12/18/17-auto-tuned-rate-limiter.html. the default value is true.
 #rate-limiter-auto-tuned : true
 
 ################################## RocksDB Blob Configure #####################

include/pika_client_processor.h

@@ -19,12 +19,10 @@ class PikaClientProcessor {
   int Start();
   void Stop();
   void SchedulePool(net::TaskFunc func, void* arg);
-  void ScheduleBgThreads(net::TaskFunc func, void* arg, const std::string& hash_str);
   size_t ThreadPoolCurQueueSize();
   size_t ThreadPoolMaxQueueSize();
 
  private:
   std::unique_ptr<net::ThreadPool> pool_;
-  std::vector<std::unique_ptr<net::BGThread>> bg_threads_;
 };
 #endif  // PIKA_CLIENT_PROCESSOR_H_