Merge branch 'unstable' into kvrocks2redis_ci

cmake/snappy.cmake

@@ -20,8 +20,8 @@ include_guard()
 include(cmake/utils.cmake)
 
 FetchContent_DeclareGitHubWithMirror(snappy
-  google/snappy f725f6766bfc62418c6491b504c8e5865ec99412
-  MD5=17a982c9b0c667b3744e1fecba0046f7
+  google/snappy 1.2.0
+  MD5=555f4af7472ce585c12709121808a060
 )
 
 FetchContent_MakeAvailableWithArgs(snappy

kvrocks.conf

@@ -519,6 +519,13 @@ compaction-checker-range 0-7
 # e.g. bgsave-cron 0 3 * * * 0 4 * * *
 # would bgsave the db at 3am and 4am every day
 
+# Kvrocks doesn't store the key number directly. It needs to scan the DB and
+# then retrieve the key number by using the dbsize scan command.
+# The Dbsize scan scheduler auto-recalculates the estimated keys at scheduled time.
+# Time expression format is the same as crontab (currently only support * and int)
+# e.g. dbsize-scan-cron 0 * * * *
+# would recalculate the keyspace infos of the db every hour.
+
 # Command renaming.
 #
 # It is possible to change the name of dangerous commands in a shared

src/commands/cmd_cluster.cc

@@ -295,12 +295,12 @@ class CommandClusterX : public Commander {
   std::unique_ptr<SyncMigrateContext> sync_migrate_ctx_ = nullptr;
 };
 
-static uint64_t GenerateClusterFlag(const std::vector<std::string> &args) {
+static uint64_t GenerateClusterFlag(uint64_t flags, const std::vector<std::string> &args) {
   if (args.size() >= 2 && Cluster::SubCommandIsExecExclusive(args[1])) {
-    return kCmdExclusive;
+    return flags | kCmdExclusive;
   }
 
-  return 0;
+  return flags;
 }
 
 class CommandReadOnly : public Commander {

src/commands/cmd_function.cc

@@ -99,7 +99,16 @@ struct CommandFCall : Commander {
 
 CommandKeyRange GetScriptEvalKeyRange(const std::vector<std::string> &args);
 
-REDIS_REGISTER_COMMANDS(MakeCmdAttr<CommandFunction>("function", -2, "exclusive no-script", 0, 0, 0),
+uint64_t GenerateFunctionFlags(uint64_t flags, const std::vector<std::string> &args) {
+  if (util::EqualICase(args[1], "load") || util::EqualICase(args[1], "delete")) {
+    return flags | kCmdWrite;
+  }
+
+  return flags;
+}
+
+REDIS_REGISTER_COMMANDS(MakeCmdAttr<CommandFunction>("function", -2, "exclusive no-script", 0, 0, 0,
+                                                     GenerateFunctionFlags),
                         MakeCmdAttr<CommandFCall<>>("fcall", -3, "exclusive write no-script", GetScriptEvalKeyRange),
                         MakeCmdAttr<CommandFCall<true>>("fcall_ro", -3, "read-only ro-script no-script",
                                                         GetScriptEvalKeyRange));

src/commands/cmd_key.cc

@@ -64,6 +64,41 @@ class CommandMove : public Commander {
   }
 };
 
+class CommandMoveX : public Commander {
+ public:
+  Status Execute(Server *srv, Connection *conn, std::string *output) override {
+    std::string &key = args_[1], &token = args_[2];
+
+    redis::Database redis(srv->storage, conn->GetNamespace());
+
+    std::string ns;
+    AuthResult auth_result = srv->AuthenticateUser(token, &ns);
+    switch (auth_result) {
+      case AuthResult::NO_REQUIRE_PASS:
+        return {Status::NotOK, "Forbidden to move key when requirepass is empty"};
+      case AuthResult::INVALID_PASSWORD:
+        return {Status::NotOK, "Invalid password"};
+      case AuthResult::IS_USER:
+      case AuthResult::IS_ADMIN:
+        break;
+    }
+
+    bool ret = true;
+    bool key_exist = true;
+    std::string ns_key = redis.AppendNamespacePrefix(key);
+    std::string new_ns_key = ComposeNamespaceKey(ns, key, srv->storage->IsSlotIdEncoded());
+    auto s = redis.Move(ns_key, new_ns_key, true, &ret, &key_exist);
+    if (!s.ok()) return {Status::RedisExecErr, s.ToString()};
+
+    if (ret && key_exist) {
+      *output = redis::Integer(1);
+    } else {
+      *output = redis::Integer(0);
+    }
+    return Status::OK();
+  }
+};
+
 class CommandObject : public Commander {
  public:
   Status Execute(Server *srv, Connection *conn, std::string *output) override {
@@ -312,9 +347,12 @@ class CommandRename : public Commander {
   Status Execute(Server *srv, Connection *conn, std::string *output) override {
     redis::Database redis(srv->storage, conn->GetNamespace());
     bool ret = true;
-
-    auto s = redis.Rename(args_[1], args_[2], false, &ret);
+    bool key_exist = true;
+    std::string ns_key = redis.AppendNamespacePrefix(args_[1]);
+    std::string new_ns_key = redis.AppendNamespacePrefix(args_[2]);
+    auto s = redis.Move(ns_key, new_ns_key, false, &ret, &key_exist);
     if (!s.ok()) return {Status::RedisExecErr, s.ToString()};
+    if (!key_exist) return {Status::RedisExecErr, rocksdb::Status::InvalidArgument("ERR no such key").ToString()};
 
     *output = redis::SimpleString("OK");
     return Status::OK();
@@ -326,8 +364,12 @@ class CommandRenameNX : public Commander {
   Status Execute(Server *srv, Connection *conn, std::string *output) override {
     redis::Database redis(srv->storage, conn->GetNamespace());
     bool ret = true;
-    auto s = redis.Rename(args_[1], args_[2], true, &ret);
+    bool key_exist = true;
+    std::string ns_key = redis.AppendNamespacePrefix(args_[1]);
+    std::string new_ns_key = redis.AppendNamespacePrefix(args_[2]);
+    auto s = redis.Move(ns_key, new_ns_key, true, &ret, &key_exist);
     if (!s.ok()) return {Status::RedisExecErr, s.ToString()};
+    if (!key_exist) return {Status::RedisExecErr, rocksdb::Status::InvalidArgument("ERR no such key").ToString()};
     if (ret) {
       *output = redis::Integer(1);
     } else {
@@ -341,6 +383,7 @@ REDIS_REGISTER_COMMANDS(MakeCmdAttr<CommandTTL>("ttl", 2, "read-only", 1, 1, 1),
                         MakeCmdAttr<CommandPTTL>("pttl", 2, "read-only", 1, 1, 1),
                         MakeCmdAttr<CommandType>("type", 2, "read-only", 1, 1, 1),
                         MakeCmdAttr<CommandMove>("move", 3, "write", 1, 1, 1),
+                        MakeCmdAttr<CommandMoveX>("movex", 3, "write", 1, 1, 1),
                         MakeCmdAttr<CommandObject>("object", 3, "read-only", 2, 2, 1),
                         MakeCmdAttr<CommandExists>("exists", -2, "read-only", 1, -1, 1),
                         MakeCmdAttr<CommandPersist>("persist", 2, "write", 1, 1, 1),

src/commands/cmd_script.cc

@@ -116,6 +116,14 @@ CommandKeyRange GetScriptEvalKeyRange(const std::vector<std::string> &args) {
   return {3, 2 + numkeys, 1};
 }
 
+uint64_t GenerateScriptFlags(uint64_t flags, const std::vector<std::string> &args) {
+  if (util::EqualICase(args[1], "load") || util::EqualICase(args[1], "flush")) {
+    return flags | kCmdWrite;
+  }
+
+  return flags;
+}
+
 REDIS_REGISTER_COMMANDS(MakeCmdAttr<CommandEval>("eval", -3, "exclusive write no-script", GetScriptEvalKeyRange),
                         MakeCmdAttr<CommandEvalSHA>("evalsha", -3, "exclusive write no-script", GetScriptEvalKeyRange),
                         MakeCmdAttr<CommandEvalRO>("eval_ro", -3, "read-only no-script ro-script",

src/commands/cmd_server.cc

@@ -37,48 +37,26 @@
 
 namespace redis {
 
-enum class AuthResult {
-  OK,
-  INVALID_PASSWORD,
-  NO_REQUIRE_PASS,
-};
-
-AuthResult AuthenticateUser(Server *srv, Connection *conn, const std::string &user_password) {
-  auto ns = srv->GetNamespace()->GetByToken(user_password);
-  if (ns.IsOK()) {
-    conn->SetNamespace(ns.GetValue());
-    conn->BecomeUser();
-    return AuthResult::OK;
-  }
-
-  const auto &requirepass = srv->GetConfig()->requirepass;
-  if (!requirepass.empty() && user_password != requirepass) {
-    return AuthResult::INVALID_PASSWORD;
-  }
-
-  conn->SetNamespace(kDefaultNamespace);
-  conn->BecomeAdmin();
-  if (requirepass.empty()) {
-    return AuthResult::NO_REQUIRE_PASS;
-  }
-
-  return AuthResult::OK;
-}
-
 class CommandAuth : public Commander {
  public:
   Status Execute(Server *srv, Connection *conn, std::string *output) override {
     auto &user_password = args_[1];
-    AuthResult result = AuthenticateUser(srv, conn, user_password);
+    std::string ns;
+    AuthResult result = srv->AuthenticateUser(user_password, &ns);
     switch (result) {
-      case AuthResult::OK:
-        *output = redis::SimpleString("OK");
-        break;
-      case AuthResult::INVALID_PASSWORD:
-        return {Status::RedisExecErr, "invalid password"};
       case AuthResult::NO_REQUIRE_PASS:
         return {Status::RedisExecErr, "Client sent AUTH, but no password is set"};
+      case AuthResult::INVALID_PASSWORD:
+        return {Status::RedisExecErr, "Invalid password"};
+      case AuthResult::IS_USER:
+        conn->BecomeUser();
+        break;
+      case AuthResult::IS_ADMIN:
+        conn->BecomeAdmin();
+        break;
     }
+    conn->SetNamespace(ns);
+    *output = redis::SimpleString("OK");
     return Status::OK();
   }
 };
@@ -773,20 +751,26 @@ class CommandHello final : public Commander {
       if (util::ToLower(opt) == "auth" && more_args != 0) {
         if (more_args == 2 || more_args == 4) {
           if (args_[next_arg + 1] != "default") {
-            return {Status::NotOK, "invalid password"};
+            return {Status::NotOK, "Invalid password"};
           }
           next_arg++;
         }
         const auto &user_password = args_[next_arg + 1];
-        auto auth_result = AuthenticateUser(srv, conn, user_password);
+        std::string ns;
+        AuthResult auth_result = srv->AuthenticateUser(user_password, &ns);
         switch (auth_result) {
-          case AuthResult::INVALID_PASSWORD:
-            return {Status::NotOK, "invalid password"};
           case AuthResult::NO_REQUIRE_PASS:
             return {Status::NotOK, "Client sent AUTH, but no password is set"};
-          case AuthResult::OK:
+          case AuthResult::INVALID_PASSWORD:
+            return {Status::NotOK, "Invalid password"};
+          case AuthResult::IS_USER:
+            conn->BecomeUser();
+            break;
+          case AuthResult::IS_ADMIN:
+            conn->BecomeAdmin();
             break;
         }
+        conn->SetNamespace(ns);
         next_arg += 1;
       } else if (util::ToLower(opt) == "setname" && more_args != 0) {
         const std::string &name = args_[next_arg + 1];
@@ -1066,12 +1050,12 @@ class CommandStats : public Commander {
   }
 };
 
-static uint64_t GenerateConfigFlag(const std::vector<std::string> &args) {
+static uint64_t GenerateConfigFlag(uint64_t flags, const std::vector<std::string> &args) {
   if (args.size() >= 2 && util::EqualICase(args[1], "set")) {
-    return kCmdExclusive;
+    return flags | kCmdExclusive;
   }
 
-  return 0;
+  return flags;
 }
 
 class CommandLastSave : public Commander {
@@ -1207,72 +1191,6 @@ class CommandRdb : public Commander {
   uint32_t db_index_ = 0;
 };
 
-class CommandAnalyze : public Commander {
- public:
-  Status Parse(const std::vector<std::string> &args) override {
-    if (args.size() <= 1) return {Status::RedisExecErr, errInvalidSyntax};
-    for (unsigned int i = 1; i < args.size(); ++i) {
-      command_args_.push_back(args[i]);
-    }
-    return Status::OK();
-  }
-  Status Execute(Server *srv, Connection *conn, std::string *output) override {
-    auto commands = redis::CommandTable::Get();
-    auto cmd_iter = commands->find(util::ToLower(command_args_[0]));
-    if (cmd_iter == commands->end()) {
-      // unsupported redis command
-      return {Status::RedisExecErr, errInvalidSyntax};
-    }
-    auto redis_cmd = cmd_iter->second;
-    auto cmd = redis_cmd->factory();
-    cmd->SetAttributes(redis_cmd);
-    cmd->SetArgs(command_args_);
-
-    int arity = cmd->GetAttributes()->arity;
-    if ((arity > 0 && static_cast<int>(command_args_.size()) != arity) ||
-        (arity < 0 && static_cast<int>(command_args_.size()) < -arity)) {
-      *output = redis::Error("ERR wrong number of arguments");
-      return {Status::RedisExecErr, errWrongNumOfArguments};
-    }
-
-    auto s = cmd->Parse(command_args_);
-    if (!s.IsOK()) {
-      return s;
-    }
-
-    auto prev_perf_level = rocksdb::GetPerfLevel();
-    rocksdb::SetPerfLevel(rocksdb::PerfLevel::kEnableTimeExceptForMutex);
-    rocksdb::get_perf_context()->Reset();
-    rocksdb::get_iostats_context()->Reset();
-
-    std::string command_output;
-    s = cmd->Execute(srv, conn, &command_output);
-    if (!s.IsOK()) {
-      return s;
-    }
-
-    if (command_output[0] == '-') {
-      *output = command_output;
-      return s;
-    }
-
-    std::string perf_context = rocksdb::get_perf_context()->ToString(true);
-    std::string iostats_context = rocksdb::get_iostats_context()->ToString(true);
-    rocksdb::get_perf_context()->Reset();
-    rocksdb::get_iostats_context()->Reset();
-    rocksdb::SetPerfLevel(prev_perf_level);
-
-    *output = redis::MultiLen(3);  // command output + perf context + iostats context
-    *output += command_output;
-    *output += redis::BulkString(perf_context);
-    *output += redis::BulkString(iostats_context);
-    return Status::OK();
-  }
-
- private:
-  std::vector<std::string> command_args_;
-};
-
 class CommandReset : public Commander {
  public:
   Status Execute(Server *srv, Connection *conn, std::string *output) override {
@@ -1374,7 +1292,6 @@ REDIS_REGISTER_COMMANDS(MakeCmdAttr<CommandAuth>("auth", 2, "read-only ok-loadin
                         MakeCmdAttr<CommandSlaveOf>("slaveof", 3, "read-only exclusive no-script", 0, 0, 0),
                         MakeCmdAttr<CommandStats>("stats", 1, "read-only", 0, 0, 0),
                         MakeCmdAttr<CommandRdb>("rdb", -3, "write exclusive", 0, 0, 0),
-                        MakeCmdAttr<CommandAnalyze>("analyze", -1, "", 0, 0, 0),
                         MakeCmdAttr<CommandReset>("reset", 1, "ok-loading multi no-script pub-sub", 0, 0, 0),
                         MakeCmdAttr<CommandApplyBatch>("applybatch", -2, "write no-multi", 0, 0, 0), )
 }  // namespace redis

src/commands/commander.h

@@ -112,7 +112,7 @@ using CommandKeyRangeGen = std::function<CommandKeyRange(const std::vector<std::
 
 using CommandKeyRangeVecGen = std::function<std::vector<CommandKeyRange>(const std::vector<std::string> &)>;
 
-using AdditionalFlagGen = std::function<uint64_t(const std::vector<std::string> &)>;
+using AdditionalFlagGen = std::function<uint64_t(uint64_t, const std::vector<std::string> &)>;
 
 struct CommandAttributes {
   // command name
@@ -146,7 +146,7 @@ struct CommandAttributes {
 
   auto GenerateFlags(const std::vector<std::string> &args) const {
     uint64_t res = flags;
-    if (flag_gen) res |= flag_gen(args);
+    if (flag_gen) res = flag_gen(res, args);
     return res;
   }
 

src/config/config.cc

@@ -137,6 +137,7 @@ Config::Config() {
       {"slaveof", true, new StringField(&slaveof_, "")},
       {"compact-cron", false, new StringField(&compact_cron_str_, "")},
       {"bgsave-cron", false, new StringField(&bgsave_cron_str_, "")},
+      {"dbsize-scan-cron", false, new StringField(&dbsize_scan_cron_str_, "")},
       {"replica-announce-ip", false, new StringField(&replica_announce_ip, "")},
       {"replica-announce-port", false, new UInt32Field(&replica_announce_port, 0, 0, PORT_LIMIT)},
       {"compaction-checker-range", false, new StringField(&compaction_checker_range_str_, "")},
@@ -292,6 +293,11 @@ void Config::initFieldValidator() {
          std::vector<std::string> args = util::Split(v, " \t");
          return bgsave_cron.SetScheduleTime(args);
        }},
+      {"dbsize-scan-cron",
+       [this](const std::string &k, const std::string &v) -> Status {
+         std::vector<std::string> args = util::Split(v, " \t");
+         return dbsize_scan_cron.SetScheduleTime(args);
+       }},
       {"compaction-checker-range",
        [this](const std::string &k, const std::string &v) -> Status {
          if (v.empty()) {
@@ -732,7 +738,7 @@ void Config::ClearMaster() {
 
 Status Config::parseConfigFromPair(const std::pair<std::string, std::string> &input, int line_number) {
   std::string field_key = util::ToLower(input.first);
-  const char ns_str[] = "namespace.";
+  constexpr const char ns_str[] = "namespace.";
   size_t ns_str_size = sizeof(ns_str) - 1;
   if (strncasecmp(input.first.data(), ns_str, ns_str_size) == 0) {
     // namespace should keep key case-sensitive