From 314660addc4831b998969a88b1d0acb3ee6021b8 Mon Sep 17 00:00:00 2001 From: chenbt <34958405+chenbt-hz@users.noreply.github.com> Date: Fri, 22 Mar 2024 20:24:07 +0800 Subject: [PATCH] fix: spop binlog , rewritten as srem (#2541) * fix: spop binlog --- include/pika_set.h | 61 +++++++++++++++------------ src/pika_set.cc | 18 ++++++++ tests/integration/README.md | 58 +++++++++++++++++++++++++ tests/integration/README_CN.md | 50 ++++++++++++++++++++++ tests/integration/replication_test.go | 36 ++++++++++++++++ 5 files changed, 197 insertions(+), 26 deletions(-) create mode 100644 tests/integration/README.md create mode 100644 tests/integration/README_CN.md diff --git a/include/pika_set.h b/include/pika_set.h index fb41d568e4..34e47d5edc 100644 --- a/include/pika_set.h +++ b/include/pika_set.h @@ -36,10 +36,38 @@ class SAddCmd : public Cmd { void DoInitial() override; }; +class SRemCmd : public Cmd { + public: + SRemCmd(const std::string& name, int arity, uint32_t flag) + : Cmd(name, arity, flag, static_cast(AclCategory::SET)) {} + std::vector current_key() const override { + std::vector res; + res.push_back(key_); + return res; + } + void Do() override; + void DoUpdateCache() override; + void DoThroughDB() override; + void Split(const HintKeys& hint_keys) override{}; + void Merge() override{}; + Cmd* Clone() override { return new SRemCmd(*this); } + + private: + void DoInitial() override; + + private: + std::string key_; + std::vector members_; + rocksdb::Status s_; + int32_t deleted_ = 0; +}; + class SPopCmd : public Cmd { public: SPopCmd(const std::string& name, int arity, uint32_t flag) - : Cmd(name, arity, flag, static_cast(AclCategory::SET)) {} + : Cmd(name, arity, flag, static_cast(AclCategory::SET)) { + srem_cmd_ = std::make_shared(kCmdNameSRem, -3, kCmdFlagsWrite | kCmdFlagsSet); + } std::vector current_key() const override { std::vector res; res.push_back(key_); @@ -51,13 +79,18 @@ class SPopCmd : public Cmd { void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new SPopCmd(*this); } + void DoBinlog() override; + + private: + void DoInitial() override; private: std::string key_; std::vector members_; + // used for write binlog + std::shared_ptr srem_cmd_; int64_t count_ = 1; rocksdb::Status s_; - void DoInitial() override; }; class SCardCmd : public Cmd { @@ -131,30 +164,6 @@ class SScanCmd : public Cmd { } }; -class SRemCmd : public Cmd { - public: - SRemCmd(const std::string& name, int arity, uint32_t flag) - : Cmd(name, arity, flag, static_cast(AclCategory::SET)) {} - std::vector current_key() const override { - std::vector res; - res.push_back(key_); - return res; - } - void Do() override; - void DoUpdateCache() override; - void DoThroughDB() override; - void Split(const HintKeys& hint_keys) override{}; - void Merge() override{}; - Cmd* Clone() override { return new SRemCmd(*this); } - - private: - std::string key_; - std::vector members_; - rocksdb::Status s_; - int32_t deleted_ = 0; - void DoInitial() override; -}; - class SUnionCmd : public Cmd { public: SUnionCmd(const std::string& name, int arity, uint32_t flag) diff --git a/src/pika_set.cc b/src/pika_set.cc index 6f44478c9d..31a92a305a 100644 --- a/src/pika_set.cc +++ b/src/pika_set.cc @@ -91,6 +91,24 @@ void SPopCmd::DoUpdateCache() { } } +void SPopCmd::DoBinlog() { + if (!s_.ok()) { + return; + } + + PikaCmdArgsType srem_args; + srem_args.emplace_back("srem"); + srem_args.emplace_back(key_); + for (auto m = members_.begin(); m != members_.end(); ++m) { + srem_args.emplace_back(*m); + } + + srem_cmd_->Initial(srem_args, db_name_); + srem_cmd_->SetConn(GetConn()); + srem_cmd_->SetResp(resp_.lock()); + srem_cmd_->DoBinlog(); +} + void SCardCmd::DoInitial() { if (!CheckArg(argv_.size())) { res_.SetRes(CmdRes::kWrongNum, kCmdNameSCard); diff --git a/tests/integration/README.md b/tests/integration/README.md new file mode 100644 index 0000000000..99f80a76f2 --- /dev/null +++ b/tests/integration/README.md @@ -0,0 +1,58 @@ +# README +This is an integration test code for Pika written in Golang. By default, the tests are automatically executed after code is submitted to the Pika repository. + +[中文](https://github.com/OpenAtomFoundation/pika/blob/unstable/tests/integration/README_CN.md) + +## Running Golang Integration Tests Locally +If you want to run the tests locally, you need to complete the following preparations: + +### 1. Prepare the Program and Configuration Files +Ensure that the compiled Pika program is present in the ../../output/pika directory. +(You can also compile the Pika program for Mac in advance and manually copy the Pika file to the directory specified in start_master_and_slave.sh, copy the unchanged pika configuration files to the test directory; or directly modify the startup path in start_master_and_slave.sh.) + +The prerequisite for manually executing the tests is having Ginkgo installed, for example: +``` +cd tests/integration/ +go get github.com/onsi/ginkgo/v2/ginkgo +go install github.com/onsi/ginkgo/v2/ginkgo +go get github.com/onsi/gomega/... +``` + +### 2.Start the Pika Service +Execute in the project root directory: +``` +cd tests + +sh ./integration/start_master_and_slave.sh +``` + +### 3.Run Tests +Execute in the tests directory: +``` +cd integration +sh integrate_test.sh +``` + +### 4.Run Tests for a Specific File + +Add environment variables: +``` +go env |grep GOBIN +export PATH="$PATH:$GOBIN" +``` + +Execute`ginkgo --focus-file="slowlog_test.go" -vv` + +Refer to the Ginkgo framework: https://onsi.github.io/ginkgo/#mental-model-ginkgo-assumes-specs-are-independent +Note: +`--focus-file` executes matching files + +`--skip-file` filters out non-matching files + +`--focus` executes tests matching descriptions + +`--skip` filters out tests matching descriptions + +For example, `ginkgo --focus=dog --focus=fish --skip=cat --skip=purple` + +This will only run tests described as "likes dogs", "likes dog fish", while skipping tests related to "purple". diff --git a/tests/integration/README_CN.md b/tests/integration/README_CN.md new file mode 100644 index 0000000000..9cfa09fdb1 --- /dev/null +++ b/tests/integration/README_CN.md @@ -0,0 +1,50 @@ +# README +这是用golang编写的pika 集成测试代码,默认提交代码到pika仓库后会自动运行。 + +## 本地跑golang集成测试 +如果你想在本地运行测试，需要完成以下的准备工作： + +### 1.准备程序和配置文件 +在../../output/pika目录确保有编译好的pika程序。 +（也可以提前编译好mac版本的pika程序,并手动将pika文件拷贝到start_master_and_slave.sh中制定的目录，将pika未改动的conf文件拷贝到test目录；或者直接修改start_master_and_slave.sh启动路径。） + +手动执行测试的前提是，已安装ginkgo,例如 +``` +cd tests/integration/ +go get github.com/onsi/ginkgo/v2/ginkgo +go install github.com/onsi/ginkgo/v2/ginkgo +go get github.com/onsi/gomega/... +``` + +### 2.启动Pika服务 +在项目主目录下执行 +``` +cd tests + +sh ./integration/start_master_and_slave.sh +``` + +### 3.运行测试 +在tests目录下执行 +cd integration +sh integrate_test.sh + +### 4.运行指定文件的测试 + + +添加环境变量 +``` +go env |grep GOBIN +export PATH="$PATH:$GOBIN" +``` + +执行`ginkgo --focus-file="slowlog_test.go" -vv` + +ginkgo框架参考: https://onsi.github.io/ginkgo/#mental-model-ginkgo-assumes-specs-are-independent +备注: +`--focus-file`执行匹配文件 +`--skip-file`过滤不匹配的文件 +`--focus`执行匹配描述的测试 +`--skip`过滤匹配描述的测试 +例如，`ginkgo --focus=dog --focus=fish --skip=cat --skip=purple` +则只运行运行It(描述内容中)例如"likes dogs"、"likes dog fish"的单测，而跳过"purple"相关的测试。 diff --git a/tests/integration/replication_test.go b/tests/integration/replication_test.go index 2984170051..e72d01be86 100644 --- a/tests/integration/replication_test.go +++ b/tests/integration/replication_test.go @@ -284,6 +284,27 @@ func randomSunionstroeThread(ctx *context.Context, clientMaster *redis.Client, w } } +func randomSpopstroeThread(ctx *context.Context, clientMaster *redis.Client, wg *sync.WaitGroup) { + defer wg.Done() + for i := 0; i < 5; i++ { + clientMaster.SAdd(*ctx, "set1", randomString(5)) + clientMaster.SAdd(*ctx, "set1", randomString(5)) + clientMaster.SAdd(*ctx, "set1", randomString(5)) + clientMaster.SAdd(*ctx, "set1", randomString(5)) + clientMaster.SAdd(*ctx, "set1", randomString(5)) + clientMaster.SAdd(*ctx, "set1", randomString(5)) + clientMaster.SPop(*ctx, "set1") + + clientMaster.SAdd(*ctx, "set2", randomString(5)) + clientMaster.SAdd(*ctx, "set2", randomString(5)) + clientMaster.SAdd(*ctx, "set2", randomString(5)) + clientMaster.SAdd(*ctx, "set2", randomString(5)) + clientMaster.SAdd(*ctx, "set2", randomString(5)) + clientMaster.SAdd(*ctx, "set2", randomString(5)) + clientMaster.SPopN(*ctx, "set2", int64(randomInt(5))) + } +} + func randomXaddThread(ctx *context.Context, clientMaster *redis.Client, wg *sync.WaitGroup) { defer wg.Done() for i := 0; i < 5; i++ { @@ -566,6 +587,21 @@ var _ = Describe("should replication ", func() { clientMaster.Del(ctx, "set1", "set2", "set_out") log.Println("randomSunionstore test success") + log.Println("randomSpopstore test start") + execute(&ctx, clientMaster, 4, randomSpopstroeThread) + master_spopstore_set := clientMaster.SMembers(ctx, "set1") + Expect(master_spopstore_set.Err()).NotTo(HaveOccurred()) + slave_spopstore_set := clientSlave.SMembers(ctx, "set1") + Expect(slave_spopstore_set.Err()).NotTo(HaveOccurred()) + Expect(master_spopstore_set.Val()).To(Equal(slave_spopstore_set.Val())) + master_spopstore_set2 := clientMaster.SMembers(ctx, "set2") + Expect(master_spopstore_set2.Err()).NotTo(HaveOccurred()) + slave_spopstore_set2 := clientSlave.SMembers(ctx, "set2") + Expect(slave_spopstore_set2.Err()).NotTo(HaveOccurred()) + Expect(master_spopstore_set2.Val()).To(Equal(slave_spopstore_set2.Val())) + clientMaster.Del(ctx, "set1", "set2") + log.Println("randomSpopstore test success") + // Stream replication test log.Println("randomXadd test start") clientMaster.Del(ctx, "mystream")