From 56cd4c49075b773e192d6b21974ff5915d2908a9 Mon Sep 17 00:00:00 2001 From: chenbt Date: Mon, 18 Mar 2024 15:39:42 +0800 Subject: [PATCH 1/8] fix: spop binlog --- CMakeLists.txt | 8 +++++--- include/pika_set.h | 2 ++ src/pika_set.cc | 25 +++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 485ff0fe25..d94bb907cc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,8 +25,8 @@ endif() ############# You should enable sanitizer if you are developing pika ############# # Uncomment the following two lines to enable AddressSanitizer to detect memory leaks and other memory-related bugs. -#set(CMAKE_BUILD_TYPE "Debug") -#set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fsanitize=address -O0 -fno-omit-frame-pointer -fno-optimize-sibling-calls") +# set(CMAKE_BUILD_TYPE "Debug") +# set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fsanitize=address -O0 -fno-omit-frame-pointer -fno-optimize-sibling-calls") # [Notice] AddressSanitizer and ThreadSanitizer can not be enabled at the same time. @@ -51,7 +51,9 @@ elseif(${BUILD_TYPE} STREQUAL RELWITHDEBINFO) set(LIB_BUILD_TYPE RELWITHDEBINFO) else() set(LIB_BUILD_TYPE RELEASE) - set(CMAKE_CXX_FLAGS_RELEASE "-O2 -g -DNDEBUG") + # set(CMAKE_CXX_FLAGS_RELEASE "-O2 -g -DNDEBUG") + set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fsanitize=address -O0 -fno-omit-frame-pointer -fno-optimize-sibling-calls") + endif() if(CMAKE_SYSTEM_NAME MATCHES "Darwin") diff --git a/include/pika_set.h b/include/pika_set.h index fb41d568e4..acdcf46c32 100644 --- a/include/pika_set.h +++ b/include/pika_set.h @@ -51,10 +51,12 @@ class SPopCmd : public Cmd { void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new SPopCmd(*this); } + void DoBinlog() override; private: std::string key_; std::vector members_; + std::vector rem_members_; int64_t count_ = 1; rocksdb::Status s_; void DoInitial() override; diff --git a/src/pika_set.cc b/src/pika_set.cc index 6f44478c9d..5793164bbb 100644 --- a/src/pika_set.cc +++ b/src/pika_set.cc @@ -72,6 +72,7 @@ void SPopCmd::Do() { for (const auto& member : members_) { res_.AppendStringLenUint64(member.size()); res_.AppendContent(member); + // rem_members_.emplace_back(member); } } else if (s_.IsNotFound()) { res_.AppendContent("$-1"); @@ -91,6 +92,30 @@ void SPopCmd::DoUpdateCache() { } } +void SPopCmd::DoBinlog() { + if (!s_.ok()) { + return; + } + PikaCmdArgsType srem_args; + std::shared_ptr srem_cmd_; + //SPopCmd use "SREM" + srem_args.emplace_back("srem"); + srem_args.emplace_back(key_); + // for (auto m = rem_members_.begin(); m != rem_members_.end(); ++m) { + // srem_args.emplace_back(*m); + // } + for (auto m = members_.begin(); m != members_.end(); ++m) { + srem_args.emplace_back(*m); + } + + srem_cmd_->Initial(srem_args, db_name_); + + srem_cmd_->SetConn(GetConn()); + srem_cmd_->SetResp(resp_.lock()); + + srem_cmd_->DoBinlog(); +} + void SCardCmd::DoInitial() { if (!CheckArg(argv_.size())) { res_.SetRes(CmdRes::kWrongNum, kCmdNameSCard); From 0e6b6f9e69e0f7a0dc465eaa9ebe5e927cb45c06 Mon Sep 17 00:00:00 2001 From: chenbt Date: Thu, 21 Mar 2024 14:40:13 +0800 Subject: [PATCH 2/8] fix: spop binlog --- include/pika_set.h | 55 ++++++++++++++++++++++++---------------------- src/pika_set.cc | 8 +------ 2 files changed, 30 insertions(+), 33 deletions(-) diff --git a/include/pika_set.h b/include/pika_set.h index acdcf46c32..b805eae59a 100644 --- a/include/pika_set.h +++ b/include/pika_set.h @@ -36,10 +36,36 @@ class SAddCmd : public Cmd { void DoInitial() override; }; +class SRemCmd : public Cmd { + public: + SRemCmd(const std::string& name, int arity, uint32_t flag) + : Cmd(name, arity, flag, static_cast(AclCategory::SET)) {} + std::vector current_key() const override { + std::vector res; + res.push_back(key_); + return res; + } + void Do() override; + void DoUpdateCache() override; + void DoThroughDB() override; + void Split(const HintKeys& hint_keys) override{}; + void Merge() override{}; + Cmd* Clone() override { return new SRemCmd(*this); } + + private: + std::string key_; + std::vector members_; + rocksdb::Status s_; + int32_t deleted_ = 0; + void DoInitial() override; +}; + class SPopCmd : public Cmd { public: SPopCmd(const std::string& name, int arity, uint32_t flag) - : Cmd(name, arity, flag, static_cast(AclCategory::SET)) {} + : Cmd(name, arity, flag, static_cast(AclCategory::SET)) { + srem_cmd_ = std::make_shared(kCmdNameSRem, -3, kCmdFlagsWrite | kCmdFlagsSet); + } std::vector current_key() const override { std::vector res; res.push_back(key_); @@ -56,7 +82,8 @@ class SPopCmd : public Cmd { private: std::string key_; std::vector members_; - std::vector rem_members_; + // used for write binlog + std::shared_ptr srem_cmd_; int64_t count_ = 1; rocksdb::Status s_; void DoInitial() override; @@ -133,30 +160,6 @@ class SScanCmd : public Cmd { } }; -class SRemCmd : public Cmd { - public: - SRemCmd(const std::string& name, int arity, uint32_t flag) - : Cmd(name, arity, flag, static_cast(AclCategory::SET)) {} - std::vector current_key() const override { - std::vector res; - res.push_back(key_); - return res; - } - void Do() override; - void DoUpdateCache() override; - void DoThroughDB() override; - void Split(const HintKeys& hint_keys) override{}; - void Merge() override{}; - Cmd* Clone() override { return new SRemCmd(*this); } - - private: - std::string key_; - std::vector members_; - rocksdb::Status s_; - int32_t deleted_ = 0; - void DoInitial() override; -}; - class SUnionCmd : public Cmd { public: SUnionCmd(const std::string& name, int arity, uint32_t flag) diff --git a/src/pika_set.cc b/src/pika_set.cc index 5793164bbb..b6653ebaff 100644 --- a/src/pika_set.cc +++ b/src/pika_set.cc @@ -96,23 +96,17 @@ void SPopCmd::DoBinlog() { if (!s_.ok()) { return; } + PikaCmdArgsType srem_args; - std::shared_ptr srem_cmd_; - //SPopCmd use "SREM" srem_args.emplace_back("srem"); srem_args.emplace_back(key_); - // for (auto m = rem_members_.begin(); m != rem_members_.end(); ++m) { - // srem_args.emplace_back(*m); - // } for (auto m = members_.begin(); m != members_.end(); ++m) { srem_args.emplace_back(*m); } srem_cmd_->Initial(srem_args, db_name_); - srem_cmd_->SetConn(GetConn()); srem_cmd_->SetResp(resp_.lock()); - srem_cmd_->DoBinlog(); } From 84ebbc11e4c7c1c8fa65c5cd5acd0aa9d258348d Mon Sep 17 00:00:00 2001 From: chenbt Date: Thu, 21 Mar 2024 16:23:51 +0800 Subject: [PATCH 3/8] rollback --- CMakeLists.txt | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d94bb907cc..485ff0fe25 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,8 +25,8 @@ endif() ############# You should enable sanitizer if you are developing pika ############# # Uncomment the following two lines to enable AddressSanitizer to detect memory leaks and other memory-related bugs. -# set(CMAKE_BUILD_TYPE "Debug") -# set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fsanitize=address -O0 -fno-omit-frame-pointer -fno-optimize-sibling-calls") +#set(CMAKE_BUILD_TYPE "Debug") +#set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fsanitize=address -O0 -fno-omit-frame-pointer -fno-optimize-sibling-calls") # [Notice] AddressSanitizer and ThreadSanitizer can not be enabled at the same time. @@ -51,9 +51,7 @@ elseif(${BUILD_TYPE} STREQUAL RELWITHDEBINFO) set(LIB_BUILD_TYPE RELWITHDEBINFO) else() set(LIB_BUILD_TYPE RELEASE) - # set(CMAKE_CXX_FLAGS_RELEASE "-O2 -g -DNDEBUG") - set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fsanitize=address -O0 -fno-omit-frame-pointer -fno-optimize-sibling-calls") - + set(CMAKE_CXX_FLAGS_RELEASE "-O2 -g -DNDEBUG") endif() if(CMAKE_SYSTEM_NAME MATCHES "Darwin") From 1174c65a778a8e3e3a2674ac8e4398b40a931b7d Mon Sep 17 00:00:00 2001 From: chenbt Date: Thu, 21 Mar 2024 16:32:27 +0800 Subject: [PATCH 4/8] delete a note --- src/pika_set.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/pika_set.cc b/src/pika_set.cc index b6653ebaff..31a92a305a 100644 --- a/src/pika_set.cc +++ b/src/pika_set.cc @@ -72,7 +72,6 @@ void SPopCmd::Do() { for (const auto& member : members_) { res_.AppendStringLenUint64(member.size()); res_.AppendContent(member); - // rem_members_.emplace_back(member); } } else if (s_.IsNotFound()) { res_.AppendContent("$-1"); @@ -96,7 +95,7 @@ void SPopCmd::DoBinlog() { if (!s_.ok()) { return; } - + PikaCmdArgsType srem_args; srem_args.emplace_back("srem"); srem_args.emplace_back(key_); From 6c9cc24449e451e63219893129744ac8a923c331 Mon Sep 17 00:00:00 2001 From: chenbt Date: Fri, 22 Mar 2024 09:22:47 +0800 Subject: [PATCH 5/8] Adjust function declaration postion --- include/pika_set.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/include/pika_set.h b/include/pika_set.h index b805eae59a..34e47d5edc 100644 --- a/include/pika_set.h +++ b/include/pika_set.h @@ -52,12 +52,14 @@ class SRemCmd : public Cmd { void Merge() override{}; Cmd* Clone() override { return new SRemCmd(*this); } + private: + void DoInitial() override; + private: std::string key_; std::vector members_; rocksdb::Status s_; int32_t deleted_ = 0; - void DoInitial() override; }; class SPopCmd : public Cmd { @@ -79,6 +81,9 @@ class SPopCmd : public Cmd { Cmd* Clone() override { return new SPopCmd(*this); } void DoBinlog() override; + private: + void DoInitial() override; + private: std::string key_; std::vector members_; @@ -86,7 +91,6 @@ class SPopCmd : public Cmd { std::shared_ptr srem_cmd_; int64_t count_ = 1; rocksdb::Status s_; - void DoInitial() override; }; class SCardCmd : public Cmd { From 7f1c61dd8064c12977d729cb40ce7fa162bee0f0 Mon Sep 17 00:00:00 2001 From: chenbt Date: Fri, 22 Mar 2024 11:28:02 +0800 Subject: [PATCH 6/8] add spop in replication test --- tests/integration/README.md | 50 +++++++++++++++++++++++++++ tests/integration/replication_test.go | 36 +++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 tests/integration/README.md diff --git a/tests/integration/README.md b/tests/integration/README.md new file mode 100644 index 0000000000..0d4d566353 --- /dev/null +++ b/tests/integration/README.md @@ -0,0 +1,50 @@ +# README +这是用golang编写的pika 集成测试代码,默认提交代码到pika仓库后会自动运行。 + +## 本地跑golang集成测试 +如果你想在本地运行测试，需要完成以下的准备工作： + +### 1.准备程序和配置文件 +在../../output/pika目录确保有编译好的pika程序。 +（也可以提前编译好mac版本的pika程序,并手动将pika文件拷贝到start_master_and_slave.sh中制定的目录，将pika未改动的conf文件拷贝到test目录；或者直接修改start_master_and_slave.sh启动路径。） + +手动执行测试的前提是，已安装ginkgo,例如 +``` +cd tests/integration/ +go get github.com/onsi/ginkgo/v2/ginkgo +go install github.com/onsi/ginkgo/v2/ginkgo +go get github.com/onsi/gomega/... +``` + +### 2.启动Pika服务 +在项目主目录下执行 +``` +cd test + +sh ./integration/start_master_and_slave.sh +``` + +### 3.运行单测 +在test目录下执行 +cd integration +sh integrate_test.sh + +### 4.运行指定文件的测试 + + +添加环境变量 +``` +go env |grep GOPATH +export PATH="$PATH:$GOPATH" +``` + +执行`ginkgo --focus-file="slowlog_test.go" -vv` + +ginkgo框架参考 https://onsi.github.io/ginkgo/#mental-model-ginkgo-assumes-specs-are-independent +备注: +--focus-file执行匹配文件 +--skip-file过滤不匹配的文件 +--focus执行匹配描述的测试 +--skip过滤匹配描述的测试 +例如，`ginkgo --focus=dog --focus=fish --skip=cat --skip=purple` +则只运行运行It(描述内容中)例如"likes dogs"、"likes dog fish"的单测，而跳过"purple"相关的测试。 diff --git a/tests/integration/replication_test.go b/tests/integration/replication_test.go index 2984170051..e72d01be86 100644 --- a/tests/integration/replication_test.go +++ b/tests/integration/replication_test.go @@ -284,6 +284,27 @@ func randomSunionstroeThread(ctx *context.Context, clientMaster *redis.Client, w } } +func randomSpopstroeThread(ctx *context.Context, clientMaster *redis.Client, wg *sync.WaitGroup) { + defer wg.Done() + for i := 0; i < 5; i++ { + clientMaster.SAdd(*ctx, "set1", randomString(5)) + clientMaster.SAdd(*ctx, "set1", randomString(5)) + clientMaster.SAdd(*ctx, "set1", randomString(5)) + clientMaster.SAdd(*ctx, "set1", randomString(5)) + clientMaster.SAdd(*ctx, "set1", randomString(5)) + clientMaster.SAdd(*ctx, "set1", randomString(5)) + clientMaster.SPop(*ctx, "set1") + + clientMaster.SAdd(*ctx, "set2", randomString(5)) + clientMaster.SAdd(*ctx, "set2", randomString(5)) + clientMaster.SAdd(*ctx, "set2", randomString(5)) + clientMaster.SAdd(*ctx, "set2", randomString(5)) + clientMaster.SAdd(*ctx, "set2", randomString(5)) + clientMaster.SAdd(*ctx, "set2", randomString(5)) + clientMaster.SPopN(*ctx, "set2", int64(randomInt(5))) + } +} + func randomXaddThread(ctx *context.Context, clientMaster *redis.Client, wg *sync.WaitGroup) { defer wg.Done() for i := 0; i < 5; i++ { @@ -566,6 +587,21 @@ var _ = Describe("should replication ", func() { clientMaster.Del(ctx, "set1", "set2", "set_out") log.Println("randomSunionstore test success") + log.Println("randomSpopstore test start") + execute(&ctx, clientMaster, 4, randomSpopstroeThread) + master_spopstore_set := clientMaster.SMembers(ctx, "set1") + Expect(master_spopstore_set.Err()).NotTo(HaveOccurred()) + slave_spopstore_set := clientSlave.SMembers(ctx, "set1") + Expect(slave_spopstore_set.Err()).NotTo(HaveOccurred()) + Expect(master_spopstore_set.Val()).To(Equal(slave_spopstore_set.Val())) + master_spopstore_set2 := clientMaster.SMembers(ctx, "set2") + Expect(master_spopstore_set2.Err()).NotTo(HaveOccurred()) + slave_spopstore_set2 := clientSlave.SMembers(ctx, "set2") + Expect(slave_spopstore_set2.Err()).NotTo(HaveOccurred()) + Expect(master_spopstore_set2.Val()).To(Equal(slave_spopstore_set2.Val())) + clientMaster.Del(ctx, "set1", "set2") + log.Println("randomSpopstore test success") + // Stream replication test log.Println("randomXadd test start") clientMaster.Del(ctx, "mystream") From 5d89da6bb69a459ecf76063fd29e269d0a9f0d92 Mon Sep 17 00:00:00 2001 From: chenbt Date: Fri, 22 Mar 2024 11:38:41 +0800 Subject: [PATCH 7/8] fix README.md --- tests/integration/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/README.md b/tests/integration/README.md index 0d4d566353..b1e5b8483d 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -34,8 +34,8 @@ sh integrate_test.sh 添加环境变量 ``` -go env |grep GOPATH -export PATH="$PATH:$GOPATH" +go env |grep GOBIN +export PATH="$PATH:$GOBIN" ``` 执行`ginkgo --focus-file="slowlog_test.go" -vv` From b29584f51ef0c368996dcfe4e3904892acc5b2b1 Mon Sep 17 00:00:00 2001 From: chenbt Date: Fri, 22 Mar 2024 13:51:47 +0800 Subject: [PATCH 8/8] Add a English README.md --- tests/integration/README.md | 58 +++++++++++++++++++--------------- tests/integration/README_CN.md | 50 +++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 25 deletions(-) create mode 100644 tests/integration/README_CN.md diff --git a/tests/integration/README.md b/tests/integration/README.md index b1e5b8483d..99f80a76f2 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -1,14 +1,16 @@ # README -这是用golang编写的pika 集成测试代码,默认提交代码到pika仓库后会自动运行。 +This is an integration test code for Pika written in Golang. By default, the tests are automatically executed after code is submitted to the Pika repository. -## 本地跑golang集成测试 -如果你想在本地运行测试，需要完成以下的准备工作： +[中文](https://github.com/OpenAtomFoundation/pika/blob/unstable/tests/integration/README_CN.md) -### 1.准备程序和配置文件 -在../../output/pika目录确保有编译好的pika程序。 -（也可以提前编译好mac版本的pika程序,并手动将pika文件拷贝到start_master_and_slave.sh中制定的目录，将pika未改动的conf文件拷贝到test目录；或者直接修改start_master_and_slave.sh启动路径。） +## Running Golang Integration Tests Locally +If you want to run the tests locally, you need to complete the following preparations: -手动执行测试的前提是，已安装ginkgo,例如 +### 1. Prepare the Program and Configuration Files +Ensure that the compiled Pika program is present in the ../../output/pika directory. +(You can also compile the Pika program for Mac in advance and manually copy the Pika file to the directory specified in start_master_and_slave.sh, copy the unchanged pika configuration files to the test directory; or directly modify the startup path in start_master_and_slave.sh.) + +The prerequisite for manually executing the tests is having Ginkgo installed, for example: ``` cd tests/integration/ go get github.com/onsi/ginkgo/v2/ginkgo @@ -16,35 +18,41 @@ go install github.com/onsi/ginkgo/v2/ginkgo go get github.com/onsi/gomega/... ``` -### 2.启动Pika服务 -在项目主目录下执行 +### 2.Start the Pika Service +Execute in the project root directory: ``` -cd test +cd tests sh ./integration/start_master_and_slave.sh ``` -### 3.运行单测 -在test目录下执行 -cd integration +### 3.Run Tests +Execute in the tests directory: +``` +cd integration sh integrate_test.sh +``` -### 4.运行指定文件的测试 - +### 4.Run Tests for a Specific File -添加环境变量 +Add environment variables: ``` go env |grep GOBIN export PATH="$PATH:$GOBIN" ``` -执行`ginkgo --focus-file="slowlog_test.go" -vv` +Execute`ginkgo --focus-file="slowlog_test.go" -vv` + +Refer to the Ginkgo framework: https://onsi.github.io/ginkgo/#mental-model-ginkgo-assumes-specs-are-independent +Note: +`--focus-file` executes matching files + +`--skip-file` filters out non-matching files + +`--focus` executes tests matching descriptions + +`--skip` filters out tests matching descriptions + +For example, `ginkgo --focus=dog --focus=fish --skip=cat --skip=purple` -ginkgo框架参考 https://onsi.github.io/ginkgo/#mental-model-ginkgo-assumes-specs-are-independent -备注: ---focus-file执行匹配文件 ---skip-file过滤不匹配的文件 ---focus执行匹配描述的测试 ---skip过滤匹配描述的测试 -例如，`ginkgo --focus=dog --focus=fish --skip=cat --skip=purple` -则只运行运行It(描述内容中)例如"likes dogs"、"likes dog fish"的单测，而跳过"purple"相关的测试。 +This will only run tests described as "likes dogs", "likes dog fish", while skipping tests related to "purple". diff --git a/tests/integration/README_CN.md b/tests/integration/README_CN.md new file mode 100644 index 0000000000..9cfa09fdb1 --- /dev/null +++ b/tests/integration/README_CN.md @@ -0,0 +1,50 @@ +# README +这是用golang编写的pika 集成测试代码,默认提交代码到pika仓库后会自动运行。 + +## 本地跑golang集成测试 +如果你想在本地运行测试，需要完成以下的准备工作： + +### 1.准备程序和配置文件 +在../../output/pika目录确保有编译好的pika程序。 +（也可以提前编译好mac版本的pika程序,并手动将pika文件拷贝到start_master_and_slave.sh中制定的目录，将pika未改动的conf文件拷贝到test目录；或者直接修改start_master_and_slave.sh启动路径。） + +手动执行测试的前提是，已安装ginkgo,例如 +``` +cd tests/integration/ +go get github.com/onsi/ginkgo/v2/ginkgo +go install github.com/onsi/ginkgo/v2/ginkgo +go get github.com/onsi/gomega/... +``` + +### 2.启动Pika服务 +在项目主目录下执行 +``` +cd tests + +sh ./integration/start_master_and_slave.sh +``` + +### 3.运行测试 +在tests目录下执行 +cd integration +sh integrate_test.sh + +### 4.运行指定文件的测试 + + +添加环境变量 +``` +go env |grep GOBIN +export PATH="$PATH:$GOBIN" +``` + +执行`ginkgo --focus-file="slowlog_test.go" -vv` + +ginkgo框架参考: https://onsi.github.io/ginkgo/#mental-model-ginkgo-assumes-specs-are-independent +备注: +`--focus-file`执行匹配文件 +`--skip-file`过滤不匹配的文件 +`--focus`执行匹配描述的测试 +`--skip`过滤匹配描述的测试 +例如，`ginkgo --focus=dog --focus=fish --skip=cat --skip=purple` +则只运行运行It(描述内容中)例如"likes dogs"、"likes dog fish"的单测，而跳过"purple"相关的测试。