Introduce proxy_server and new-mock-engine-store (#120)

Introduce crate proxy_server for a decouple of component/server, it will be used to start a proxy service. Introduce crate new-mock-engine-store as a new mock-engine-store plus a test_raftstore implementation for proxy. This will enable us to run mix-mode tests. Tests in tests/failpoints and tests/integrations are obsolete tests and will be removed once the refactor ends. New tests should be added in tests/proxy with support of new-mock-engine-store. Introduce crate engine_tiflash as a new KvEngine. It currently just works like former RocksEngine, unless we enable compat_new_proxy. Since some PRs have not been merged in TiKV, we can only enable this feature in some tests in tests/proxy. It can not be enabled in formal codes, since it can filter some writes. cherry pick newly merge TiKV PRs, aka on_empty_cmd. Fix some bugs. * f Signed-off-by: CalvinNeo <[email protected]> * fix Signed-off-by: CalvinNeo <[email protected]> * add engine_tiflash with this version of engine_rocks Signed-off-by: CalvinNeo <[email protected]> * fmt Signed-off-by: CalvinNeo <[email protected]> * add engine_tiflash actually Signed-off-by: CalvinNeo <[email protected]> * fix Signed-off-by: CalvinNeo <[email protected]> * fix some tests Signed-off-by: CalvinNeo <[email protected]> * f Signed-off-by: CalvinNeo <[email protected]> * it runs if we do_write everytime Signed-off-by: CalvinNeo <[email protected]> * add ensure_no_common_unrecognized_keys to config checker Signed-off-by: CalvinNeo <[email protected]> * reorg tests Signed-off-by: CalvinNeo <[email protected]> * raftstore: Implement coprocessor observer on_empty_cmd (tikv#12851) ref tikv#12849 Support new observers on_empty_cmd. Signed-off-by: CalvinNeo <[email protected]> Co-authored-by: Ti Chi Robot <[email protected]> * fix tests Signed-off-by: CalvinNeo <[email protected]> * move proxy-related code in component/server into proxy_server Signed-off-by: CalvinNeo <[email protected]> * move codes in component/server/src/setup.rs into proxy_server Signed-off-by: CalvinNeo <[email protected]> * support new ffis in mock-engine-store and new-mock-engine-store Signed-off-by: CalvinNeo <[email protected]> * add disclaimer Signed-off-by: CalvinNeo <[email protected]> * enlength some test time Signed-off-by: CalvinNeo <[email protected]> * Remove pub fields of PdCluster, add feature to origin import Signed-off-by: CalvinNeo <[email protected]> * f Signed-off-by: CalvinNeo <[email protected]> * add Signed-off-by: CalvinNeo <[email protected]> Co-authored-by: Ti Chi Robot <[email protected]>
pingcap · Jul 14, 2022 · b33b1eb · b33b1eb
1 parent 42ede65
commit b33b1eb
Show file tree

Hide file tree

Showing 81 changed files with 15,171 additions and 354 deletions.
diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml
@@ -55,6 +55,7 @@ jobs:
           # export RUSTC_WRAPPER=~/.cargo/bin/sccache
           # make test
           # make debug
+          export ENGINE_LABEL_VALUE=tiflash
           export RUST_BACKTRACE=full
           cargo check
           cargo test --package tests --test failpoints cases::test_normal
@@ -71,4 +72,5 @@ jobs:
           cargo test --package tests --test failpoints cases::test_merge
           cargo test --package tests --test failpoints cases::test_import_service
           cargo test --package tests --test failpoints cases::test_proxy_replica_read
-          cargo test --package tests --test failpoints cases::test_proxy
+          cargo test --features compat_new_proxy --package tests --test proxy normal
+          cargo test --package tests --test proxy proxy
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -58,6 +58,7 @@ cloud-storage-grpc = ["sst_importer/cloud-storage-grpc"]
 cloud-storage-dylib = ["sst_importer/cloud-storage-dylib"]
 pprof-dwarf = ["pprof/backtrace-rs"]
 pprof-fp = ["pprof/frame-pointer"]
+compat_new_proxy = ["engine_tiflash/compat_new_proxy"]
 
 # for testing configure propegate to other crates
 # https://stackoverflow.com/questions/41700543/can-we-share-test-utilites-between-crates
@@ -88,6 +89,7 @@ encryption_export = { path = "components/encryption/export", default-features =
 engine_panic = { path = "components/engine_panic", default-features = false }
 engine_rocks = { path = "components/engine_rocks", default-features = false }
 engine_test = { path = "components/engine_test", default-features = false }
+engine_tiflash = { path = "engine_tiflash", default-features = false }
 engine_traits = { path = "components/engine_traits", default-features = false }
 engine_traits_tests = { path = "components/engine_traits_tests", default-features = false }
 error_code = { path = "components/error_code", default-features = false }

diff --git a/components/engine_rocks/src/lib.rs b/components/engine_rocks/src/lib.rs
@@ -64,7 +64,7 @@ mod perf_context_metrics;
 mod engine_iterator;
 pub use crate::engine_iterator::*;
 
-mod options;
+pub mod options;
 pub mod raw_util;
 pub mod util;
 

diff --git a/components/proxy_server/Cargo.toml b/components/proxy_server/Cargo.toml
@@ -0,0 +1,96 @@
+[package]
+name = "proxy_server"
+version = "0.0.1"
+license = "Apache-2.0"
+edition = "2018"
+publish = false
+
+[features]
+tcmalloc = ["tikv/tcmalloc"]
+jemalloc = ["tikv/jemalloc"]
+mimalloc = ["tikv/mimalloc"]
+snmalloc = ["tikv/snmalloc"]
+portable = ["tikv/portable"]
+sse = ["tikv/sse"]
+mem-profiling = ["tikv/mem-profiling"]
+failpoints = ["tikv/failpoints"]
+bcc-iosnoop = ["tikv/bcc-iosnoop"]
+
+cloud-aws = ["encryption_export/cloud-aws"]
+cloud-gcp = ["encryption_export/cloud-gcp"]
+cloud-azure = ["encryption_export/cloud-azure"]
+test-engine-kv-rocksdb = [
+  "tikv/test-engine-kv-rocksdb"
+]
+test-engine-raft-raft-engine = [
+  "tikv/test-engine-raft-raft-engine"
+]
+test-engines-rocksdb = [
+  "tikv/test-engines-rocksdb",
+]
+test-engines-panic = [
+  "tikv/test-engines-panic",
+]
+nortcheck = ["engine_rocks/nortcheck"]
+backup-stream-debug = ["backup-stream/backup-stream-debug"]
+
+[dependencies]
+api_version = { path = "../api_version" }
+backup = { path = "../backup", default-features = false }
+backup-stream = { path = "../backup-stream", default-features = false }
+causal_ts = { path = "../causal_ts" }
+cdc = { path = "../cdc", default-features = false }
+chrono = "0.4"
+clap = "2.32"
+collections = { path = "../collections" }
+concurrency_manager = { path = "../concurrency_manager", default-features = false }
+crossbeam = "0.8"
+encryption = { path = "../encryption", default-features = false }
+encryption_export = { path = "../encryption/export", default-features = false }
+engine_rocks = { path = "../engine_rocks", default-features = false }
+engine_rocks_helper = { path = "../engine_rocks_helper" }
+engine_traits = { path = "../engine_traits", default-features = false }
+error_code = { path = "../error_code", default-features = false }
+file_system = { path = "../file_system", default-features = false }
+fs2 = "0.4"
+futures = "0.3"
+grpcio = { version = "0.10", default-features = false, features = ["openssl-vendored"] }
+grpcio-health = { version = "0.10", default-features = false, features = ["protobuf-codec"] }
+hex = "0.4"
+
+itertools = "0.10"
+keys = { path = "../keys", default-features = false }
+kvproto = { git = "https://github.com/pingcap/kvproto.git" }
+libc = "0.2"
+log = { version = "0.4", features = ["max_level_trace", "release_max_level_debug"] }
+log_wrappers = { path = "../log_wrappers" }
+nix = "0.23"
+online_config = { path = "../online_config" }
+pd_client = { path = "../pd_client", default-features = false }
+prometheus = { version = "0.13", features = ["nightly"] }
+protobuf = { version = "2.8", features = ["bytes"] }
+raft = { version = "0.7.0", default-features = false, features = ["protobuf-codec"] }
+raft_log_engine = { path = "../raft_log_engine", default-features = false }
+raftstore = { path = "../raftstore", default-features = false }
+rand = "0.8"
+resolved_ts = { path = "../../components/resolved_ts", default-features = false }
+resource_metering = { path = "../resource_metering" }
+security = { path = "../security", default-features = false }
+serde = "1.0"
+serde_derive = "1.0"
+serde_ignored = "0.1"
+serde_json = "1.0"
+server = { path = "../server", default-features = false }
+slog = { version = "2.3", features = ["max_level_trace", "release_max_level_debug"] }
+slog-global = { version = "0.1", git = "https://github.com/breeswish/slog-global.git", rev = "d592f88e4dbba5eb439998463054f1a44fbf17b9" }
+tempfile = "3.0"
+tikv = { path = "../..", default-features = false }
+tikv_alloc = { path = "../tikv_alloc" }
+tikv_util = { path = "../tikv_util", default-features = false }
+tokio = { version = "1.5", features = ["rt-multi-thread"] }
+toml = "0.5"
+txn_types = { path = "../txn_types", default-features = false }
+yatp = { git = "https://github.com/tikv/yatp.git", branch = "master" }
+
+[target.'cfg(unix)'.dependencies]
+signal = "0.6"