*: speed up test build (tikv#7012)

Tests has been built slowly lately. That's because we use multiple compile
command to build tests for different crates. Things go worse when cdc is
split as a component. On my local machine, running `make test`, cargo
will be invoked 6 times. 4 of them takes minutes to finish.

The reason why we try to run multiple compile command is to get around
the problem mentioned in tikv#6097, which is introduced when trying to
support prost and protobuf in tests at the same time. But in practice, we
don't run tests with prost. Hence overhead is introduced with just potential
benefit. In fact, after tikv#6317, it's impossible to run tests by
`PROST=1 make test` due to rust-lang/cargo#6669.  It's still possible to
build, clippy check or release with prost.

Given that tests with PROST=1 does not work, this PR combines all the cargo
commands into single one, which save a lot of compile time.

Signed-off-by: Jay Lee <[email protected]>

Makefile

@@ -215,7 +215,8 @@ docker-tag-with-git-tag:
 # Run tests under a variety of conditions. This should pass before
 # submitting pull requests. Note though that the CI system tests TiKV
 # through its own scripts and does not use this rule.
-test:
+.PHONY: run-test
+run-test:
 	# When SIP is enabled, DYLD_LIBRARY_PATH will not work in subshell, so we have to set it
 	# again here. LOCAL_DIR is defined in .travis.yml.
 	# The special linux case below is testing the mem-profiling
@@ -227,31 +228,15 @@ test:
 	export DYLD_LIBRARY_PATH="${DYLD_LIBRARY_PATH}:${LOCAL_DIR}/lib" && \
 	export LOG_LEVEL=DEBUG && \
 	export RUST_BACKTRACE=1 && \
-	cargo test --no-default-features --features "${ENABLE_FEATURES}" --all --exclude tests --exclude \
-		cdc --exclude fuzz-targets --exclude fuzzer-honggfuzz --exclude fuzzer-afl --exclude fuzzer-libfuzzer \
-		${EXTRA_CARGO_ARGS} -- --nocapture && \
-	cd tests && cargo test --features "${ENABLE_FEATURES}" ${EXTRA_CARGO_ARGS} -- --nocapture && cd .. && \
-	cargo test --no-default-features --features "${ENABLE_FEATURES}" -p tests --bench misc ${EXTRA_CARGO_ARGS} -- --nocapture && \
-	cd components/cdc && cargo test --no-default-features --features "${ENABLE_FEATURES}" -p cdc ${EXTRA_CARGO_ARGS} -- --nocapture && cd ../.. && \
-	cd components/cdc && cargo test --no-default-features --features "${ENABLE_FEATURES}" ${EXTRA_CARGO_ARGS} -- --nocapture && cd ../.. && \
+	cargo test --workspace --features "${ENABLE_FEATURES} mem-profiling" ${EXTRA_CARGO_ARGS} -- --nocapture && \
 	if [[ "`uname`" == "Linux" ]]; then \
 		export MALLOC_CONF=prof:true,prof_active:false && \
-		cargo test --no-default-features --features "${ENABLE_FEATURES},mem-profiling" ${EXTRA_CARGO_ARGS} --bin tikv-server -- --nocapture --ignored; \
+		cargo test --features "${ENABLE_FEATURES} mem-profiling" ${EXTRA_CARGO_ARGS} -p tikv_alloc -- --nocapture --ignored; \
 	fi
-	bash scripts/check-bins-for-jemalloc.sh
-	bash scripts/check-udeps.sh
-
-# This is used for CI test
-ci_test: ci_doc_test
-	cargo test --no-default-features --features "${ENABLE_FEATURES}" --all --exclude tests --exclude cdc \
-		--exclude fuzz-targets --exclude fuzzer-honggfuzz --exclude fuzzer-afl --exclude fuzzer-libfuzzer \
-		--all-targets --no-run --message-format=json
-	cd tests && cargo test --no-default-features --features "${ENABLE_FEATURES}" --no-run --message-format=json
-	cd components/cdc && cargo test --no-default-features --features "${ENABLE_FEATURES}" --no-run --message-format=json
 
-ci_doc_test:
-	cargo test --no-default-features --features "${ENABLE_FEATURES}" --all --exclude tests --exclude cdc \
-		--exclude fuzz-targets --exclude fuzzer-honggfuzz --exclude fuzzer-afl --exclude fuzzer-libfuzzer --doc
+.PHONY: test
+test: run-test
+	bash scripts/check-bins-for-jemalloc.sh
 
 ## Static analysis
 ## ---------------
@@ -308,6 +293,10 @@ pre-audit:
 audit: pre-audit
 	cargo audit
 
+.PHONY: check-udeps
+check-udeps:
+	bash scripts/check-udeps.sh
+
 FUZZER ?= Honggfuzz
 
 .PHONY: fuzz

components/sst_importer/src/sst_importer.rs

@@ -917,8 +917,6 @@ mod tests {
             .unwrap()
             .unwrap();
 
-        dbg!(&range);
-
         assert_eq!(range.get_start(), b"t123_r01");
         assert_eq!(range.get_end(), b"t123_r13");
 

components/tikv_alloc/src/jemalloc.rs

@@ -161,11 +161,11 @@ mod profiling {
 
             let os_path = dir.path().to_path_buf().join("test1.dump").into_os_string();
             let path = os_path.into_string().unwrap();
-            super::dump_prof(&path);
+            super::dump_prof(&path).unwrap();
 
             let os_path = dir.path().to_path_buf().join("test2.dump").into_os_string();
             let path = os_path.into_string().unwrap();
-            super::dump_prof(&path);
+            super::dump_prof(&path).unwrap();
 
             let files = fs::read_dir(dir.path()).unwrap().count();
             assert_eq!(files, 2);

tests/Cargo.toml

@@ -32,6 +32,7 @@ path = "benches/hierarchy/mod.rs"
 [[bench]]
 name = "misc"
 path = "benches/misc/mod.rs"
+test = true
 
 [[bench]]
 name = "deadlock_detector"
@@ -41,9 +42,10 @@ path = "benches/deadlock_detector/mod.rs"
 [[bench]]
 name = "channel"
 path = "benches/channel/mod.rs"
+test = true
 
 [features]
-default = ["failpoints", "testexport"]
+default = ["failpoints", "testexport", "protobuf-codec"]
 failpoints = ["fail/failpoints", "tikv/failpoints"]
 testexport = ["raftstore/testexport", "tikv/testexport"]
 profiling = ["profiler/profiling"]