diff --git a/.circleci/template.yml b/.circleci/template.yml index 6a91c59d36..71a6d69141 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -581,8 +581,14 @@ jobs: - run: name: Run Big Tests command: | - ./tools/test.sh -p $PRESET -s false + ./tools/test.sh -p $PRESET -h ct_surefire -s false no_output_timeout: 40m + - run: + name: Copy test results for Insights + command: | + cp big_tests/ct_report/*/junit_report.xml . + - store_test_results: + path: junit_report.xml - run_coverage_analysis - run: name: Build Failed - Logs diff --git a/big_tests/Makefile b/big_tests/Makefile index 59fbaa1c45..44260ffedc 100644 --- a/big_tests/Makefile +++ b/big_tests/Makefile @@ -4,6 +4,7 @@ all: test # user overridable TESTSPEC ?= default.spec PRESET ?= all +CT_HOOKS ?= "" PREPARE ?= prepare ADD_OPTS ?= -sasl sasl_error_logger false -lager handlers [] TLS_DIST ?= false @@ -55,7 +56,7 @@ cover_quicktest: $(PREPARE) cover_test_preset: $(PREPARE) $(RUN) erl -noinput $(COMMON_OPTS) $(ADD_OPTS) \ - -s run_common_test main test=full spec=$(TESTSPEC) \ + -s run_common_test main test=full spec=$(TESTSPEC) hooks=$(CT_HOOKS) preset="$(PRESET)" cover=true cover_test: $(PREPARE) @@ -64,7 +65,8 @@ cover_test: $(PREPARE) test_preset: $(PREPARE) $(RUN) erl -noinput $(COMMON_OPTS) $(ADD_OPTS) \ - -s run_common_test main test=full spec=$(TESTSPEC) preset="$(PRESET)" + -s run_common_test main test=full spec=$(TESTSPEC) hooks=$(CT_HOOKS) + preset="$(PRESET)" test: $(PREPARE) $(RUN) erl -noinput $(COMMON_OPTS) $(ADD_OPTS) \ diff --git a/big_tests/run_common_test.erl b/big_tests/run_common_test.erl index b3b6dc6d91..b5a5123495 100644 --- a/big_tests/run_common_test.erl +++ b/big_tests/run_common_test.erl @@ -35,7 +35,8 @@ -record(opts, {test, spec, cover, - preset = all}). + preset = all, + hooks}). %% Accepted options formatted as: %% {opt_name, opt_index_in_opts_record, fun value_sanitizer/1}. @@ -44,7 +45,8 @@ opts() -> [{test, #opts.test, fun quick_or_full/1}, {spec, #opts.spec, fun list_to_atom/1}, {cover, #opts.cover, fun bool_or_module_list/1}, - {preset, #opts.preset, fun preset/1}]. + {preset, #opts.preset, fun preset/1}, + {hooks, #opts.hooks, fun module_list/1}]. %% Raw args are 'key=val' atoms. %% Args are {key :: atom(), val :: string()} pairs. @@ -84,13 +86,14 @@ init() -> run(#opts{test = quick, cover = Cover, spec = Spec}) -> do_run_quick_test(tests_to_run(Spec), Cover); -run(#opts{test = full, spec = Spec, preset = Preset, cover = Cover}) -> - run_test(tests_to_run(Spec), case Preset of - all -> all; - undefined -> all; - _ when is_list(Preset) -> Preset; - _ -> [Preset] - end, Cover). +run(#opts{test = full, spec = Spec, preset = Preset, cover = Cover, hooks = HookModules}) -> + run_test(tests_to_run(Spec) ++ ct_hooks(HookModules), + case Preset of + all -> all; + undefined -> all; + _ when is_list(Preset) -> Preset; + _ -> [Preset] + end, Cover). apply_preset_enabled(#opts{} = Opts) -> case os:getenv("PRESET_ENABLED") of @@ -150,6 +153,11 @@ tests_to_run(TestSpec) -> {spec, TestSpecFile} ] ++ ct_opts(). +ct_hooks([]) -> + []; +ct_hooks(HookModules) -> + [{ct_hooks, HookModules}]. + save_count(Test, Configs) -> Repeat = case proplists:get_value(repeat, Test) of undefined -> 1; @@ -476,11 +484,18 @@ get_cover_header() -> bool_or_module_list("true") -> true; +bool_or_module_list("false") -> + false; bool_or_module_list(ModuleList) when is_list(ModuleList) -> - [ list_to_atom(L) || L <- string:tokens("asd,qwe,zxc", ",") ]; -bool_or_module_list(_) -> + module_list(ModuleList); +bool_or_module_list(undefined) -> false. +module_list(undefined) -> + []; +module_list(ModuleList) -> + [ list_to_atom(L) || L <- string:tokens(ModuleList, ", ") ]. + modules_to_analyze(true) -> lists:usort(cover:imported_modules() ++ cover:modules()); modules_to_analyze(ModuleList) when is_list(ModuleList) -> diff --git a/tools/test.sh b/tools/test.sh index 0448387d44..88477472bf 100755 --- a/tools/test.sh +++ b/tools/test.sh @@ -5,6 +5,7 @@ # - EUNIT_TESTS # - COVER_ENABLED # - STOP_NODES (default false) +# - CT_HOOKS set -o pipefail shopt -s nullglob IFS=$'\n\t' @@ -30,6 +31,9 @@ while getopts ":p:s:e:c:" opt; do e) EUNIT_TESTS=$OPTARG ;; + h) + CT_HOOKS=$OPTARG + ;; \?) echo "Invalid option: -$OPTARG" >&2 exit 1 @@ -145,10 +149,10 @@ run_test_preset() { local MAKE_RESULT=0 TESTSPEC=${TESTSPEC:-default.spec} if [ "$COVER_ENABLED" = "true" ]; then - make cover_test_preset TESTSPEC=$TESTSPEC PRESET=$PRESET + make cover_test_preset TESTSPEC=$TESTSPEC PRESET=$PRESET CT_HOOKS=$CT_HOOKS MAKE_RESULT=$? else - make test_preset TESTSPEC=$TESTSPEC PRESET=$PRESET + make test_preset TESTSPEC=$TESTSPEC PRESET=$PRESET CT_HOOKS=$CT_HOOKS MAKE_RESULT=$? fi cd -