diff --git a/.circleci/template.yml b/.circleci/template.yml index 6a91c59d36..bb37bd2eba 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -560,11 +560,16 @@ jobs: type: boolean description: Erlang distribution with TLS enabled default: false + store_results: + type: boolean + description: Collect and store test results in XML for Insights + default: false environment: TESTSPEC: <> PRESET: <> DB: <> TLS_DIST: <> + STORE_RESULTS: <> SKIP_AUTO_COMPILE: true KEEP_COVER_RUNNING: 1 steps: @@ -581,8 +586,22 @@ jobs: - run: name: Run Big Tests command: | - ./tools/test.sh -p $PRESET -s false + if $STORE_RESULTS; then + ./tools/test.sh -p $PRESET -h cth_surefire -s false + else + ./tools/test.sh -p $PRESET -s false + fi no_output_timeout: 40m + - run: + name: Copy test results for Insights + when: + condition: <> + command: | + cp big_tests/ct_report/*/junit_report.xml . + - store_test_results: + when: + condition: <> + path: junit_report.xml - run_coverage_analysis - run: name: Build Failed - Logs @@ -756,6 +775,7 @@ workflows: requires: - otp_25_docker filters: *all_tags + store_results: true - big_tests_in_docker: name: mysql_redis_25 executor: otp_25_mysql_redis @@ -764,6 +784,7 @@ workflows: requires: - otp_25_docker filters: *all_tags + store_results: true - big_tests_in_docker: name: dynamic_domains_mysql_redis_25 spec: dynamic_domains.spec @@ -773,6 +794,7 @@ workflows: requires: - otp_25_docker filters: *all_tags + store_results: true - big_tests_in_docker: name: pgsql_mnesia_24 executor: otp_24_pgsql_redis @@ -789,6 +811,7 @@ workflows: requires: - otp_25_docker filters: *all_tags + store_results: true - big_tests_in_docker: name: dynamic_domains_pgsql_mnesia_25 spec: dynamic_domains.spec @@ -798,6 +821,7 @@ workflows: requires: - otp_25_docker filters: *all_tags + store_results: true - big_tests_in_docker: name: dynamic_domains_pgsql_mnesia_24 spec: dynamic_domains.spec @@ -816,6 +840,7 @@ workflows: requires: - otp_25_docker filters: *all_tags + store_results: true - big_tests_in_docker: name: dynamic_domains_mssql_mnesia_25 spec: dynamic_domains.spec @@ -826,6 +851,7 @@ workflows: requires: - otp_25_docker filters: *all_tags + store_results: true - big_tests_in_docker: name: riak_mnesia_24 executor: otp_24_riak_redis @@ -853,6 +879,7 @@ workflows: requires: - otp_25_docker filters: *all_tags + store_results: true - big_tests_in_docker: name: elasticsearch_and_cassandra_25 executor: otp_25_elasticsearch_cassandra @@ -862,6 +889,7 @@ workflows: requires: - otp_25_docker filters: *all_tags + store_results: true # ============= DOCKER IMAGE BUILD & UPLOAD ============= - docker_image: name: docker_build_and_ship diff --git a/big_tests/Makefile b/big_tests/Makefile index 59fbaa1c45..b63c1be922 100644 --- a/big_tests/Makefile +++ b/big_tests/Makefile @@ -4,6 +4,7 @@ all: test # user overridable TESTSPEC ?= default.spec PRESET ?= all +CT_HOOKS ?= "" PREPARE ?= prepare ADD_OPTS ?= -sasl sasl_error_logger false -lager handlers [] TLS_DIST ?= false @@ -55,7 +56,7 @@ cover_quicktest: $(PREPARE) cover_test_preset: $(PREPARE) $(RUN) erl -noinput $(COMMON_OPTS) $(ADD_OPTS) \ - -s run_common_test main test=full spec=$(TESTSPEC) \ + -s run_common_test main test=full spec=$(TESTSPEC) hooks=$(CT_HOOKS) \ preset="$(PRESET)" cover=true cover_test: $(PREPARE) @@ -64,7 +65,8 @@ cover_test: $(PREPARE) test_preset: $(PREPARE) $(RUN) erl -noinput $(COMMON_OPTS) $(ADD_OPTS) \ - -s run_common_test main test=full spec=$(TESTSPEC) preset="$(PRESET)" + -s run_common_test main test=full spec=$(TESTSPEC) hooks=$(CT_HOOKS) \ + preset="$(PRESET)" test: $(PREPARE) $(RUN) erl -noinput $(COMMON_OPTS) $(ADD_OPTS) \ diff --git a/big_tests/run_common_test.erl b/big_tests/run_common_test.erl index b3b6dc6d91..a10b63ae7f 100644 --- a/big_tests/run_common_test.erl +++ b/big_tests/run_common_test.erl @@ -35,7 +35,8 @@ -record(opts, {test, spec, cover, - preset = all}). + preset = all, + hooks}). %% Accepted options formatted as: %% {opt_name, opt_index_in_opts_record, fun value_sanitizer/1}. @@ -44,7 +45,8 @@ opts() -> [{test, #opts.test, fun quick_or_full/1}, {spec, #opts.spec, fun list_to_atom/1}, {cover, #opts.cover, fun bool_or_module_list/1}, - {preset, #opts.preset, fun preset/1}]. + {preset, #opts.preset, fun preset/1}, + {hooks, #opts.hooks, fun module_list/1}]. %% Raw args are 'key=val' atoms. %% Args are {key :: atom(), val :: string()} pairs. @@ -84,13 +86,14 @@ init() -> run(#opts{test = quick, cover = Cover, spec = Spec}) -> do_run_quick_test(tests_to_run(Spec), Cover); -run(#opts{test = full, spec = Spec, preset = Preset, cover = Cover}) -> - run_test(tests_to_run(Spec), case Preset of - all -> all; - undefined -> all; - _ when is_list(Preset) -> Preset; - _ -> [Preset] - end, Cover). +run(#opts{test = full, spec = Spec, preset = Preset, cover = Cover, hooks = HookModules}) -> + run_test(tests_to_run(Spec) ++ ct_hooks(HookModules), + case Preset of + all -> all; + undefined -> all; + _ when is_list(Preset) -> Preset; + _ -> [Preset] + end, Cover). apply_preset_enabled(#opts{} = Opts) -> case os:getenv("PRESET_ENABLED") of @@ -150,6 +153,11 @@ tests_to_run(TestSpec) -> {spec, TestSpecFile} ] ++ ct_opts(). +ct_hooks([]) -> + []; +ct_hooks(HookModules) -> + [{ct_hooks, HookModules}]. + save_count(Test, Configs) -> Repeat = case proplists:get_value(repeat, Test) of undefined -> 1; @@ -476,10 +484,17 @@ get_cover_header() -> bool_or_module_list("true") -> true; +bool_or_module_list("false") -> + false; +bool_or_module_list(undefined) -> + false; bool_or_module_list(ModuleList) when is_list(ModuleList) -> - [ list_to_atom(L) || L <- string:tokens("asd,qwe,zxc", ",") ]; -bool_or_module_list(_) -> - false. + module_list(ModuleList). + +module_list(undefined) -> + []; +module_list(ModuleList) -> + [ list_to_atom(L) || L <- string:tokens(ModuleList, ", ") ]. modules_to_analyze(true) -> lists:usort(cover:imported_modules() ++ cover:modules()); diff --git a/tools/test.sh b/tools/test.sh index 0448387d44..70652b2b79 100755 --- a/tools/test.sh +++ b/tools/test.sh @@ -5,6 +5,7 @@ # - EUNIT_TESTS # - COVER_ENABLED # - STOP_NODES (default false) +# - CT_HOOKS set -o pipefail shopt -s nullglob IFS=$'\n\t' @@ -16,7 +17,7 @@ EUNIT_TESTS="${EUNIT_TESTS:-false}" COVER_ENABLED="${COVER_ENABLED:-true}" REBAR_CT_EXTRA_ARGS="${REBAR_CT_EXTRA_ARGS:-}" -while getopts ":p:s:e:c:" opt; do +while getopts ":p:s:e:c:h:" opt; do case $opt in p) PRESET=$OPTARG @@ -30,6 +31,9 @@ while getopts ":p:s:e:c:" opt; do e) EUNIT_TESTS=$OPTARG ;; + h) + CT_HOOKS=$OPTARG + ;; \?) echo "Invalid option: -$OPTARG" >&2 exit 1 @@ -145,10 +149,10 @@ run_test_preset() { local MAKE_RESULT=0 TESTSPEC=${TESTSPEC:-default.spec} if [ "$COVER_ENABLED" = "true" ]; then - make cover_test_preset TESTSPEC=$TESTSPEC PRESET=$PRESET + make cover_test_preset TESTSPEC=$TESTSPEC PRESET=$PRESET CT_HOOKS=$CT_HOOKS MAKE_RESULT=$? else - make test_preset TESTSPEC=$TESTSPEC PRESET=$PRESET + make test_preset TESTSPEC=$TESTSPEC PRESET=$PRESET CT_HOOKS=$CT_HOOKS MAKE_RESULT=$? fi cd -