Test

esl · Dec 12, 2022 · 7ac75c6 · 7ac75c6
1 parent d9a5c46
commit 7ac75c6
Show file tree

Hide file tree

Showing 4 changed files with 67 additions and 18 deletions.
diff --git a/.circleci/template.yml b/.circleci/template.yml
@@ -560,11 +560,16 @@ jobs:
         type: boolean
         description: Erlang distribution with TLS enabled
         default: false
+      store_results:
+        type: boolean
+        description: Collect and store test results in XML for Insights
+        default: false
     environment:
       TESTSPEC: <<parameters.spec>>
       PRESET: <<parameters.preset>>
       DB: <<parameters.db>>
       TLS_DIST: <<parameters.tls_dist>>
+      STORE_RESULTS: <<parameters.store_results>>
       SKIP_AUTO_COMPILE: true
       KEEP_COVER_RUNNING: 1
     steps:
@@ -581,8 +586,22 @@ jobs:
       - run:
           name: Run Big Tests
           command: |
-            ./tools/test.sh -p $PRESET -s false
+            if $STORE_RESULTS; then
+              ./tools/test.sh -p $PRESET -h cth_surefire -s false
+            else
+              ./tools/test.sh -p $PRESET -s false
+            fi
           no_output_timeout: 40m
+      - run:
+          name: Copy test results for Insights
+          when:
+            condition: <<parameters.store_results>>
+          command: |
+            cp big_tests/ct_report/*/junit_report.xml .
+      - store_test_results:
+          when:
+            condition: <<parameters.store_results>>
+          path: junit_report.xml
       - run_coverage_analysis
       - run:
           name: Build Failed - Logs
@@ -756,6 +775,7 @@ workflows:
           requires:
             - otp_25_docker
           filters: *all_tags
+          store_results: true
       - big_tests_in_docker:
           name: mysql_redis_25
           executor: otp_25_mysql_redis
@@ -764,6 +784,7 @@ workflows:
           requires:
             - otp_25_docker
           filters: *all_tags
+          store_results: true
       - big_tests_in_docker:
           name: dynamic_domains_mysql_redis_25
           spec: dynamic_domains.spec
@@ -773,6 +794,7 @@ workflows:
           requires:
             - otp_25_docker
           filters: *all_tags
+          store_results: true
       - big_tests_in_docker:
           name: pgsql_mnesia_24
           executor: otp_24_pgsql_redis
@@ -789,6 +811,7 @@ workflows:
           requires:
             - otp_25_docker
           filters: *all_tags
+          store_results: true
       - big_tests_in_docker:
           name: dynamic_domains_pgsql_mnesia_25
           spec: dynamic_domains.spec
@@ -798,6 +821,7 @@ workflows:
           requires:
             - otp_25_docker
           filters: *all_tags
+          store_results: true
       - big_tests_in_docker:
           name: dynamic_domains_pgsql_mnesia_24
           spec: dynamic_domains.spec
@@ -816,6 +840,7 @@ workflows:
           requires:
             - otp_25_docker
           filters: *all_tags
+          store_results: true
       - big_tests_in_docker:
           name: dynamic_domains_mssql_mnesia_25
           spec: dynamic_domains.spec
@@ -826,6 +851,7 @@ workflows:
           requires:
             - otp_25_docker
           filters: *all_tags
+          store_results: true
       - big_tests_in_docker:
           name: riak_mnesia_24
           executor: otp_24_riak_redis
@@ -853,6 +879,7 @@ workflows:
           requires:
             - otp_25_docker
           filters: *all_tags
+          store_results: true
       - big_tests_in_docker:
           name: elasticsearch_and_cassandra_25
           executor: otp_25_elasticsearch_cassandra
@@ -862,6 +889,7 @@ workflows:
           requires:
             - otp_25_docker
           filters: *all_tags
+          store_results: true
       # ============= DOCKER IMAGE BUILD & UPLOAD =============
       - docker_image:
           name: docker_build_and_ship

diff --git a/big_tests/Makefile b/big_tests/Makefile
@@ -4,6 +4,7 @@ all: test
 # user overridable
 TESTSPEC ?= default.spec
 PRESET   ?= all
+CT_HOOKS ?= ""
 PREPARE  ?= prepare
 ADD_OPTS ?= -sasl sasl_error_logger false -lager handlers []
 TLS_DIST ?= false
@@ -55,7 +56,7 @@ cover_quicktest: $(PREPARE)
 
 cover_test_preset: $(PREPARE)
 	$(RUN) erl -noinput $(COMMON_OPTS) $(ADD_OPTS) \
-			-s run_common_test main test=full spec=$(TESTSPEC) \
+			-s run_common_test main test=full spec=$(TESTSPEC) hooks=$(CT_HOOKS) \
 			preset="$(PRESET)" cover=true
 
 cover_test: $(PREPARE)
@@ -64,7 +65,8 @@ cover_test: $(PREPARE)
 
 test_preset: $(PREPARE)
 	$(RUN) erl -noinput $(COMMON_OPTS) $(ADD_OPTS) \
-			-s run_common_test main test=full spec=$(TESTSPEC) preset="$(PRESET)"
+			-s run_common_test main test=full spec=$(TESTSPEC) hooks=$(CT_HOOKS) \
+                        preset="$(PRESET)"
 
 test: $(PREPARE)
 	$(RUN) erl -noinput $(COMMON_OPTS) $(ADD_OPTS) \

diff --git a/big_tests/run_common_test.erl b/big_tests/run_common_test.erl
@@ -35,7 +35,8 @@
 -record(opts, {test,
                spec,
                cover,
-               preset = all}).
+               preset = all,
+               hooks}).
 
 %% Accepted options formatted as:
 %% {opt_name, opt_index_in_opts_record, fun value_sanitizer/1}.
@@ -44,7 +45,8 @@ opts() ->
     [{test,   #opts.test,   fun quick_or_full/1},
      {spec,   #opts.spec,   fun list_to_atom/1},
      {cover,  #opts.cover,  fun bool_or_module_list/1},
-     {preset, #opts.preset, fun preset/1}].
+     {preset, #opts.preset, fun preset/1},
+     {hooks,  #opts.hooks,  fun module_list/1}].
 
 %% Raw args are 'key=val' atoms.
 %% Args are {key :: atom(), val :: string()} pairs.
@@ -84,13 +86,14 @@ init() ->
 
 run(#opts{test = quick, cover = Cover, spec = Spec}) ->
     do_run_quick_test(tests_to_run(Spec), Cover);
-run(#opts{test = full, spec = Spec, preset = Preset, cover = Cover}) ->
-    run_test(tests_to_run(Spec), case Preset of
-                                     all -> all;
-                                     undefined -> all;
-                                     _ when is_list(Preset) -> Preset;
-                                     _   -> [Preset]
-                                 end, Cover).
+run(#opts{test = full, spec = Spec, preset = Preset, cover = Cover, hooks = HookModules}) ->
+    run_test(tests_to_run(Spec) ++ ct_hooks(HookModules),
+             case Preset of
+                 all -> all;
+                 undefined -> all;
+                 _ when is_list(Preset) -> Preset;
+                 _   -> [Preset]
+             end, Cover).
 
 apply_preset_enabled(#opts{} = Opts) ->
     case os:getenv("PRESET_ENABLED") of
@@ -150,6 +153,11 @@ tests_to_run(TestSpec) ->
      {spec, TestSpecFile}
     ] ++ ct_opts().
 
+ct_hooks([]) ->
+    [];
+ct_hooks(HookModules) ->
+    [{ct_hooks, HookModules}].
+
 save_count(Test, Configs) ->
     Repeat = case proplists:get_value(repeat, Test) of
         undefined -> 1;
@@ -476,10 +484,17 @@ get_cover_header() ->
 
 bool_or_module_list("true") ->
     true;
+bool_or_module_list("false") ->
+    false;
+bool_or_module_list(undefined) ->
+    false;
 bool_or_module_list(ModuleList) when is_list(ModuleList) ->
-    [ list_to_atom(L) || L <- string:tokens("asd,qwe,zxc", ",") ];
-bool_or_module_list(_) ->
-    false.
+    module_list(ModuleList).
+
+module_list(undefined) ->
+    [];
+module_list(ModuleList) ->
+    [ list_to_atom(L) || L <- string:tokens(ModuleList, ", ") ].
 
 modules_to_analyze(true) ->
     lists:usort(cover:imported_modules() ++ cover:modules());

diff --git a/tools/test.sh b/tools/test.sh
@@ -5,6 +5,7 @@
 # - EUNIT_TESTS
 # - COVER_ENABLED
 # - STOP_NODES (default false)
+# - CT_HOOKS
 set -o pipefail
 shopt -s nullglob
 IFS=$'\n\t'
@@ -16,7 +17,7 @@ EUNIT_TESTS="${EUNIT_TESTS:-false}"
 COVER_ENABLED="${COVER_ENABLED:-true}"
 REBAR_CT_EXTRA_ARGS="${REBAR_CT_EXTRA_ARGS:-}"
 
-while getopts ":p:s:e:c:" opt; do
+while getopts ":p:s:e:c:h:" opt; do
   case $opt in
     p)
       PRESET=$OPTARG
@@ -30,6 +31,9 @@ while getopts ":p:s:e:c:" opt; do
     e)
       EUNIT_TESTS=$OPTARG
       ;;
+    h)
+      CT_HOOKS=$OPTARG
+      ;;
     \?)
       echo "Invalid option: -$OPTARG" >&2
       exit 1
@@ -145,10 +149,10 @@ run_test_preset() {
   local MAKE_RESULT=0
   TESTSPEC=${TESTSPEC:-default.spec}
   if [ "$COVER_ENABLED" = "true" ]; then
-    make cover_test_preset TESTSPEC=$TESTSPEC PRESET=$PRESET
+    make cover_test_preset TESTSPEC=$TESTSPEC PRESET=$PRESET CT_HOOKS=$CT_HOOKS
     MAKE_RESULT=$?
   else
-    make test_preset TESTSPEC=$TESTSPEC PRESET=$PRESET
+    make test_preset TESTSPEC=$TESTSPEC PRESET=$PRESET CT_HOOKS=$CT_HOOKS
     MAKE_RESULT=$?
   fi
   cd -