Merge branch 'master' into c2s/merge_master

Makefile

@@ -16,7 +16,6 @@ clean:
 	-rm -rf asngen
 	-rm configure.out
 	-rm rel/configure.vars.config
-	-rm rel/vars-toml.config
 
 # REBAR_CT_EXTRA_ARGS comes from a test runner
 ct:
@@ -27,7 +26,7 @@ ct:
 eunit:
 	@$(RUN) $(REBAR) eunit
 
-rel: certs configure.out rel/vars-toml.config
+rel: certs configure.out rel/configure.vars.config
 	. ./configure.out && $(REBAR) as prod release
 
 shell: certs etc/mongooseim.cfg
@@ -40,9 +39,6 @@ rock:
 	elif [ "$(BRANCH)" ]; then tools/rock_changed.sh $(BRANCH); \
 	else tools/rock_changed.sh; fi
 
-rel/vars-toml.config: rel/vars-toml.config.in rel/configure.vars.config
-	cat $^ > $@
-
 ## Don't allow these files to go out of sync!
 configure.out rel/configure.vars.config:
 	./tools/configure with-all without-jingle-sip

big_tests/default.spec

@@ -46,6 +46,7 @@
 {suites, "tests", graphql_mnesia_SUITE}.
 {suites, "tests", graphql_vcard_SUITE}.
 {suites, "tests", graphql_http_upload_SUITE}.
+{suites, "tests", graphql_server_SUITE}.
 {suites, "tests", graphql_metric_SUITE}.
 {suites, "tests", inbox_SUITE}.
 {suites, "tests", inbox_extensions_SUITE}.
@@ -103,7 +104,6 @@
 % {suites, "tests", sic_SUITE}.
 {suites, "tests", smart_markers_SUITE}.
 % {suites, "tests", sm_SUITE}.
-{suites, "tests", users_api_SUITE}.
 {suites, "tests", vcard_SUITE}.
 {suites, "tests", vcard_simple_SUITE}.
 {suites, "tests", websockets_SUITE}.

big_tests/dynamic_domains.config

@@ -27,7 +27,7 @@
                  {hidden_service_port, 8189},
                  {gd_endpoint_port, 5555},
                  {http_notifications_port, 8000}]},
-         {mim2, [{node, ejabberd2@localhost},
+         {mim2, [{node, mongooseim2@localhost},
                  {domain, <<"domain.example.com">>},
                  {host_type, <<"test type">>},
                  {dynamic_domains, [{<<"test type">>, [<<"domain.example.com">>]}]},

big_tests/dynamic_domains.spec

@@ -63,6 +63,7 @@
 {suites, "tests", graphql_token_SUITE}.
 {suites, "tests", graphql_mnesia_SUITE}.
 {suites, "tests", graphql_http_upload_SUITE}.
+{suites, "tests", graphql_server_SUITE}.
 {suites, "tests", graphql_metric_SUITE}.
 
 {suites, "tests", inbox_SUITE}.
@@ -150,7 +151,6 @@
 
 {suites, "tests", smart_markers_SUITE}.
 % {suites, "tests", sm_SUITE}.
-{suites, "tests", users_api_SUITE}.
 {suites, "tests", vcard_SUITE}.
 {suites, "tests", vcard_simple_SUITE}.
 {suites, "tests", websockets_SUITE}.

big_tests/run_common_test.erl

@@ -278,22 +278,20 @@ is_test_host_enabled(HostName) ->
 enable_preset_on_node(Node, PresetVars, HostVarsFilePrefix) ->
     {ok, Cwd} = call(Node, file, get_cwd, []),
     TemplatePath = filename:join([repo_dir(), "rel", "files", "mongooseim.toml"]),
-    DefaultVarsPath = filename:join([repo_dir(), "rel", "vars-toml.config"]),
     NodeVarsPath = filename:join([repo_dir(), "rel", HostVarsFilePrefix ++ ".vars-toml.config"]),
 
     {ok, Template} = handle_file_error(TemplatePath, file:read_file(TemplatePath)),
-    {ok, DefaultVars} = handle_file_error(DefaultVarsPath, file:consult(DefaultVarsPath)),
-    {ok, NodeVars} = handle_file_error(NodeVarsPath, file:consult(NodeVarsPath)),
+    NodeVars = read_vars(NodeVarsPath),
 
-    TemplatedConfig = template_config(Template, [DefaultVars, NodeVars, PresetVars]),
+    TemplatedConfig = template_config(Template, NodeVars ++ PresetVars),
     CfgPath = filename:join([Cwd, "etc", "mongooseim.toml"]),
     ok = call(Node, file, write_file, [CfgPath, TemplatedConfig]),
     call(Node, application, stop, [mongooseim]),
     call(Node, application, start, [mongooseim]),
     ok.
 
-template_config(Template, Vars) ->
-    MergedVars = ensure_binary_strings(merge_vars(Vars)),
+template_config(Template, RawVars) ->
+    MergedVars = ensure_binary_strings(maps:from_list(RawVars)),
     %% Render twice to replace variables in variables
     Tmp = bbmustache:render(Template, MergedVars, [{key_type, atom}]),
     bbmustache:render(Tmp, MergedVars, [{key_type, atom}]).
@@ -305,11 +303,20 @@ merge_vars([Vars1, Vars2|Rest]) ->
     merge_vars([Vars|Rest]);
 merge_vars([Vars]) -> Vars.
 
+read_vars(File) ->
+    {ok, Terms} = handle_file_error(File, file:consult(File)),
+    lists:flatmap(fun({Key, Val}) ->
+                          [{Key, Val}];
+                     (IncludedFile) when is_list(IncludedFile) ->
+                          Path = filename:join(filename:dirname(File), IncludedFile),
+                          read_vars(Path)
+                  end, Terms).
+
 %% bbmustache tries to iterate over lists, so we need to make them binaries
 ensure_binary_strings(Vars) ->
-    lists:map(fun({dbs, V}) -> {dbs, V};
-                 ({K, V}) when is_list(V) -> {K, list_to_binary(V)};
-                 ({K, V}) -> {K, V}
+    maps:map(fun(dbs, V) -> V;
+                (_K, V) when is_list(V) -> list_to_binary(V);
+                (_K, V) -> V
               end, Vars).
 
 call(Node, M, F, A) ->

big_tests/test.config

@@ -7,7 +7,7 @@
 %%       See s2s_SUITE for example on using `hosts` to RPC into nodes (uses CT "require").
 %% the Erlang node name of tested ejabberd/MongooseIM
 {ejabberd_node, 'mongooseim@localhost'}.
-{ejabberd2_node, 'ejabberd2@localhost'}.
+{ejabberd2_node, 'mongooseim2@localhost'}.
 {ejabberd_cookie, ejabberd}.
 {ejabberd_string_format, bin}.
 
@@ -30,7 +30,6 @@
                  {muc_light_service_pattern, <<"muclight.@HOST@">>},
                  {s2s_port, 5269},
                  {incoming_s2s_port, 5269},
-                 {metrics_rest_port, 5288},
                  {c2s_port, 5222},
                  {c2s_tls_port, 5223},
                  {cowboy_port, 5280},
@@ -41,13 +40,12 @@
                  {hidden_service_port, 8189},
                  {gd_endpoint_port, 5555},
                  {http_notifications_port, 8000}]},
-         {mim2, [{node, ejabberd2@localhost},
+         {mim2, [{node, mongooseim2@localhost},
                  {domain, <<"localhost">>},
                  {host_type, <<"localhost">>},
                  {vars, "mim2"},
                  {cluster, mim},
                  {c2s_tls_port, 5233},
-                 {metrics_rest_port, 5289},
                  {gd_endpoint_port, 6666},
                  {service_port, 8899}]},
          {mim3, [{node, mongooseim3@localhost},

big_tests/tests/cluster_commands_SUITE.erl

@@ -332,7 +332,7 @@ leave_using_rpc(Config) ->
     add_node_to_cluster(Node2, Config),
     %% when
     Result = distributed_helper:rpc(Node1#{timeout => timer:seconds(30)},
-                                    ejabberd_admin, leave_cluster, []),
+                                    mongoose_server_api, leave_cluster, []),
     ct:pal("leave_using_rpc result ~p~n", [Result]),
     %% then
     distributed_helper:verify_result(Node2, remove),
@@ -395,7 +395,7 @@ remove_dead_from_cluster(Config) ->
     ok = rpc(Node2#{timeout => Timeout}, mongoose_cluster, join, [Node1Nodename]),
     ok = rpc(Node3#{timeout => Timeout}, mongoose_cluster, join, [Node1Nodename]),
     %% when
-    distributed_helper:stop_node(Node3, Config),
+    distributed_helper:stop_node(Node3Nodename, Config),
     {_, OpCode1} = mongooseimctl_interactive(Node1, "remove_from_cluster",
                                              [atom_to_list(Node3Nodename)], "yes\n", Config),
     %% then
@@ -405,7 +405,7 @@ remove_dead_from_cluster(Config) ->
     have_node_in_mnesia(Node1, Node3, false),
     have_node_in_mnesia(Node2, Node3, false),
     % after node awakening nodes are clustered again
-    distributed_helper:start_node(Node3, Config),
+    distributed_helper:start_node(Node3Nodename, Config),
     have_node_in_mnesia(Node1, Node3, true),
     have_node_in_mnesia(Node2, Node3, true).
 

big_tests/tests/distributed_helper.erl

@@ -168,8 +168,6 @@ get_or_fail(Key) ->
 
 start_node(Node, Config) ->
     {_, 0} = mongooseimctl_helper:mongooseimctl(Node, "start", [], Config),
-    {_, 0} = mongooseimctl_helper:mongooseimctl(Node, "started", [], Config),
-    %% TODO Looks like "started" run by mongooseimctl fun is not really synchronous
     timer:sleep(3000).
 
 stop_node(Node, Config) ->

big_tests/tests/domain_helper.erl

@@ -1,22 +1,6 @@
 -module(domain_helper).
 
--export([insert_configured_domains/0,
-         delete_configured_domains/0,
-         insert_domain/3,
-         delete_domain/2,
-         set_domain_password/3,
-         delete_domain_password/2,
-         make_metrics_prefix/1,
-         host_types/0,
-         host_types/1,
-         host_type/0,
-         host_type/1,
-         domain_to_host_type/2,
-         domain/0,
-         secondary_domain/0,
-         domain/1,
-         secondary_host_type/0,
-         secondary_host_type/1]).
+-compile([export_all, nowarn_export_all]).
 
 -import(distributed_helper, [get_or_fail/1, rpc/4, mim/0]).
 

big_tests/tests/domain_removal_SUITE.erl

@@ -17,6 +17,7 @@ all() ->
      {group, auth_removal},
      {group, cache_removal},
      {group, mam_removal},
+     {group, mam_removal_incremental},
      {group, inbox_removal},
      {group, muc_light_removal},
      {group, muc_removal},
@@ -25,7 +26,8 @@ all() ->
      {group, offline_removal},
      {group, markers_removal},
      {group, vcard_removal},
-     {group, last_removal}
+     {group, last_removal},
+     {group, removal_failures}
     ].
 
 groups() ->
@@ -34,6 +36,8 @@ groups() ->
      {cache_removal, [], [cache_removal]},
      {mam_removal, [], [mam_pm_removal,
                         mam_muc_removal]},
+     {mam_removal_incremental, [], [mam_pm_removal,
+                                    mam_muc_removal]},
      {inbox_removal, [], [inbox_removal]},
      {muc_light_removal, [], [muc_light_removal,
                               muc_light_blocking_removal]},
@@ -43,7 +47,8 @@ groups() ->
      {offline_removal, [], [offline_removal]},
      {markers_removal, [], [markers_removal]},
      {vcard_removal, [], [vcard_removal]},
-     {last_removal, [], [last_removal]}
+     {last_removal, [], [last_removal]},
+     {removal_failures, [], [removal_stops_if_handler_fails]}
     ].
 
 %%%===================================================================
@@ -80,6 +85,8 @@ end_per_group(_Groupname, Config) ->
     end,
     ok.
 
+group_to_modules(removal_failures) ->
+    group_to_modules(mam_removal);
 group_to_modules(auth_removal) ->
     [];
 group_to_modules(cache_removal) ->
@@ -89,6 +96,10 @@ group_to_modules(mam_removal) ->
     MucHost = subhost_pattern(muc_light_helper:muc_host_pattern()),
     [{mod_mam, mam_helper:config_opts(#{pm => #{}, muc => #{host => MucHost}})},
      {mod_muc_light, mod_config(mod_muc_light, #{backend => rdbms})}];
+group_to_modules(mam_removal_incremental) ->
+    MucHost = subhost_pattern(muc_light_helper:muc_host_pattern()),
+    [{mod_mam, mam_helper:config_opts(#{delete_domain_limit => 1, pm => #{}, muc => #{host => MucHost}})},
+     {mod_muc_light, mod_config(mod_muc_light, #{backend => rdbms})}];
 group_to_modules(muc_light_removal) ->
     [{mod_muc_light, mod_config(mod_muc_light, #{backend => rdbms})}];
 group_to_modules(muc_removal) ->
@@ -166,10 +177,11 @@ cache_removal(Config) ->
 
 mam_pm_removal(Config) ->
     F = fun(Alice, Bob) ->
-        escalus:send(Alice, escalus_stanza:chat_to(Bob, <<"OH, HAI!">>)),
-        escalus:wait_for_stanza(Bob),
-        mam_helper:wait_for_archive_size(Alice, 1),
-        mam_helper:wait_for_archive_size(Bob, 1),
+        N = 3,
+        [ escalus:send(Alice, escalus_stanza:chat_to(Bob, <<"OH, HAI!">>)) || _ <- lists:seq(1, N) ],
+        escalus:wait_for_stanzas(Bob, N),
+        mam_helper:wait_for_archive_size(Alice, N),
+        mam_helper:wait_for_archive_size(Bob, N),
         run_remove_domain(),
         mam_helper:wait_for_archive_size(Alice, 0),
         mam_helper:wait_for_archive_size(Bob, 0)
@@ -178,14 +190,15 @@ mam_pm_removal(Config) ->
 
 mam_muc_removal(Config0) ->
     F = fun(Config, Alice) ->
+        N = 3,
         Room = muc_helper:fresh_room_name(),
         MucHost = muc_light_helper:muc_host(),
         muc_light_helper:create_room(Room, MucHost, alice,
                                      [], Config, muc_light_helper:ver(1)),
         RoomAddr = <<Room/binary, "@", MucHost/binary>>,
-        escalus:send(Alice, escalus_stanza:groupchat_to(RoomAddr, <<"text">>)),
-        escalus:wait_for_stanza(Alice),
-        mam_helper:wait_for_room_archive_size(MucHost, Room, 1),
+        [ escalus:send(Alice, escalus_stanza:groupchat_to(RoomAddr, <<"text">>)) || _ <- lists:seq(1, N) ],
+        escalus:wait_for_stanzas(Alice, N),
+        mam_helper:wait_for_room_archive_size(MucHost, Room, N),
         run_remove_domain(),
         mam_helper:wait_for_room_archive_size(MucHost, Room, 0)
         end,
@@ -394,25 +407,63 @@ last_removal(Config0) ->
 
             PresUn = escalus_client:wait_for_stanza(Alice),
             escalus:assert(is_presence_with_type, [<<"unavailable">>], PresUn),
-    
+
             %% Alice asks for Bob's last availability
             BobShortJID = escalus_client:short_jid(Bob),
             GetLast = escalus_stanza:last_activity(BobShortJID),
             Stanza = escalus_client:send_iq_and_wait_for_result(Alice, GetLast),
-    
+
             %% Alice receives Bob's status and last online time > 0
             escalus:assert(is_last_result, Stanza),
             true = (1 =< get_last_activity(Stanza)),
             <<"I am a banana!">> = get_last_status(Stanza),
-    
-            run_remove_domain(),                                         
+
+            run_remove_domain(),
             escalus_client:send(Alice, GetLast),
             Error = escalus_client:wait_for_stanza(Alice),
             escalus:assert(is_error, [<<"auth">>, <<"forbidden">>], Error)
         end,
     escalus:fresh_story_with_config(Config0, [{alice, 1}, {bob, 1}], F).
 
+removal_stops_if_handler_fails(Config0) ->
+    mongoose_helper:inject_module(?MODULE),
+    F = fun(Config, Alice) ->
+        start_domain_removal_hook(),
+        Room = muc_helper:fresh_room_name(),
+        MucHost = muc_light_helper:muc_host(),
+        muc_light_helper:create_room(Room, MucHost, alice, [], Config, muc_light_helper:ver(1)),
+        RoomAddr = <<Room/binary, "@", MucHost/binary>>,
+        escalus:send(Alice, escalus_stanza:groupchat_to(RoomAddr, <<"text">>)),
+        escalus:wait_for_stanza(Alice),
+        mam_helper:wait_for_room_archive_size(MucHost, Room, 1),
+        run_remove_domain(),
+        mam_helper:wait_for_room_archive_size(MucHost, Room, 1),
+        stop_domain_removal_hook(),
+        run_remove_domain(),
+        mam_helper:wait_for_room_archive_size(MucHost, Room, 0)
+        end,
+    escalus_fresh:story_with_config(Config0, [{alice, 1}], F).
+
 %% Helpers
+start_domain_removal_hook() ->
+    rpc(mim(), ?MODULE, rpc_start_domain_removal_hook, [host_type()]).
+
+stop_domain_removal_hook() ->
+    rpc(mim(), ?MODULE, rpc_stop_domain_removal_hook, [host_type()]).
+
+rpc_start_domain_removal_hook(HostType) ->
+    gen_hook:add_handler(remove_domain, HostType,
+                         fun ?MODULE:domain_removal_hook_fn/3,
+                         #{}, 30). %% Priority is so that it comes before muclight and mam
+
+rpc_stop_domain_removal_hook(HostType) ->
+    gen_hook:delete_handler(remove_domain, HostType,
+                            fun ?MODULE:domain_removal_hook_fn/3,
+                            #{}, 30).
+
+domain_removal_hook_fn(Acc, _Params, _Extra) ->
+    F = fun() -> throw(first_time_needs_to_fail) end,
+    mongoose_domain_api:remove_domain_wrapper(Acc, F, ?MODULE).
 
 connect_and_disconnect(Spec) ->
     {ok, Client, _} = escalus_connection:start(Spec),