Merge pull request #3266 from esl/cluster-tests-with-dynamic-domains

Test clustering and domain management with dynamic domains

big_tests/dynamic_domains.config

@@ -30,12 +30,20 @@
          {mim2, [{node, ejabberd2@localhost},
                  {domain, <<"domain.example.com">>},
                  {host_type, <<"test type">>},
+                 {dynamic_domains, [{<<"test type">>, [<<"domain.example.com">>]}]},
                  {vars, "mim2"},
                  {cluster, mim},
+                 {c2s_port, 5232},
                  {c2s_tls_port, 5233},
                  {metrics_rest_port, 5289},
                  {gd_endpoint_port, 6666},
                  {service_port, 8899}]},
+         {mim3, [{node, mongooseim3@localhost},
+                 {domain, <<"domain.example.com">>},
+                 {host_type, <<"test type">>},
+                 {vars, "mim3"},
+                 {c2s_tls_port, 5263},
+                 {cluster, mim}]},
          %% used to test s2s features
          {fed,  [{node, fed1@localhost},
                  {domain, <<"fed1">>},
@@ -110,6 +118,12 @@
         {username, <<"admin">>},
         {server, <<"localhost">>},
         {password, <<"bruce_almighty">>}]},
+    {clusterguy, [
+        {username, <<"clusterguy">>},
+        {server, <<"domain.example.com">>},
+        {host, <<"localhost">>},
+        {password, <<"distributionftw">>},
+        {port, 5232}]},
     {alice3, [ %% used in dynamic_domains_SUITE
         {username, <<"alice">>},
         {server, <<"example.com">>},

big_tests/dynamic_domains.spec

@@ -23,10 +23,14 @@
 
 {suites, "tests", carboncopy_SUITE}.
 
+{suites, "tests", cluster_commands_SUITE}.
+
 {suites, "tests", disco_and_caps_SUITE}.
 
 {suites, "tests", domain_isolation_SUITE}.
 
+{suites, "tests", dynamic_domains_SUITE}.
+
 {suites, "tests", inbox_SUITE}.
 
 {suites, "tests", inbox_extensions_SUITE}.
@@ -77,6 +81,8 @@
 
 {suites, "tests", rest_client_SUITE}.
 
+{suites, "tests", service_domain_db_SUITE}.
+
 {suites, "tests", sic_SUITE}.
 
 {suites, "tests", sm_SUITE}.

big_tests/test.config

@@ -254,6 +254,7 @@
   connection.tls.verify_peer = true
   connection.tls.cacertfile = \"priv/ssl/cacert.pem\"
   connection.tls.server_name_indication = false"},
+      {service_domain_db, ""},
       {mod_last, "  backend = \"rdbms\""},
       {mod_privacy, "  backend = \"rdbms\""},
       {mod_private, "  backend = \"rdbms\""},

big_tests/tests/cluster_commands_SUITE.erl

@@ -24,6 +24,7 @@
                              require_rpc_nodes/1,
                              rpc/4]).
 -import(ejabberdctl_helper, [ejabberdctl/3, rpc_call/3]).
+-import(domain_helper, [host_type/1]).
 
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
@@ -41,11 +42,10 @@ all() ->
      {group, clustering_three}].
 
 groups() ->
-    G = [{clustered, [], [one_to_one_message]},
-         {clustering_two, [], clustering_two_tests()},
-         {clustering_three, [], clustering_three_tests()},
-         {mnesia, [], [set_master_test]}],
-    ct_helper:repeat_all_until_all_ok(G).
+    [{clustered, [], [one_to_one_message]},
+     {clustering_two, [], clustering_two_tests()},
+     {clustering_three, [], clustering_three_tests()},
+     {mnesia, [], [set_master_test]}].
 
 suite() ->
     require_rpc_nodes([mim, mim2, mim3]) ++ escalus:suite().
@@ -189,12 +189,10 @@ one_to_one_message(ConfigIn) ->
 %%--------------------------------------------------------------------
 
 set_master_test(ConfigIn) ->
-    Host = ct:get_config({hosts, mim, domain}),
-
     %% To ensure that passwd table exists.
     %% We also need at least two nodes for set_master to work.
-    catch distributed_helper:rpc(mim(), ejabberd_auth_internal, start, [Host]),
-    catch distributed_helper:rpc(mim2(), ejabberd_auth_internal, start, [Host]),
+    catch distributed_helper:rpc(mim(), ejabberd_auth_internal, start, [host_type(mim1)]),
+    catch distributed_helper:rpc(mim2(), ejabberd_auth_internal, start, [host_type(mim2)]),
 
     TableName = passwd,
     NodeList =  rpc_call(mnesia, system_info, [running_db_nodes]),

big_tests/tests/domain_helper.erl

@@ -36,19 +36,34 @@ make_metrics_prefix(HostType) ->
     rpc(mim(), mongoose_metrics, make_host_type_name, [HostType]).
 
 insert_configured_domains() ->
-    for_each_configured_domain(fun insert_domain/3).
+    for_each_configured_domain(fun insert_persistent_domain/3).
 
 delete_configured_domains() ->
-    for_each_configured_domain(fun(Node, Domain, _) -> delete_domain(Node, Domain) end).
+    for_each_configured_domain(fun delete_persistent_domain/3).
 
 insert_domain(Node, Domain, HostType) ->
     ok = rpc(Node, mongoose_domain_core, insert, [Domain, HostType, dummy_source]).
 
 delete_domain(Node, Domain) ->
     ok = rpc(Node, mongoose_domain_core, delete, [Domain]).
 
+insert_persistent_domain(Node, Domain, HostType) ->
+    ok = rpc(Node, mongoose_domain_api, insert_domain, [Domain, HostType]).
+
+delete_persistent_domain(Node, Domain, HostType) ->
+    ok = rpc(Node, mongoose_domain_api, delete_domain, [Domain, HostType]).
+
 for_each_configured_domain(F) ->
-    [F(#{node => proplists:get_value(node, Opts)}, Domain, HostType) ||
-        {_, Opts} <- ct:get_config(hosts),
-        {HostType, Domains} <- proplists:get_value(dynamic_domains, Opts, []),
-        Domain <- Domains].
+    [for_each_configured_domain(F, Opts) || {_, Opts} <- ct:get_config(hosts)],
+    ok.
+
+for_each_configured_domain(F, Opts) ->
+    case proplists:get_value(dynamic_domains, Opts, []) of
+        [] ->
+            ok;
+        DomainsByHostType ->
+            Node = #{node => proplists:get_value(node, Opts)},
+            [F(Node, Domain, HostType) || {HostType, Domains} <- DomainsByHostType,
+                                          Domain <- Domains],
+            rpc(Node, service_domain_db, sync_local, [])
+    end.

big_tests/tests/service_domain_db_SUITE.erl

@@ -195,6 +195,7 @@ end_per_suite(Config) ->
     Conf2 = proplists:get_value(mim_conf2, Config),
     restore_conf(mim(), Conf1),
     restore_conf(mim2(), Conf2),
+    domain_helper:insert_configured_domains(),
     dynamic_modules:restore_modules(dummy_auth_host_type(), Config),
     escalus:end_per_suite(Config).
 
@@ -504,7 +505,7 @@ db_deleted_from_one_node_while_service_disabled_on_another(_) ->
     service_disabled(mim2()),
     %% Removed from the first node
     ok = delete_domain(mim(), <<"example.com">>, <<"dbgroup">>),
-    sync(),
+    sync_local(mim()),
     {error, not_found} = get_host_type(mim(), <<"example.com">>),
     {ok, <<"dbgroup">>} = get_host_type(mim2(), <<"example.com">>),
     %% Sync is working again
@@ -529,9 +530,9 @@ db_reinserted_from_one_node_while_service_disabled_on_another(_) ->
     service_disabled(mim2()),
     %% Removed from the first node
     ok = delete_domain(mim(), <<"example.com">>, <<"dbgroup">>),
-    sync(),
+    sync_local(mim()),
     ok = insert_domain(mim(), <<"example.com">>, <<"dbgroup2">>),
-    sync(),
+    sync_local(mim()),
     %% Sync is working again
     service_enabled(mim2()),
     sync(),
@@ -1036,7 +1037,8 @@ enable_domain(Node, Domain) ->
 
 %% Call sync before get_host_type, if there are some async changes expected
 sync() ->
-    rpc(mim(), service_domain_db, sync, []).
+    sync_local(mim()),
+    sync_local(mim2()).
 
 sync_local(Node) ->
     rpc(Node, service_domain_db, sync_local, []).

rel/files/mongooseim.toml

@@ -210,6 +210,10 @@
   initial_report = 300_000
   periodic_report = 10_800_000
 
+{{#service_domain_db}}
+[services.service_domain_db]
+{{{service_domain_db}}}
+{{/service_domain_db}}
 [modules.mod_adhoc]
 
 {{#mod_amp}}

rel/mim2.vars-toml.config

@@ -11,6 +11,7 @@
 {service_port, 8899}.
 
 {hosts, "\"localhost\", \"anonymous.localhost\", \"localhost.bis\""}.
+{host_types, "\"test type\", \"dummy auth\""}.
 {default_server_domain, "\"localhost\""}.
 {s2s_addr, "[[s2s.address]]
     host = \"localhost2\"
@@ -52,7 +53,6 @@
 {c2s_dhfile, "\"priv/ssl/fake_dh_server.pem\""}.
 {s2s_dhfile, "\"priv/ssl/fake_dh_server.pem\""}.
 
-{host_types, "\"dummy auth\""}.
 {host_config,
   "[[host_config]]
   host_type = \"dummy auth\"

src/domain/service_domain_db.erl

@@ -13,13 +13,13 @@
 -export([start_link/0]).
 -export([enabled/0]).
 -export([force_check_for_updates/0]).
--export([sync/0, sync_local/0]).
+-export([sync_local/0]).
 
 %% exported for integration tests only!
 -export([reset_last_event_id/0]).
 
 -ignore_xref([code_change/3, handle_call/3, handle_cast/2, handle_info/2,
-              init/1, start_link/0, sync/0, sync_local/0, terminate/2, reset_last_event_id/0]).
+              init/1, start_link/0, sync_local/0, terminate/2, reset_last_event_id/0]).
 
 %% gen_server callbacks
 -export([init/1, handle_call/3, handle_cast/2, handle_info/2,
@@ -78,16 +78,6 @@ force_check_for_updates() ->
             ok
     end.
 
-%% Does nothing but blocks until every member processes its queue.
-sync() ->
-    case ?PG_GET_MEMBERS(?GROUP) of
-        [_|_] = Pids ->
-            [gen_server:call(Pid, ping) || Pid <- Pids],
-            ok;
-        _ ->
-            ok
-    end.
-
 sync_local() ->
     gen_server:call(?MODULE, ping).