Remove old outgoing pools configs

big_tests/run_common_test.erl

@@ -233,11 +233,8 @@ enable_preset(Name, PresetVars, Test, N, Tests) ->
     error_logger:info_msg("Configuration ~p of ~p: ~p started.~n",
                           [N, Tests, Name]).
 
-backend(Node) ->
-    rpc:call(Node, ejabberd_config, get_global_option, [sm_backend]).
-
 %% Specify just some nodes to run the tests on:
-%% TEST_HOSTS="mim1" ./tools/travis-test.sh -p odbc_mssql_mnesia
+%% TEST_HOSTS="mim" ./tools/travis-test.sh -p odbc_mssql_mnesia
 maybe_enable_preset_on_node(Node, PresetVars, HostVars, HostName) ->
     case is_test_host_enabled(HostName) of
         true ->
@@ -392,8 +389,9 @@ make_html(Modules) ->
 get_hosts(Test) ->
     {_File, Props} = get_ct_config(Test),
     {hosts, Hosts} = lists:keyfind(hosts, 1, Props),
-    %% `mim` is our assumed cluster name - it has to be defined in test.config
-    dict:fetch(mim, group_by(fun host_cluster/1, Hosts)).
+    %% We apply preset options to `mim` and `reg` clusters
+    Clusters = group_by(fun host_cluster/1, Hosts),
+    dict:fetch(mim, Clusters) ++ dict:fetch(reg, Clusters).
 
 get_ejabberd_nodes(Test) ->
     [ host_node(H) || H <- get_hosts(Test), is_test_host_enabled(host_name(H)) ].

big_tests/test.config

@@ -167,35 +167,22 @@
         {auth_method, <<"SASL-ANON">>}]}
                      ]}.
 
-{ejabberd_nodes, [
-    internal_mnesia,
-    internal_redis,
-    odbc_mnesia,
-    odbc_redis,
-    external_mnesia,
-    external_redis,
-    ldap_mnesia,
-    cassandra_mnesia,
-    elasticsearch_mnesia
-]}.
-
 {ejabberd_presets, [
     {internal_mnesia,
      %% dbs variable is used by ./tools/test_runner/presets_to_dbs.sh script
      [{dbs, [redis]},
       {sm_backend, "{mnesia, []}"},
       {auth_method, "internal"},
-      {mod_offline, "{mod_offline, []},"}]},
-    {internal_redis,
-     [{dbs, [redis]},
-      {sm_backend, "{redis, [{pool_size, 3}, {worker_config, [{host, \"localhost\"}, {port, 6379}]}]}"},
-      {auth_method, "internal"},
+      {outgoing_pools, "{outgoing_pools, [
+       {redis, global, global_distrib, [{workers, 10}], []}
+       ]}."},
       {mod_offline, "{mod_offline, []},"}]},
     {pgsql_mnesia,
      [{dbs, [redis, pgsql]},
       {sm_backend, "{mnesia, []}"},
       {auth_method, "rdbms"},
       {outgoing_pools, "{outgoing_pools, [
+       {redis, global, global_distrib, [{workers, 10}], []},
        {rdbms, global, default, [{workers, 5}],
         [{server, {pgsql, \"localhost\", \"ejabberd\", \"ejabberd\", \"mongooseim_secret\",
                    [{ssl, required}, {ssl_opts, [{verify, verify_peer},
@@ -207,27 +194,13 @@
       {mod_offline, "{mod_offline, [{backend, rdbms}]},"},
       {mod_vcard, "{mod_vcard, [{backend, rdbms}, {host, \"vjud.@HOST@\"}]},"},
       {mod_roster, "{mod_roster, [{backend, rdbms}]},"}]},
-    {odbc_pgsql_mnesia,
-     [{dbs, [redis, pgsql]},
-      {sm_backend, "{mnesia, []}"},
-      {auth_method, "rdbms"},
-      {rdbms_server_type, "{rdbms_server_type, pgsql}."},
-      {outgoing_pools, "{outgoing_pools, [
-       {rdbms, global, default, [{workers, 5}],
-        [{server, \"DSN=ejabberd-pgsql;UID=ejabberd;PWD=mongooseim_secret\"}]}
-       ]}."},
-      {mod_last, "{mod_last, [{backend, rdbms}]},"},
-      {mod_privacy, "{mod_privacy, [{backend, rdbms}]},"},
-      {mod_private, "{mod_private, [{backend, rdbms}]},"},
-      {mod_offline, "{mod_offline, [{backend, rdbms}]},"},
-      {mod_vcard, "{mod_vcard, [{backend, rdbms}, {host, \"vjud.@HOST@\"}]},"},
-      {mod_roster, "{mod_roster, [{backend, rdbms}]},"}]},
     {odbc_mssql_mnesia,
      [{dbs, [redis, mssql]},
       {sm_backend, "{mnesia, []}"},
       {auth_method, "rdbms"},
       {rdbms_server_type, "{rdbms_server_type, mssql}."},
       {outgoing_pools, "{outgoing_pools, [
+       {redis, global, global_distrib, [{workers, 10}], []},
        {rdbms, global, default, [{workers, 5}],
         [{server, \"DSN=mongoose-mssql;UID=sa;PWD=mongooseim_secret+ESL123\"}]}
        ]}."},
@@ -237,26 +210,13 @@
       {mod_offline, "{mod_offline, [{backend, rdbms}]},"},
       {mod_vcard, "{mod_vcard, [{backend, rdbms}, {host, \"vjud.@HOST@\"}]},"},
       {mod_roster, "{mod_roster, [{backend, rdbms}]},"}]},
-    {mysql_mnesia,
-     [{dbs, [redis, mysql]},
-      {sm_backend, "{mnesia, []}"},
-      {auth_method, "rdbms"},
-      {outgoing_pools, "{outgoing_pools, [
-       {rdbms, global, default, [{workers, 5}],
-        [{server, {mysql, \"localhost\", \"ejabberd\", \"ejabberd\", \"mongooseim_secret\",
-                   [{verify, verify_peer}, {cacertfile, \"priv/ssl/cacert.pem\"}]}}]}
-       ]}."},
-      {mod_last, "{mod_last, [{backend, rdbms}]},"},
-      {mod_privacy, "{mod_privacy, [{backend, rdbms}]},"},
-      {mod_private, "{mod_private, [{backend, mysql}]},"},
-      {mod_offline, "{mod_offline, [{backend, rdbms}]},"},
-      {mod_vcard, "{mod_vcard, [{backend, rdbms}, {host, \"vjud.@HOST@\"}]},"},
-      {mod_roster, "{mod_roster, [{backend, rdbms}]},"}]},
     {mysql_redis,
      [{dbs, [redis, mysql]},
-      {sm_backend, "{redis, [{pool_size, 3}, {worker_config, [{host, \"localhost\"}, {port, 6379}]}]}"},
+      {sm_backend, "{redis, []}"},
       {auth_method, "rdbms"},
       {outgoing_pools, "{outgoing_pools, [
+       {redis, global, global_distrib, [{workers, 10}], []},
+       {redis, global, default, [{workers, 10}, {strategy, random_worker}], []},
        {rdbms, global, default, [{workers, 5}],
         [{server, {mysql, \"localhost\", \"ejabberd\", \"ejabberd\", \"mongooseim_secret\",
                    [{verify, verify_peer}, {cacertfile, \"priv/ssl/cacert.pem\"}]}}]}
@@ -267,16 +227,13 @@
       {mod_offline, "{mod_offline, [{backend, rdbms}]},"},
       {mod_vcard, "{mod_vcard, [{backend, rdbms}, {host, \"vjud.@HOST@\"}]},"},
       {mod_roster, "{mod_roster, [{backend, rdbms}]},"}]},
-    {external_mnesia,
-     [{dbs, [redis]},
-      {sm_backend, "{mnesia, []}"},
-      {auth_method, "external"},
-      {ext_auth_script, "{extauth_program, \"priv/sample_external_auth.py\"}"},
-      {mod_offline, "{mod_offline, []},"}]},
     {ldap_mnesia,
      [{dbs, [redis, ldap]},
       {sm_backend, "{mnesia, []}"},
       {auth_method, "ldap"},
+      {outgoing_pools, "{outgoing_pools, [
+       {redis, global, global_distrib, [{workers, 10}], []}
+       ]}."},
       {mod_offline, "{mod_offline, []},"},
       {auth_ldap, "{ldap_servers,[\"localhost\"]}.\n"
                   "{ldap_port,3389}.\n"
@@ -300,6 +257,7 @@
       %% https://github.com/basho/riak-erlang-client/issues/232#issuecomment-178612129
       %% We also set ciphers in tools/setup_riak on the server side.
       {outgoing_pools, "{outgoing_pools, [
+       {redis, global, global_distrib, [{workers, 10}], []},
        {riak, global, default, [{workers, 5},
                                 {strategy, next_worker}],
                                [{address, \"127.0.0.1\"},{port, 8087},
@@ -315,31 +273,11 @@
       {mod_last, "{mod_last, [{backend, riak}]},"},
       {mod_privacy, "{mod_privacy, [{backend, riak}]},"}
      ]},
-    {cassandra_mnesia,
-     [{dbs, [redis, cassandra]},
-      {sm_backend, "{mnesia, []}"},
-      {outgoing_pools, "{outgoing_pools, [
-       {cassandra, global, default, [{workers, 20}],
-                               [{ssl,[{cacertfile, \"priv/ssl/cacert.pem\"},
-                                      {verify, verify_peer}] }]}
-       ]}."},
-      {auth_method, "internal"},
-      {mod_offline, "{mod_offline, []},"}
-     ]},
-    {elasticsearch_mnesia,
-     [{dbs, [redis, elasticsearch]},
-      {sm_backend, "{mnesia, []}"},
-      {auth_method, "internal"},
-      {outgoing_pools, "{outgoing_pools, [
-       {elastic, global, default, [], []}
-       ]}."},
-      {mod_offline, "{mod_offline, []},"}
-     ]},
     {elasticsearch_and_cassandra_mnesia,
      [{dbs, [redis, elasticsearch, cassandra]},
       {sm_backend, "{mnesia, []}"},
-      {elasticsearch_server, "{elasticsearch_server, []}."},
       {outgoing_pools, "{outgoing_pools, [
+       {redis, global, global_distrib, [{workers, 10}], []},
        {cassandra, global, default, [{workers, 20}],
                                [{ssl,[{cacertfile, \"priv/ssl/cacert.pem\"},
                                       {verify, verify_peer}] }]},

big_tests/tests/mod_global_distrib_SUITE.erl

@@ -109,14 +109,18 @@ suite() ->
 %%--------------------------------------------------------------------
 
 init_per_suite(Config) ->
-    case {rpc(europe_node1, eredis, start_link, []), rpc(asia_node, eredis, start_link, [])} of
+    case {rpc(europe_node1, mongoose_wpool, get_worker, [redis, global, global_distrib]),
+          rpc(asia_node, mongoose_wpool, get_worker, [redis, global, global_distrib])} of
         {{ok, _}, {ok, _}} ->
             ok = rpc(europe_node2, mongoose_cluster, join, [ct:get_config(europe_node1)]),
 
+            % We have to pass [no_opts] because [] is treated as string and converted
+            % automatically to <<>>
             escalus:init_per_suite([{add_advertised_endpoints, []},
-                                    {extra_config, []}, {redis_extra_config, []} | Config]);
-        _ ->
-            {skip, "Cannot connect to Redis server on 127.0.0.1 6379"}
+                                    {extra_config, []}, {redis_extra_config, [no_opts]} | Config]);
+        Result ->
+            ct:pal("Redis check result: ~p", [Result]),
+            {skip, "GD Redis default pool not available"}
     end.
 
 end_per_suite(Config) ->
@@ -159,7 +163,7 @@ init_per_group(_, Config0) ->
                                         {certfile, "priv/ssl/fake_server.pem"},
                                         {cafile, "priv/ssl/ca/cacert.pem"}
                                        ]},
-                            {redis, [{port, 6379} | ?config(redis_extra_config, Config1)]},
+                            {redis, ?config(redis_extra_config, Config1)},
                             {resend_after_ms, 500}]),
                   Opts = maybe_add_advertised_endpoints(NodeName, Opts0, Config1),
 
@@ -962,7 +966,7 @@ jids(Client) ->
     {FullJid, BareJid}.
 
 redis_query(Node, Query) ->
-    {ok, RedisWorker} = rpc(Node, mongoose_wpool, get_worker, [redis, global, distrib]),
+    {ok, RedisWorker} = rpc(Node, mongoose_wpool, get_worker, [redis, global, global_distrib]),
     rpc(Node, eredis, q, [RedisWorker, Query]).
 
 %% A fake address we don't try to connect to.

doc/Basic-configuration.md

@@ -26,12 +26,6 @@ There are 2 types of options: params and features. Unlike params, features can b
     * **Syntax:** `"{host_config, \"overridden-domain\", [{key, value}]}."`
     * **Example:** `"{host_config, \"localhost2\", [{auth_method, anonymous}, {allow_multiple_connections, false}]}." `
 
-* **riak_server** - feature
-    * **Description:** Riak connection pool configuration. Currently only one endpoint can be specified, to connect to more riak nodes you have to use load balancing techniques, for more details see:
-    [Load Balancing riak](http://docs.basho.com/riak/latest/ops/advanced/configs/load-balancing-proxy/) from basho.
-    To enable, remove '%%' prefix from value.
-    * **Syntax:** `"{riak_server, [{pool_size, Size}, {address, Host}, {port, Port}]}."`
-
 * **auth_ldap** - feature
     * **Description:** Put [[LDAP configuration]] here.
 

doc/migrations/3.1.1_3.1.1++.md

@@ -4,6 +4,8 @@
   E.g. `{auth_method, odbc}.` would now be `{auth_method, rdbms}.`.<br/>
   It's also important to note that all metrics that previously contained `odbc` in their names have also been renamed to contain `rdbms` instead.
 
+  Please note that `odbc_server` has been completely replaced with new `outgoing_pools` (see one of the next sections of this document) config element.
+
 * **For developers calling MongooseIM modules:** most modules, functions and atoms had `odbc` in their names replaced with `rdbms`.
   The only exceptions to this rule were names actually pertaining to the ODBC driver, e.g. `mongoose_rdbms_odbc`.
 
@@ -15,21 +17,50 @@ Rename the existing config file of MongooseIM from `ejabberd.cfg` to `mongooseim
 
 Configuring pools to external services has changed, please see [Outgoing Connection doc](../advanced-configuration/outgoing-connections.md) for more details.
 
-NOTE: Keep in mind that outgoing_pools is a list of pools, it may turn out that you will have more than one entry in the list when more than a single outgoing pool is needed.
-
-### ElasticSearch configuration migration
+NOTE: Keep in mind that `outgoing_pools` is a list of pools, it may turn out that you will have more than one entry in the list when more than a single outgoing pool is needed.
 
-Change the existing entry in the configuration file:
+### Example - Old format
 
 ```erlang
 {elasticsearch_server, [{host, "elastic.host.com"}, {port, 9042}]}.
+{riak_server, [{pool_size, 20}, {address, "127.0.0.1"}, {port, 8087}, {riak_pb_socket_opts, []}]}.
+{http_connections, [{conn1, [{server, "http://server:8080"}, {pool_size, 50}]} ]}.
+{cassandra_servers, [
+  {default, 100,
+   [
+    {servers,
+     [
+      {"cassandra_server1.example.com", 9042},
+      {"cassandra_server2.example.com", 9042},
+      {"cassandra_server3.example.com", 9042},
+      {"cassandra_server4.example.com", 9042}
+     ]
+    },
+    {keyspace, "big_mongooseim"}
+   ]
+  }
+ ]
+}.
 ```
 
-to:
+### Example - New format
+
+This section provides direct "translation" of configuration from "Old format" section.
 
 ```erlang
 {outgoing_pools, [
- {elastic, global, default, [], [{host, "elastic.host.com"}, {port, 9042}]}
+  {elastic, global, default, [], [{host, "elastic.host.com"}, {port, 9042}]},
+  {riak, global, default, [{workers, 20}], [{address, "127.0.0.1"}, {port, 8087}]},
+  {http, global, conn1, [{workers, 50}], [{server, "http://server:8080"}]},
+  {cassandra, global, default, [{workers, 100}], [
+        {servers, [
+          {"cassandra_server1.example.com", 9042},
+          {"cassandra_server2.example.com", 9042},
+          {"cassandra_server3.example.com", 9042},
+          {"cassandra_server4.example.com", 9042}
+        ]},
+        {keyspace, "big_mongooseim"}
+    ]}
 ]}.
 ```
 

doc/modules/mod_mam.md

@@ -122,56 +122,19 @@ This backend works with Riak KV 2.0 and above, but we recommend version 2.1.1.
 
 ### Cassandra backend
 
-Edit main config section adding:
+Please consult [Outgoing connections](../advanced-configuration/outgoing-connections.md#cassandra-connection-setup) page to learn how to properly configure Cassandra connection pool.
+By default, `mod_mam` Cassandra backend requires `global` pool with `default` tag:
 
 ```erlang
-{cassandra_servers, [{default, []}]}.
-```
-
-MongooseIM will create one pool with one worker to connect to localhost:9042.
-
-You can change the default settings using extra parameters:
-* 5 connections to each server with addresses from 10.0.0.1 to 10.0.0.4;
-* Keyspace "mongooseim";
-* Custom connect timeout in milliseconds;
-* Custom credentials.
-
-```erlang
-{cassandra_servers,
- [
-  {default,
-   [
-    {servers,
-     [
-      {"10.0.0.1", 9042, 5},
-      {"10.0.0.2", 9042, 5},
-      {"10.0.0.3", 9042, 5},
-      {"10.0.0.4", 9042, 5}
-     ]
-    },
-    {keyspace, "mongooseim"},
-    {connect_timeout, 5000}, % five seconds
-    {credentials, [{"username", "cassandra"}, {"password", "secret"}]}
-   ]
-  }
- ]
-}.
+{outgoing_pools, [
+    {cassandra, global, default, [], []}.
+]}.
 ```
 
 ### ElasticSearch backend
 
 First, make sure that your ElasticSearch cluster has expected indexes and mappings in place.
-Please refer to [database backends configuration](../advanced-configuration/database-backends-configuration.md#elasticsearch) page for information on how to configure ElasticSearch properly.
-
-Edit main config section adding:
-
-```erlang
-{elasticsearch_server, []}.
-```
-
-MongooseIM will create one pool with one worker which will try to connect to localhost:9200.
-Make sure that your ElasticSearch node's HTTP API is listening on this address!
-For more information on how to configure the connection pool, please refer to [advanced configuration](../Advanced-configuration.md#elasticsearch-connection-setup) page.
+Please consult [Outgoing connections](../advanced-configuration/outgoing-connections.md#elasticsearch-connection-setup) page to learn how to properly configure ElasticSearch connection pool.
 
 ### Example configuration