diff --git a/big_tests/test.config b/big_tests/test.config index b3442e4601..20fecd4365 100644 --- a/big_tests/test.config +++ b/big_tests/test.config @@ -310,9 +310,8 @@ scope = \"global\" workers = 5 connection.port = 3636 - connection.rootdn = \"cn=admin,dc=esl,dc=com\" + connection.root_dn = \"cn=admin,dc=esl,dc=com\" connection.password = \"mongooseim_secret\" - connection.encrypt = \"tls\" connection.tls.versions = [\"tlsv1.2\"] connection.tls.verify_peer = true connection.tls.cacertfile = \"priv/ssl/cacert.pem\" @@ -322,7 +321,6 @@ scope = \"global\" workers = 5 connection.port = 3636 - connection.encrypt = \"tls\" connection.tls.versions = [\"tlsv1.2\"] connection.tls.verify_peer = true connection.tls.cacertfile = \"priv/ssl/cacert.pem\" @@ -369,7 +367,7 @@ [outgoing_pools.cassandra.default] scope = \"global\" workers = 20 - connection.servers = [{ip_address = \"localhost\", port = 9142}] + connection.servers = [{host = \"localhost\", port = 9142}] connection.tls.cacertfile = \"priv/ssl/cacert.pem\" connection.tls.verify_peer = true [outgoing_pools.elastic.default] diff --git a/big_tests/tests/mod_event_pusher_http_SUITE.erl b/big_tests/tests/mod_event_pusher_http_SUITE.erl index c25e8b2cca..917f444603 100644 --- a/big_tests/tests/mod_event_pusher_http_SUITE.erl +++ b/big_tests/tests/mod_event_pusher_http_SUITE.erl @@ -10,8 +10,6 @@ -compile([export_all, nowarn_export_all]). --include_lib("escalus/include/escalus.hrl"). --include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). -define(ETS_TABLE, mod_event_pusher_http). @@ -24,7 +22,7 @@ -import(domain_helper, [host_type/0]). --import(config_parser_helper, [mod_config/2, mod_event_pusher_http_handler/0]). +-import(config_parser_helper, [config/2, mod_config/2, mod_event_pusher_http_handler/0]). %%%=================================================================== %%% Suite configuration @@ -170,14 +168,13 @@ get_prefix(Config) -> get_prefix(GroupName). start_pool() -> - HTTPOpts = #{server => http_notifications_host(), path_prefix => "/", request_timeout => 2000}, PoolOpts = #{strategy => available_worker, workers => 5}, - ejabberd_node_utils:call_fun(mongoose_wpool, start_configured_pools, - [[#{type => http, scope => global, tag => http_pool, - opts => PoolOpts, conn_opts => HTTPOpts}]]). + ConnOpts = #{host => http_notifications_host(), request_timeout => 2000}, + Pool = config([outgoing_pools, http, http_pool], #{opts => PoolOpts, conn_opts => ConnOpts}), + [{ok, _Pid}] = rpc(mim(), mongoose_wpool, start_configured_pools, [[Pool]]). stop_pool() -> - ejabberd_node_utils:call_fun(mongoose_wpool, stop, [http, global, http_pool]). + rpc(mim(), mongoose_wpool, stop, [http, global, http_pool]). set_modules(Config0, ExtraHandlerOpts) -> Config = dynamic_modules:save_modules(host_type(), Config0), diff --git a/big_tests/tests/mod_event_pusher_rabbit_SUITE.erl b/big_tests/tests/mod_event_pusher_rabbit_SUITE.erl index 60408b75a1..00f8fc5b94 100644 --- a/big_tests/tests/mod_event_pusher_rabbit_SUITE.erl +++ b/big_tests/tests/mod_event_pusher_rabbit_SUITE.erl @@ -44,15 +44,8 @@ -define(CHAT_MSG_RECV_TOPIC, <<"custom_chat_msg_recv_topic">>). -define(GROUP_CHAT_MSG_SENT_TOPIC, <<"custom_group_chat_msg_sent_topic">>). -define(GROUP_CHAT_MSG_RECV_TOPIC, <<"custom_group_chat_msg_recv_topic">>). --define(WPOOL_CFG, #{type => rabbit, scope => host, tag => event_pusher, - opts => #{workers => 20, strategy => best_worker, call_timeout => 5000}, - conn_opts => #{confirms_enabled => false, - amqp_host => "localhost", - amqp_port => 5672, - amqp_username => "guest", - amqp_password => "guest", - max_worker_queue_len => 1000} -}). +-define(WPOOL_CFG, #{scope => host, + opts => #{workers => 20, strategy => best_worker, call_timeout => 5000}}). -define(IF_EXCHANGE_EXISTS_RETRIES, 30). -define(WAIT_FOR_EXCHANGE_INTERVAL, 100). % ms @@ -113,10 +106,10 @@ suite() -> %%-------------------------------------------------------------------- init_per_suite(Config) -> - start_rabbit_wpool(domain()), - {ok, _} = application:ensure_all_started(amqp_client), case is_rabbitmq_available() of true -> + start_rabbit_wpool(domain()), + {ok, _} = application:ensure_all_started(amqp_client), muc_helper:load_muc(), escalus:init_per_suite(Config); false -> @@ -621,7 +614,8 @@ start_rabbit_wpool(Host) -> start_rabbit_wpool(Host, WpoolConfig) -> rpc(mim(), mongoose_wpool, ensure_started, []), - rpc(mim(), mongoose_wpool, start_configured_pools, [[WpoolConfig], [Host]]). + Pool = config([outgoing_pools, rabbit, event_pusher], WpoolConfig), + [{ok, _Pid}] = rpc(mim(), mongoose_wpool, start_configured_pools, [[Pool], [Host]]). stop_rabbit_wpool({Pool, Host, Tag}) -> rpc(mim(), mongoose_wpool, stop, [Pool, Host, Tag]); diff --git a/big_tests/tests/muc_SUITE.erl b/big_tests/tests/muc_SUITE.erl index 80b39324fd..042cf0576d 100644 --- a/big_tests/tests/muc_SUITE.erl +++ b/big_tests/tests/muc_SUITE.erl @@ -17,7 +17,6 @@ -module(muc_SUITE). -compile([export_all, nowarn_export_all]). --include_lib("escalus/include/escalus.hrl"). -include_lib("escalus/include/escalus_xmlns.hrl"). -include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -55,7 +54,7 @@ -import(domain_helper, [host_type/0, domain/0]). -import(mongoose_helper, [backup_and_set_config_option/3, restore_config_option/2]). --import(config_parser_helper, [default_mod_config/1]). +-import(config_parser_helper, [config/2, default_mod_config/1]). -define(PASSWORD, <<"pa5sw0rd">>). -define(SUBJECT, <<"subject">>). @@ -373,10 +372,11 @@ init_per_group(disco_rsm_with_offline, Config) -> init_per_group(G, Config) when G =:= http_auth_no_server; G =:= http_auth -> PoolOpts = #{strategy => available_worker, workers => 5}, - HTTPOpts = #{server => "http://localhost:8080", path_prefix => "/muc/auth/", request_timeout => 2000}, - [{ok, _}] = ejabberd_node_utils:call_fun(mongoose_wpool, start_configured_pools, - [[#{type => http, scope => global, tag => muc_http_auth_test, - opts => PoolOpts, conn_opts => HTTPOpts}]]), + ConnOpts = #{host => "http://localhost:8080", path_prefix => <<"/muc/auth/">>, + request_timeout => 2000}, + Pool = config([outgoing_pools, http, muc_http_auth_test], + #{opts => PoolOpts, conn_opts => ConnOpts}), + [{ok, _Pid}] = rpc(mim(), mongoose_wpool, start_configured_pools, [[Pool]]), case G of http_auth -> http_helper:start(8080, "/muc/auth/check_password", fun handle_http_auth/1); _ -> ok diff --git a/big_tests/tests/push_http_SUITE.erl b/big_tests/tests/push_http_SUITE.erl index 3a6c0b6341..e4390e75ef 100644 --- a/big_tests/tests/push_http_SUITE.erl +++ b/big_tests/tests/push_http_SUITE.erl @@ -1,17 +1,13 @@ -module(push_http_SUITE). -compile([export_all, nowarn_export_all]). --include_lib("exml/include/exml.hrl"). --include_lib("escalus/include/escalus.hrl"). --include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). --include_lib("escalus/include/escalus_xmlns.hrl"). -define(ETS_TABLE, push_http). -import(push_helper, [http_notifications_port/0, http_notifications_host/0]). -import(domain_helper, [domain/0]). --import(config_parser_helper, [mod_event_pusher_http_handler/0]). +-import(config_parser_helper, [config/2, mod_event_pusher_http_handler/0]). %%-------------------------------------------------------------------- %% Suite configuration @@ -173,14 +169,13 @@ check_default_format(From, To, Body, Msg) -> start_pool() -> PoolOpts = #{strategy => random_worker, call_timeout => 5000, workers => 10}, - HTTPOpts = #{path_prefix => "/", http_opts => [], server => http_notifications_host(), - request_timeout => 5000}, - Pool = #{type => http, scope => host, tag => http_pool, opts => PoolOpts, conn_opts => HTTPOpts}, - ejabberd_node_utils:call_fun(mongoose_wpool, start_configured_pools, - [[Pool], [<<"localhost">>]]). + ConnOpts = #{host => http_notifications_host(), request_timeout => 5000}, + Pool = config([outgoing_pools, http, http_pool], + #{scope => host, opts => PoolOpts, conn_opts => ConnOpts}), + [{ok, _Pid}] = rpc(mongoose_wpool, start_configured_pools, [[Pool], [<<"localhost">>]]). stop_pool() -> - ejabberd_node_utils:call_fun(mongoose_wpool, stop, [http, <<"localhost">>, http_pool]). + rpc(mongoose_wpool, stop, [http, <<"localhost">>, http_pool]). setup_modules() -> {Mod, Code} = rpc(dynamic_compile, from_string, [custom_module_code()]), diff --git a/big_tests/tests/push_integration_SUITE.erl b/big_tests/tests/push_integration_SUITE.erl index f52f01cc88..6618d1101d 100644 --- a/big_tests/tests/push_integration_SUITE.erl +++ b/big_tests/tests/push_integration_SUITE.erl @@ -126,17 +126,13 @@ init_per_suite(Config) -> catch mongoose_push_mock:stop(), mongoose_push_mock:start(Config), Port = mongoose_push_mock:port(), - PoolOpts = #{strategy => available_worker, workers => 20}, - HTTPOpts = #{server => "https://localhost:" ++ integer_to_list(Port), path_prefix => "/", - request_timeout => 2000}, - rpc(?RPC_SPEC, mongoose_wpool, start_configured_pools, - [[#{type => http, scope => global, tag => mongoose_push_http, opts => PoolOpts, - conn_opts => HTTPOpts}]]), + ConnOpts = #{host => "https://localhost:" ++ integer_to_list(Port), request_timeout => 2000}, + Pool = config([outgoing_pools, http, mongoose_push_http], #{opts => PoolOpts, conn_opts => ConnOpts}), + [{ok, _Pid}] = rpc(?RPC_SPEC, mongoose_wpool, start_configured_pools, [[Pool]]), ConfigWithModules = dynamic_modules:save_modules(domain(), Config), escalus:init_per_suite(ConfigWithModules). - end_per_suite(Config) -> escalus_fresh:clean(), rpc(?RPC_SPEC, mongoose_wpool, stop, [http, global, mongoose_push_http]), diff --git a/big_tests/tests/push_pubsub_SUITE.erl b/big_tests/tests/push_pubsub_SUITE.erl index 8b6d587477..372ec0e854 100644 --- a/big_tests/tests/push_pubsub_SUITE.erl +++ b/big_tests/tests/push_pubsub_SUITE.erl @@ -1,15 +1,12 @@ -module(push_pubsub_SUITE). -compile([export_all, nowarn_export_all]). --include_lib("escalus/include/escalus.hrl"). --include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). --include_lib("escalus/include/escalus_xmlns.hrl"). -include_lib("exml/include/exml.hrl"). -include("push_helper.hrl"). -import(distributed_helper, [subhost_pattern/1]). - -import(domain_helper, [domain/0]). +-import(config_parser_helper, [config/2]). %%-------------------------------------------------------------------- %% Suite configuration @@ -78,16 +75,14 @@ init_per_testcase(CaseName, Config) -> MongoosePushMockPort = setup_mock_rest(), %% Start HTTP pool - HTTPOpts = #{server => "http://localhost:" ++ integer_to_list(MongoosePushMockPort), - path_prefix => "/", - request_timeout => 2000}, PoolOpts = #{strategy => available_worker, workers => 20}, - rpc(mongoose_wpool, start_configured_pools, - [[#{type => http, scope => global, tag => mongoose_push_http, - opts => PoolOpts, conn_opts => HTTPOpts}]]), + ConnOpts = #{host => "http://localhost:" ++ integer_to_list(MongoosePushMockPort), + request_timeout => 2000}, + Pool = config([outgoing_pools, http, mongoose_push_http], + #{opts => PoolOpts, conn_opts => ConnOpts}), + [{ok, _Pid}] = rpc(mongoose_wpool, start_configured_pools, [[Pool]]), escalus:init_per_testcase(CaseName, Config). - end_per_testcase(CaseName, Config) -> rpc(mongoose_wpool, stop, [http, global, mongoose_push_http]), teardown_mock_rest(), diff --git a/doc/configuration/outgoing-connections.md b/doc/configuration/outgoing-connections.md index 5cf4feec08..247ea53515 100644 --- a/doc/configuration/outgoing-connections.md +++ b/doc/configuration/outgoing-connections.md @@ -75,10 +75,52 @@ For example: #### `outgoing_pools.rdbms.*.connection.driver` * **Syntax:** string, one of `"pgsql"`, `"mysql"` or `"odbc"` (a supported driver) +* **Default:** none - this option is mandatory * **Example:** `driver = "psgql"` Selects the driver for RDBMS connection. The choice of a driver impacts the set of available options. +#### `outgoing_pools.rdbms.*.connection.keepalive_interval` +* **Syntax:** positive integer +* **Default:** not set - disabled by default +* **Example:** `keepalive_interval = 30` + +When enabled, MongooseIM will send `SELECT 1 `query through every DB connection at given interval to keep them open. This option should be used to ensure that database connections are restarted after they became broken (e.g. due to a database restart or a load balancer dropping connections). Currently, not every network-related error returned from a database driver to a regular query will imply a connection restart. + +#### `outgoing_pools.rdbms.*.connection.max_start_interval` +* **Syntax:** positive integer +* **Default:** 30 +* **Example:** `max_start_interval = 30` + +When MongooseIM fails to connect to the DB, it retries with an exponential backoff. This option limits the backoff time for faster reconnection when the DB becomes reachable again. + +### Options for `pgsql` and `mysql` + +#### `outgoing_pools.rdbms.*.connection.host` +* **Syntax:** string +* **Default:** no default; required for `pgsql` and `mysql` +* **Example:** `host = "localhost"` + +#### `outgoing_pools.rdbms.*.connection.port` +* **Syntax:** string +* **Default:** `5432` for `pgsql`; `3306` for `mysql` +* **Example:** `port = 5343` + +#### `outgoing_pools.rdbms.*.connection.database` +* **Syntax:** string +* **Default:** no default; required for `pgsql` and `mysql` +* **Example:** `database = "mim-db"` + +#### `outgoing_pools.rdbms.*.connection.username` +* **Syntax:** string +* **Default:** no default; required for `pgsql` and `mysql` +* **Example:** `username = "mim-user"` + +#### `outgoing_pools.rdbms.*.connection.password` +* **Syntax:** string +* **Default:** no default; required for `pgsql` and `mysql` +* **Example:** `password = "mim-password"` + ### ODBC options #### `outgoing_pools.rdbms.*.connection.settings` @@ -107,35 +149,11 @@ sslmode = verify-full sslrootcert = /path/to/ca/cert ``` -### Other RDBMS backends - -#### `outgoing_pools.rdbms.*.connection.host` -* **Syntax:** string -* **Example:** `host = "localhost"` - -#### `outgoing_pools.rdbms.*.connection.database` -* **Syntax:** string -* **Example:** `database = "mim-db"` - -#### `outgoing_pools.rdbms.*.connection.username` -* **Syntax:** string -* **Example:** `username = "mim-user"` - -#### `outgoing_pools.rdbms.*.connection.password` -* **Syntax:** string -* **Example:** `password = "mim-password"` - -#### `outgoing_pools.rdbms.*.connection.keepalive_interval` -* **Syntax:** positive integer -* **Default:** undefined (keep-alive not activated) -* **Example:** `keepalive_interval = 30` - -When enabled, MongooseIM will send SELECT 1 query through every DB connection at given interval to keep them open. This option should be used to ensure that database connections are restarted after they became broken (e.g. due to a database restart or a load balancer dropping connections). Currently, not every network-related error returned from a database driver to a regular query will imply a connection restart. - ## HTTP options ### `outgoing_pools.http.*.connection.host` * **Syntax:** `"http[s]://string[:integer]"` +* **Default:** no default; this option is mandatory * **Example:** `host = "https://server.com:879"` ### `outgoing_pools.http.*.connection.path_prefix` @@ -156,7 +174,7 @@ HTTP also supports all TLS-specific options described in the TLS section. ## Redis-specific options -Redis can be used as a session manager backend. +Redis can be used as a session manager backend. Global distribution (implemented in `mod_global_distrib`) requires Redis pool. There are two important limitations: @@ -195,10 +213,12 @@ Currently, only one Riak connection pool can exist for each supported XMPP host ### `outgoing_pools.riak.*.connection.address` * **Syntax:** string +* **Default:** none, this option is mandatory * **Example:** `address = "127.0.0.1"` ### `outgoing_pools.riak.*.connection.port` * **Syntax:** integer +* **Default:** none, this option is mandatory * **Example:** `port = 8087` ### `outgoing_pools.riak.*.connection.credentials` @@ -213,9 +233,9 @@ Riak also supports all TLS-specific options described in the TLS section. ## Cassandra options ### `outgoing_pools.cassandra.*.connection.servers` -* **Syntax:** a TOML array of tables containing keys `"ip_adddress"` and `"port"` -* **Default:** `[{ip_address = "localhost", port = 9042}]` -* **Example:** `servers = [{ip_address = "host_one", port = 9042}, {ip_address = "host_two", port = 9042}]` +* **Syntax:** a TOML array of tables containing keys `"host"` and `"port"` +* **Default:** `[{host = "localhost", port = 9042}]` +* **Example:** `servers = [{host = "host_one", port = 9042}, {host = "host_two", port = 9042}]` ### `outgoing_pools.cassandra.*.connection.keyspace` * **Syntax:** string @@ -226,10 +246,12 @@ To use plain text authentication (using cqerl_auth_plain_handler module): ### `outgoing_pools.cassandra.*.connection.auth.plain.username` * **Syntax:** string +* **Default:** none, this option is mandatory * **Example:** `username = "auser"` ### `outgoing_pools.cassandra.*.connection.auth.plain.password` * **Syntax:** string +* **Default:** none, this option is mandatory * **Example:** `password = "somesecretpassword"` Support for other authentication modules may be added in the future. @@ -241,7 +263,7 @@ Cassandra also supports all TLS-specific options described in the TLS section. Currently, only one pool tagged `default` can be used. ### `outgoing_pools.elastic.default.connection.host` -* **Syntax:** string +* **Syntax:** non-empty string * **Default:** `"localhost"` * **Example:** `host = "otherhost"` @@ -286,25 +308,25 @@ The `Tag` parameter must be set to `event_pusher` in order to be able to use the pool for [`mod_event_pusher_rabbit`](../modules/mod_event_pusher_rabbit.md). Any other `Tag` can be used for other purposes. -### `outgoing_pools.rabbit.*.connection.amqp_host` +### `outgoing_pools.rabbit.*.connection.host` * **Syntax:** string * **Default:** `"localhost"` -* **Example:** `amqp_host = "anotherhost"` +* **Example:** `host = "anotherhost"` -### `outgoing_pools.rabbit.*.connection.amqp_port` +### `outgoing_pools.rabbit.*.connection.port` * **Syntax:** integer * **Default:** `5672` -* **Example:** `amqp_port = 4561` +* **Example:** `port = 4561` -### `outgoing_pools.rabbit.*.connection.amqp_username` +### `outgoing_pools.rabbit.*.connection.username` * **Syntax:** string * **Default:** `"guest"` -* **Example:** `amqp_username = "corpop"` +* **Example:** `username = "corpop"` -### `outgoing_pools.rabbit.*.connection.amqp_password` +### `outgoing_pools.rabbit.*.connection.password` * **Syntax:** string * **Default:** `"guest"` -* **Example:** `amqp_password = "guest"` +* **Example:** `password = "guest"` ### `outgoing_pools.rabbit.*.connection.confirms_enabled` * **Syntax:** boolean @@ -329,13 +351,13 @@ Sets a limit of messages in a worker's mailbox above which the worker starts dro ### `outgoing_pools.ldap.*.connection.port` * **Syntax:** integer -* **Default:** `389` (or `636` if encryption is enabled) +* **Default:** `389` (or `636` if TLS is enabled) * **Example:** `port = 800` -### `outgoing_pools.ldap.*.connection.rootdn` +### `outgoing_pools.ldap.*.connection.root_dn` * **Syntax:** string -* **Default:** empty string -* **Example:** `rootdn = "cn=admin,dc=example,dc=com"` +* **Default:** empty string +* **Example:** `root_dn = "cn=admin,dc=example,dc=com"` Leaving out this option makes it an anonymous connection, which most likely is what you want. @@ -345,18 +367,14 @@ Leaving out this option makes it an anonymous connection, which most likely is w * **Example:** `password = "topsecret"` ### `outgoing_pools.ldap.*.connection.connect_interval` -* **Syntax:** integer +* **Syntax:** positive integer * **Default:** `10000` * **Example:** `connect_interval = 20000` Reconnect interval after a failed connection. -### `outgoing_pools.ldap.*.connection.encrypt` -* **Syntax:** string, one of: `"none"` or `"tls"` -* **Default:** `"none"` -* **Example:** `encrypt = "tls"` - -LDAP also supports all TLS-specific options described in the TLS section (provided `encrypt` is set to `tls`). +LDAP also supports all TLS-specific options described in the TLS section. +To enable TLS, you need to include the `tls` subsection (it can be empty). ## TLS options diff --git a/doc/migrations/5.0.0_5.1.0.md b/doc/migrations/5.0.0_5.1.0.md index b31ec4d126..793f30c6bd 100644 --- a/doc/migrations/5.0.0_5.1.0.md +++ b/doc/migrations/5.0.0_5.1.0.md @@ -18,6 +18,15 @@ See the description of [`current_domain`](../configuration/acl.md#aclmatch) for See the [auth configuration](../configuration/auth.md) for details. +### Section `outgoing_pools` + +A few options of the outgoing connection pools were changed for consistency: + +* [Cassandra servers](../configuration/outgoing-connections.md#outgoing_poolscassandraconnectionservers): `ip_address` was renamed to `host`, +* [RabbitMQ](../configuration/outgoing-connections.md#rabbitmq-options): the `amqp_` option prefix was removed, +* [LDAP](../configuration/outgoing-connections.md#ldap-options): `rootdn` was renamed to `root_dn`; +`encrypt` was removed (the `tls` option should be used instead). + ### Section `s2s` * All options can be set globally or inside `host_config`. diff --git a/src/config/mongoose_config_spec.erl b/src/config/mongoose_config_spec.erl index d7ab6e2a2f..556537cb08 100644 --- a/src/config/mongoose_config_spec.erl +++ b/src/config/mongoose_config_spec.erl @@ -4,8 +4,7 @@ -export([root/0]). %% spec parts used by http handlers, modules and services --export([wpool_items/0, - wpool_defaults/0, +-export([wpool/1, iqdisc/0, xmpp_listener_extra/1]). @@ -23,11 +22,8 @@ process_sasl_mechanism/1, process_auth/1, process_pool/2, - process_cassandra_auth/1, - process_rdbms_connection/1, + process_ldap_connection/1, process_riak_tls/1, - process_cassandra_server/1, - process_riak_credentials/1, process_iqdisc/1, process_acl_condition/1, process_s2s_host_policy/1, @@ -487,136 +483,137 @@ outgoing_pools() -> Items = [{Type, #section{items = #{default => outgoing_pool(Type)}, validate_keys = non_empty, wrap = none}} || Type <- PoolTypes], - #section{ - items = maps:from_list(Items), - wrap = global_config - }. + #section{items = maps:from_list(Items), + wrap = global_config, + include = always}. %% path: outgoing_pools.*.* outgoing_pool(Type) -> - WPool = wpool_items(), - #section{ - items = WPool#{<<"scope">> => #option{type = atom, - validate = {enum, [global, host, single_host]}}, - <<"host">> => #option{type = binary, - validate = non_empty}, - <<"connection">> => outgoing_pool_connection(Type) - }, - process = fun ?MODULE:process_pool/2, - format_items = map, - wrap = item, - defaults = maps:merge(#{<<"scope">> => global}, wpool_defaults(Type)) - }. - -wpool_items() -> - #{<<"workers">> => #option{type = integer, - validate = positive}, - <<"strategy">> => #option{type = atom, - validate = {enum, wpool_strategy_values()}}, - <<"call_timeout">> => #option{type = integer, - validate = positive} - }. - -wpool_defaults(<<"cassandra">>) -> - maps:merge(wpool_defaults(), #{<<"workers">> => 20}); -wpool_defaults(<<"rdbms">>) -> - maps:merge(wpool_defaults(), #{<<"call_timeout">> => 60000}); -wpool_defaults(_) -> - wpool_defaults(). - -wpool_defaults() -> - #{<<"workers">> => 10, - <<"strategy">> => best_worker, - <<"call_timeout">> => 5000}. + ExtraDefaults = extra_wpool_defaults(Type), + Pool = mongoose_config_utils:merge_sections(wpool(ExtraDefaults), outgoing_pool_extra(Type)), + Pool#section{wrap = item}. + +extra_wpool_defaults(<<"cassandra">>) -> + #{<<"workers">> => 20}; +extra_wpool_defaults(<<"rdbms">>) -> + #{<<"call_timeout">> => 60000}; +extra_wpool_defaults(_) -> + #{}. + +wpool(ExtraDefaults) -> + #section{items = #{<<"workers">> => #option{type = integer, + validate = positive}, + <<"strategy">> => #option{type = atom, + validate = {enum, wpool_strategy_values()}}, + <<"call_timeout">> => #option{type = integer, + validate = positive} + }, + defaults = maps:merge(#{<<"workers">> => 10, + <<"strategy">> => best_worker, + <<"call_timeout">> => 5000}, ExtraDefaults), + format_items = map}. + +outgoing_pool_extra(Type) -> + #section{items = #{<<"scope">> => #option{type = atom, + validate = {enum, [global, host, single_host]}}, + <<"host">> => #option{type = binary, + validate = non_empty}, + <<"connection">> => outgoing_pool_connection(Type) + }, + process = fun ?MODULE:process_pool/2, + format_items = map, + defaults = #{<<"scope">> => global} + }. %% path: outgoing_pools.*.*.connection outgoing_pool_connection(<<"cassandra">>) -> #section{ - items = #{<<"servers">> => #list{items = cassandra_server()}, + items = #{<<"servers">> => #list{items = cassandra_server(), + validate = unique_non_empty}, <<"keyspace">> => #option{type = atom, validate = non_empty}, <<"auth">> => #section{items = #{<<"plain">> => cassandra_auth_plain()}, required = all, - process = fun ?MODULE:process_cassandra_auth/1}, + format_items = map}, <<"tls">> => #section{items = tls_items(), - wrap = {kv, ssl}, process = fun ?MODULE:process_tls_sni/1} }, format_items = map, include = always, - defaults = #{<<"servers">> => [{"localhost", 9042}], + defaults = #{<<"servers">> => [#{host => "localhost", port => 9042}], <<"keyspace">> => mongooseim} }; outgoing_pool_connection(<<"elastic">>) -> #section{ - items = #{<<"host">> => #option{type = string, + items = #{<<"host">> => #option{type = binary, validate = non_empty}, <<"port">> => #option{type = integer, validate = port} }, format_items = map, include = always, - defaults = #{<<"host">> => "localhost", + defaults = #{<<"host">> => <<"localhost">>, <<"port">> => 9200} }; outgoing_pool_connection(<<"http">>) -> #section{ items = #{<<"host">> => #option{type = string, - validate = non_empty, - wrap = {kv, server}}, - <<"path_prefix">> => #option{type = string, + validate = non_empty}, + <<"path_prefix">> => #option{type = binary, validate = non_empty}, <<"request_timeout">> => #option{type = integer, validate = non_negative}, <<"tls">> => #section{items = tls_items(), - wrap = {kv, http_opts}, process = fun ?MODULE:process_tls_sni/1} }, format_items = map, include = always, - defaults = #{<<"path_prefix">> => "/", + required = [<<"host">>], + defaults = #{<<"path_prefix">> => <<"/">>, <<"request_timeout">> => 2000} }; outgoing_pool_connection(<<"ldap">>) -> #section{ - items = #{<<"port">> => #option{type = integer, + items = #{<<"servers">> => #list{items = #option{type = string}, + validate = unique_non_empty}, + <<"port">> => #option{type = integer, validate = port}, - <<"rootdn">> => #option{type = binary}, + <<"root_dn">> => #option{type = binary}, <<"password">> => #option{type = binary}, - <<"encrypt">> => #option{type = atom, - validate = {enum, [none, tls]}}, - <<"servers">> => #list{items = #option{type = string}}, <<"connect_interval">> => #option{type = integer, validate = positive}, <<"tls">> => #section{items = tls_items(), - wrap = {kv, tls_options}, process = fun ?MODULE:process_tls_sni/1} }, format_items = map, include = always, - defaults = #{<<"rootdn">> => <<>>, + defaults = #{<<"servers">> => ["localhost"], + <<"root_dn">> => <<>>, <<"password">> => <<>>, - <<"encrypt">> => none, - <<"servers">> => ["localhost"], - <<"connect_interval">> => 10000} + <<"connect_interval">> => 10000}, + process = fun ?MODULE:process_ldap_connection/1 }; outgoing_pool_connection(<<"rabbit">>) -> #section{ - items = #{<<"amqp_host">> => #option{type = string, - validate = non_empty}, - <<"amqp_port">> => #option{type = integer, - validate = port}, - <<"amqp_username">> => #option{type = binary, - validate = non_empty}, - <<"amqp_password">> => #option{type = binary, - validate = non_empty}, + items = #{<<"host">> => #option{type = string, + validate = non_empty}, + <<"port">> => #option{type = integer, + validate = port}, + <<"username">> => #option{type = binary, + validate = non_empty}, + <<"password">> => #option{type = binary, + validate = non_empty}, <<"confirms_enabled">> => #option{type = boolean}, <<"max_worker_queue_len">> => #option{type = int_or_infinity, validate = non_negative} }, format_items = map, include = always, - defaults = #{<<"confirms_enabled">> => false, + defaults = #{<<"host">> => "localhost", + <<"port">> => 5672, + <<"username">> => <<"guest">>, + <<"password">> => <<"guest">>, + <<"confirms_enabled">> => false, <<"max_worker_queue_len">> => 1000} }; outgoing_pool_connection(<<"rdbms">>) -> @@ -625,6 +622,8 @@ outgoing_pool_connection(<<"rdbms">>) -> validate = {enum, [odbc, pgsql, mysql]}}, <<"keepalive_interval">> => #option{type = integer, validate = positive}, + <<"max_start_interval">> => #option{type = integer, + validate = positive}, % odbc <<"settings">> => #option{type = string}, @@ -642,7 +641,9 @@ outgoing_pool_connection(<<"rdbms">>) -> validate = port}, <<"tls">> => sql_tls() }, - process = fun ?MODULE:process_rdbms_connection/1, + required = [<<"driver">>], + defaults = #{<<"max_start_interval">> => 30}, + process = fun mongoose_rdbms:process_options/1, format_items = map }; outgoing_pool_connection(<<"redis">>) -> @@ -670,19 +671,20 @@ outgoing_pool_connection(<<"riak">>) -> validate = port}, <<"credentials">> => riak_credentials(), <<"tls">> => #section{items = tls_items(), - process = fun ?MODULE:process_riak_tls/1, - wrap = none}}, + process = fun ?MODULE:process_riak_tls/1}}, + required = [<<"address">>, <<"port">>], format_items = map }. cassandra_server() -> #section{ - items = #{<<"ip_address">> => #option{type = string, - validate = non_empty}, + items = #{<<"host">> => #option{type = string, + validate = non_empty}, <<"port">> => #option{type = integer, validate = port}}, - required = [<<"ip_address">>], - process = fun ?MODULE:process_cassandra_server/1 + required = [<<"host">>], + defaults = #{<<"port">> => 9042}, + format_items = map }. %% path: outgoing_pools.cassandra.*.connection.auth.plain @@ -690,7 +692,8 @@ cassandra_auth_plain() -> #section{ items = #{<<"username">> => #option{type = binary}, <<"password">> => #option{type = binary}}, - required = all + required = all, + format_items = map }. %% path: outgoing_pools.riak.*.connection.credentials @@ -701,7 +704,7 @@ riak_credentials() -> <<"password">> => #option{type = string, validate = non_empty}}, required = all, - process = fun ?MODULE:process_riak_credentials/1 + format_items = map }. %% path: outgoing_pools.rdbms.*.connection.tls @@ -1129,9 +1132,8 @@ check_auth_method(Method, Opts) -> false -> error(#{what => missing_section_for_auth_method, auth_method => Method}) end. -process_pool([Tag, Type|_], AllOpts = #{scope := ScopeIn}) -> +process_pool([Tag, Type|_], AllOpts = #{scope := ScopeIn, connection := Connection}) -> Scope = pool_scope(ScopeIn, maps:get(host, AllOpts, none)), - Connection = maps:get(connection, AllOpts, #{}), Opts = maps:without([scope, host, connection], AllOpts), #{type => b2a(Type), scope => Scope, @@ -1146,45 +1148,9 @@ pool_scope(single_host, Host) -> Host; pool_scope(host, none) -> host; pool_scope(global, none) -> global. -process_cassandra_server(KVs) -> - {[[{ip_address, IPAddr}]], Opts} = proplists:split(KVs, [ip_address]), - case Opts of - [] -> IPAddr; - [{port, Port}] -> {IPAddr, Port} - end. - -process_cassandra_auth([{plain, KVs}]) -> - {[[{username, User}], [{password, Pass}]], []} = proplists:split(KVs, [username, password]), - {cqerl_auth_plain_handler, [{User, Pass}]}. - -process_rdbms_connection(Map) -> - KIMap = maps:with([keepalive_interval], Map), - maps:merge(KIMap, #{server => rdbms_server(Map)}). - -rdbms_server(Opts = #{driver := odbc}) -> - maps:get(settings, Opts); -rdbms_server(Opts = #{host := Host, database := DB, username := User, password := Pass, driver := Driver}) -> - PortOpts = maps:get(port, Opts, none), - TLSOpts = maps:get(tls, Opts, none), - list_to_tuple([Driver, Host] ++ db_port(PortOpts) ++ - [DB, User, Pass] ++ db_tls(Driver, TLSOpts)). - -db_port(none) -> []; -db_port(Port) -> [Port]. - -db_tls(_, none) -> []; -db_tls(Driver, KVs) -> - {[ModeOpts], Opts} = proplists:split(KVs, [required]), - [ssl_mode(Driver, ModeOpts) ++ ssl_opts(Driver, Opts)]. - -ssl_mode(pgsql, [{required, true}]) -> [{ssl, required}]; -ssl_mode(pgsql, [{required, false}]) -> [{ssl, true}]; -ssl_mode(pgsql, []) -> [{ssl, true}]; -ssl_mode(mysql, []) -> []. - -ssl_opts(pgsql, []) -> []; -ssl_opts(pgsql, Opts) -> [{ssl_opts, Opts}]; -ssl_opts(mysql, Opts) -> Opts. +process_ldap_connection(ConnOpts = #{port := _}) -> ConnOpts; +process_ldap_connection(ConnOpts = #{tls := _}) -> ConnOpts#{port => 636}; +process_ldap_connection(ConnOpts) -> ConnOpts#{port => 389}. process_riak_tls(KVs) -> KVsWithSNI = process_tls_sni(KVs), @@ -1216,10 +1182,6 @@ process_tls_sni(KVs) -> [{server_name_indication, SNIHost} | SSLOpts] end. -process_riak_credentials(KVs) -> - {[[{user, User}], [{password, Pass}]], []} = proplists:split(KVs, [user, password]), - {User, Pass}. - b2a(B) -> binary_to_atom(B, utf8). a2b(A) -> atom_to_binary(A, utf8). diff --git a/src/event_pusher/mod_event_pusher_push.erl b/src/event_pusher/mod_event_pusher_push.erl index b49b3b6b8a..c6dc34fbab 100644 --- a/src/event_pusher/mod_event_pusher_push.erl +++ b/src/event_pusher/mod_event_pusher_push.erl @@ -108,11 +108,8 @@ config_spec() -> }. wpool_spec() -> - WpoolDefaults = mongoose_config_spec:wpool_defaults(), - #section{items = mongoose_config_spec:wpool_items(), - defaults = WpoolDefaults#{<<"strategy">> := available_worker}, - format_items = map, - include = always}. + Wpool = mongoose_config_spec:wpool(#{<<"strategy">> => available_worker}), + Wpool#section{include = always}. %%-------------------------------------------------------------------- %% mod_event_pusher callbacks diff --git a/src/mongoose_amqp.erl b/src/mongoose_amqp.erl index a74dbb4b63..c8ce612dc3 100644 --- a/src/mongoose_amqp.erl +++ b/src/mongoose_amqp.erl @@ -23,7 +23,7 @@ -include_lib("amqp_client/include/amqp_client.hrl"). --export([network_params/0, network_params/1, exchange_declare/2, +-export([network_params/1, exchange_declare/2, exchange_declare_ok/0, exchange_delete/1, basic_publish/2, confirm_select/0, confirm_select_ok/0, message/1]). @@ -53,13 +53,9 @@ %%% API %%%=================================================================== --spec network_params() -> network_params(). -network_params() -> - network_params([]). - --spec network_params(proplists:proplist()) -> #amqp_params_network{}. -network_params(Opts) -> - network_params(Opts, #amqp_params_network{}). +-spec network_params(map()) -> #amqp_params_network{}. +network_params(#{host := Host, port := Port, username := UserName, password := Password}) -> + #amqp_params_network{host = Host, port = Port, username = UserName, password = Password}. -spec exchange_declare(Exchange :: binary(), Type :: binary()) -> method(). exchange_declare(Exchange, Type) -> @@ -88,15 +84,3 @@ confirm_select_ok() -> -spec message(Payload :: binary()) -> message(). message(Payload) -> #amqp_msg{payload = Payload}. - -%%%=================================================================== -%%% Helpers -%%%=================================================================== - -network_params(Opts, #amqp_params_network{host = Host, username = UserName, - password = Password}) -> - #amqp_params_network{ - host = proplists:get_value(host, Opts, Host), - port = proplists:get_value(port, Opts, ?DEFAULT_PORT), - username = proplists:get_value(username, Opts, UserName), - password = proplists:get_value(password, Opts, Password)}. diff --git a/src/mongoose_ldap_worker.erl b/src/mongoose_ldap_worker.erl index 8f0a637f53..022e58fef4 100644 --- a/src/mongoose_ldap_worker.erl +++ b/src/mongoose_ldap_worker.erl @@ -15,7 +15,6 @@ -type state() :: #{handle := none | eldap:handle(), servers := [string()], - encrypt := none | tls, tls_options := list(), port := pos_integer(), root_dn := binary(), @@ -58,23 +57,8 @@ code_change(_OldVsn, State, _Extra) -> %% internal functions -initial_state(Opts = #{servers := Servers, encrypt := Encrypt, rootdn := RootDN, password := Password, - connect_interval := ConnectInterval}) -> - TLSOptions = maps:get(tls_options, Opts, []), - DefaultPort = case Encrypt of - tls -> ?LDAPS_PORT; - starttls -> ?LDAP_PORT; - _ -> ?LDAP_PORT - end, - Port = maps:get(port, Opts, DefaultPort), - #{handle => none, - servers => Servers, - encrypt => Encrypt, - tls_options => TLSOptions, - port => Port, - root_dn => RootDN, - password => Password, - connect_interval => ConnectInterval}. +initial_state(Opts) -> + Opts#{handle => none}. call_eldap(Request, State) -> case do_call_eldap(Request, State) of @@ -91,16 +75,14 @@ call_eldap(Request, State) -> connect(State = #{handle := none, servers := Servers, - encrypt := Encrypt, - tls_options := TLSOptions, port := Port, root_dn := RootDN, password := Password, connect_interval := ConnectInterval}) -> AnonAuth = RootDN =:= <<>> andalso Password =:= <<>>, - SSLConfig = case Encrypt of - tls -> [{ssl, true}, {sslopts, TLSOptions}]; - none -> [{ssl, false}] + SSLConfig = case State of + #{tls := TLSOptions} -> [{ssl, true}, {sslopts, TLSOptions}]; + #{} -> [{ssl, false}] end, case eldap:open(Servers, [{port, Port}, {anon_auth, AnonAuth}] ++ SSLConfig) of {ok, Handle} -> diff --git a/src/mongoose_rabbit_worker.erl b/src/mongoose_rabbit_worker.erl index 3be8ea3e7b..896f80d8c5 100644 --- a/src/mongoose_rabbit_worker.erl +++ b/src/mongoose_rabbit_worker.erl @@ -112,7 +112,7 @@ terminate(_Reason, #{connection := Connection, channel := Channel, %%%=================================================================== do_init(Opts) -> - Host = proplists:get_value(host, Opts), + Host = proplists:get_value(host_type, Opts), PoolTag = proplists:get_value(pool_tag, Opts), AMQPClientOpts = proplists:get_value(amqp_client_opts, Opts), {Connection, Channel} = diff --git a/src/rdbms/mongoose_rdbms.erl b/src/rdbms/mongoose_rdbms.erl index e988ea4e44..a10c4e6328 100644 --- a/src/rdbms/mongoose_rdbms.erl +++ b/src/rdbms/mongoose_rdbms.erl @@ -56,6 +56,8 @@ sql_query/0, sql_query_part/0]). +-export([process_options/1]). + %% External exports -export([prepare/4, prepared/1, @@ -166,10 +168,29 @@ -export_type([query_result/0, server/0]). +-type options() :: #{driver := pgsql | mysql | odbc, + max_start_interval := pos_integer(), + atom() => any()}. + +-export_type([options/0]). + %%%---------------------------------------------------------------------- %%% API %%%---------------------------------------------------------------------- +-spec process_options(map()) -> options(). +process_options(Opts = #{driver := odbc, settings := _}) -> + Opts; +process_options(Opts = #{host := _Host, database := _DB, username := _User, + password := _Pass, driver := _Driver}) -> + ensure_db_port(Opts); +process_options(Opts) -> + error(#{what => invalid_rdbms_connection_options, options => Opts}). + +ensure_db_port(Opts = #{port := _}) -> Opts; +ensure_db_port(Opts = #{driver := pgsql}) -> Opts#{port => 5432}; +ensure_db_port(Opts = #{driver := mysql}) -> Opts#{port => 3306}. + -spec prepare(Name, Table :: binary() | atom(), Fields :: [binary() | atom()], Statement :: iodata()) -> {ok, Name} | {error, already_exists} @@ -565,15 +586,13 @@ to_bool(_) -> false. %%%---------------------------------------------------------------------- %%% Callback functions from gen_server %%%---------------------------------------------------------------------- --spec init(term()) -> {ok, state()}. -init(Opts) -> +-spec init(options()) -> {ok, state()}. +init(Opts = #{max_start_interval := MaxStartInterval}) -> process_flag(trap_exit, true), - {server, Settings} = lists:keyfind(server, 1, Opts), - KeepaliveInterval = proplists:get_value(keepalive_interval, Opts), - % retries are delayed exponentially, this param limits the delay - % so if we start with 2 and try 6 times, we have 2, 4, 8, 16, 30 - MaxStartInterval = proplists:get_value(start_interval, Opts, 30), - case connect(Settings, ?CONNECT_RETRIES, 2, MaxStartInterval) of + KeepaliveInterval = maps:get(keepalive_interval, Opts, undefined), + % retries are delayed exponentially, max_start_interval limits the delay + % e.g. if the limit is 30, the delays are: 2, 4, 8, 16, 30, 30, ... + case connect(Opts, ?CONNECT_RETRIES, 2, MaxStartInterval) of {ok, DbRef} -> schedule_keepalive(KeepaliveInterval), {ok, #state{db_ref = DbRef, keepalive_interval = KeepaliveInterval}}; @@ -809,10 +828,10 @@ db_type() -> _ -> generic end. --spec connect(Settings :: tuple(), Retry :: non_neg_integer(), RetryAfter :: non_neg_integer(), +-spec connect(options(), Retry :: non_neg_integer(), RetryAfter :: non_neg_integer(), MaxRetryDelay :: non_neg_integer()) -> {ok, term()} | {error, any()}. -connect(Settings, Retry, RetryAfter, MaxRetryDelay) -> - case mongoose_rdbms_backend:connect(Settings, ?QUERY_TIMEOUT) of +connect(Options, Retry, RetryAfter, MaxRetryDelay) -> + case mongoose_rdbms_backend:connect(Options, ?QUERY_TIMEOUT) of {ok, _} = Ok -> Ok; Error when Retry =:= 0 -> @@ -824,7 +843,7 @@ connect(Settings, Retry, RetryAfter, MaxRetryDelay) -> error => Error, sleep_for => SleepFor}), timer:sleep(timer:seconds(SleepFor)), NextRetryDelay = RetryAfter * RetryAfter, - connect(Settings, Retry - 1, min(MaxRetryDelay, NextRetryDelay), MaxRetryDelay) + connect(Options, Retry - 1, min(MaxRetryDelay, NextRetryDelay), MaxRetryDelay) end. diff --git a/src/rdbms/mongoose_rdbms_backend.erl b/src/rdbms/mongoose_rdbms_backend.erl index b7eb524307..2f64817935 100644 --- a/src/rdbms/mongoose_rdbms_backend.erl +++ b/src/rdbms/mongoose_rdbms_backend.erl @@ -16,12 +16,13 @@ -define(MAIN_MODULE, mongoose_rdbms). +-type options() :: mongoose_rdbms:options(). -callback escape_binary(binary()) -> mongoose_rdbms:sql_query_part(). -callback escape_string(binary()|list()) -> mongoose_rdbms:sql_query_part(). -callback unescape_binary(binary()) -> binary(). --callback connect(Args :: any(), QueryTimeout :: non_neg_integer()) -> +-callback connect(options(), QueryTimeout :: non_neg_integer()) -> {ok, Connection :: term()} | {error, Reason :: any()}. -callback disconnect(Connection :: term()) -> any(). -callback query(Connection :: term(), Query :: any(), Timeout :: infinity | non_neg_integer()) -> @@ -51,10 +52,10 @@ unescape_binary(Binary) -> Args = [Binary], mongoose_backend:call(global, ?MAIN_MODULE, ?FUNCTION_NAME, Args). --spec connect(Settings :: tuple(), QueryTimeout :: non_neg_integer()) -> +-spec connect(options(), QueryTimeout :: non_neg_integer()) -> {ok, Connection :: term()} | {error, Reason :: any()}. -connect(Settings, QueryTimeout) -> - Args = [Settings, QueryTimeout], +connect(Options, QueryTimeout) -> + Args = [Options, QueryTimeout], mongoose_backend:call(global, ?MAIN_MODULE, ?FUNCTION_NAME, Args). -spec disconnect(Connection :: term()) -> any(). diff --git a/src/rdbms/mongoose_rdbms_mysql.erl b/src/rdbms/mongoose_rdbms_mysql.erl index f14f786c15..439a4a360d 100644 --- a/src/rdbms/mongoose_rdbms_mysql.erl +++ b/src/rdbms/mongoose_rdbms_mysql.erl @@ -18,9 +18,12 @@ -author('konrad.zemek@erlang-solutions.com'). -behaviour(mongoose_rdbms_backend). --include("mongoose.hrl"). - --define(MYSQL_PORT, 3306). +-type options() :: #{host := string(), + port := inet:port(), + database := string(), + username := string(), + password := string(), + atom() => any()}. -export([escape_binary/1, unescape_binary/1, connect/2, disconnect/1, query/3, prepare/5, execute/4]). @@ -35,10 +38,10 @@ escape_binary(Bin) when is_binary(Bin) -> unescape_binary(Bin) when is_binary(Bin) -> Bin. --spec connect(Args :: any(), QueryTimeout :: non_neg_integer()) -> +-spec connect(options(), QueryTimeout :: non_neg_integer()) -> {ok, Connection :: term()} | {error, Reason :: any()}. -connect(Settings, QueryTimeout) -> - case mysql:start_link([{query_timeout, QueryTimeout} | db_opts(Settings)]) of +connect(Options, QueryTimeout) -> + case mysql:start_link([{query_timeout, QueryTimeout} | db_opts(Options)]) of {ok, Ref} -> mysql:query(Ref, <<"set names 'utf8mb4';">>), mysql:query(Ref, <<"SET SESSION query_cache_type=1;">>), @@ -70,32 +73,14 @@ execute(Connection, StatementRef, Params, _Timeout) -> %% Helpers --spec db_opts(Settings :: term()) -> list(). -db_opts({mysql, Server, DB, User, Pass}) -> - db_opts({mysql, Server, ?MYSQL_PORT, DB, User, Pass}); -db_opts({mysql, Server, Port, DB, User, Pass}) when is_integer(Port) -> - get_db_basic_opts({Server, Port, DB, User, Pass}); -db_opts({mysql, Server, DB, User, Pass, SSLConnOpts}) -> - db_opts({mysql, Server, ?MYSQL_PORT, DB, User, Pass, SSLConnOpts}); -db_opts({mysql, Server, Port, DB, User, Pass, SSLConnOpts}) - when is_integer(Port) -> - DBBasicOpts = get_db_basic_opts({Server, Port, DB, User, Pass}), - extend_db_opts_with_ssl(DBBasicOpts, SSLConnOpts). - --spec get_db_basic_opts(Settings :: term()) -> [term()]. -get_db_basic_opts({Server, Port, DB, User, Pass}) -> - [ - {host, Server}, - {port, Port}, - {user, User}, - {password, Pass}, - {database, DB}, - {found_rows, true} - ]. - --spec extend_db_opts_with_ssl(Opts :: [term()], SSLConnOpts :: [term()]) -> [term()]. -extend_db_opts_with_ssl(Opts, SSLConnOpts) -> - Opts ++ [{ssl, SSLConnOpts}]. +-spec db_opts(options()) -> [mysql:option()]. +db_opts(Options) -> + FilteredOpts = maps:with([host, port, database, username, password, tls], Options), + [{found_rows, true} | lists:map(fun process_opt/1, maps:to_list(FilteredOpts))]. + +process_opt({tls, TLSOpts}) -> {ssl, TLSOpts}; +process_opt({username, UserName}) -> {user, UserName}; +process_opt(Opt) -> Opt. %% @doc Convert MySQL query result to Erlang RDBMS result formalism -spec mysql_to_rdbms(mysql:query_result(), Conn :: term()) -> mongoose_rdbms:query_result(). diff --git a/src/rdbms/mongoose_rdbms_odbc.erl b/src/rdbms/mongoose_rdbms_odbc.erl index efb95659de..76fd652995 100644 --- a/src/rdbms/mongoose_rdbms_odbc.erl +++ b/src/rdbms/mongoose_rdbms_odbc.erl @@ -25,6 +25,8 @@ -type tabcol() :: {binary(), binary()}. +-type options() :: #{settings := string(), atom() => any()}. + %% API -spec escape_binary(binary()) -> iodata(). @@ -39,9 +41,9 @@ escape_string(Iolist) -> unescape_binary(Bin) when is_binary(Bin) -> base16:decode(Bin). --spec connect(Args :: any(), QueryTimeout :: non_neg_integer()) -> +-spec connect(options(), QueryTimeout :: non_neg_integer()) -> {ok, Connection :: term()} | {error, Reason :: any()}. -connect(Settings, _QueryTimeout) when is_list(Settings) -> +connect(#{settings := Settings}, _QueryTimeout) when is_list(Settings) -> %% We need binary_strings=off to distinguish between: %% - UTF-16 encoded NVARCHARs - encoded as binaries. %% - Binaries/regular strings - encoded as list of small integers. diff --git a/src/rdbms/mongoose_rdbms_pgsql.erl b/src/rdbms/mongoose_rdbms_pgsql.erl index 9e664e4a1e..3ba3a5e86c 100644 --- a/src/rdbms/mongoose_rdbms_pgsql.erl +++ b/src/rdbms/mongoose_rdbms_pgsql.erl @@ -20,7 +20,12 @@ -include_lib("epgsql/include/epgsql.hrl"). --define(PGSQL_PORT, 5432). +-type options() :: #{host := string(), + port := inet:port(), + database := string(), + username := string(), + password := string(), + atom() => any()}. -export([escape_binary/1, unescape_binary/1, connect/2, disconnect/1, query/3, prepare/5, execute/4]). @@ -37,10 +42,10 @@ unescape_binary(<<"\\x", Bin/binary>>) -> unescape_binary(Bin) when is_binary(Bin) -> Bin. --spec connect(Args :: any(), QueryTimeout :: non_neg_integer()) -> +-spec connect(options(), QueryTimeout :: non_neg_integer()) -> {ok, Connection :: term()} | {error, Reason :: any()}. -connect(Settings, QueryTimeout) -> - case epgsql:connect(db_opts(Settings)) of +connect(Options, QueryTimeout) -> + case epgsql:connect(db_opts(Options)) of {ok, Pid} -> epgsql:squery(Pid, [<<"SET statement_timeout=">>, integer_to_binary(QueryTimeout)]), epgsql:squery(Pid, <<"SET standard_conforming_strings=off">>), @@ -76,32 +81,23 @@ execute(Connection, StatementRef, Params, _Timeout) -> %% Helpers --spec db_opts(Settings :: term()) -> [term()]. -db_opts({pgsql, Server, DB, User, Pass}) -> - db_opts({pgsql, Server, ?PGSQL_PORT, DB, User, Pass}); -db_opts({pgsql, Server, Port, DB, User, Pass}) when is_integer(Port) -> - get_db_basic_opts({Server, Port, DB, User, Pass}); -db_opts({pgsql, Server, DB, User, Pass, SSLConnOpts}) -> - db_opts({pgsql, Server, ?PGSQL_PORT, DB, User, Pass, SSLConnOpts}); -db_opts({pgsql, Server, Port, DB, User, Pass, SSLConnOpts}) when is_integer(Port) -> - DBBasicOpts = get_db_basic_opts({Server, Port, DB, User, Pass}), - extend_db_opts_with_ssl(DBBasicOpts, SSLConnOpts). - --spec get_db_basic_opts(Settings :: term()) -> [term()]. -get_db_basic_opts({Server, Port, DB, User, Pass}) -> - [ - {host, Server}, - {port, Port}, - {database, DB}, - {username, User}, - {password, Pass}, - %% Encode 0 and 1 as booleans, as well as true and false - {codecs, [{mongoose_rdbms_pgsql_codec_boolean, []}]} - ]. - --spec extend_db_opts_with_ssl(Opts :: [term()], SSLConnOpts :: [term()]) -> [term()]. -extend_db_opts_with_ssl(Opts, SSLConnOpts) -> - Opts ++ SSLConnOpts. +-spec db_opts(options()) -> epgsql:connect_opts(). +db_opts(Options) -> + BasicOpts = maps:with([host, port, database, username, password], Options), + TLSOpts = tls_opts(Options), + maps:merge(BasicOpts#{codecs => [{mongoose_rdbms_pgsql_codec_boolean, []}]}, TLSOpts). + +tls_opts(#{tls := KVs}) -> + {[ModeOpts], Opts} = proplists:split(KVs, [required]), + (ssl_opts(Opts))#{ssl => ssl_mode(ModeOpts)}; +tls_opts(#{}) -> + #{}. + +ssl_mode([{required, true}]) -> required; +ssl_mode(_) -> true. + +ssl_opts([]) -> #{}; +ssl_opts(Opts) -> #{ssl_opts => Opts}. -spec pgsql_to_rdbms(epgsql:reply(term())) -> mongoose_rdbms:query_result(). pgsql_to_rdbms(Items) when is_list(Items) -> diff --git a/src/system_metrics/mongoose_system_metrics_collector.erl b/src/system_metrics/mongoose_system_metrics_collector.erl index dad0ee4dbd..6f9a4e1a46 100644 --- a/src/system_metrics/mongoose_system_metrics_collector.erl +++ b/src/system_metrics/mongoose_system_metrics_collector.erl @@ -166,7 +166,7 @@ extract_tls_options(#{tls := Opts}) -> extract_tls_options(_) -> []. get_outgoing_pools() -> - OutgoingPools = mongoose_config:get_opt(outgoing_pools, []), + OutgoingPools = mongoose_config:get_opt(outgoing_pools), [#{report_name => outgoing_pools, key => type, value => Type} || #{type := Type} <- OutgoingPools]. diff --git a/src/wpool/mongoose_wpool.erl b/src/wpool/mongoose_wpool.erl index 9b0771a90b..bff15ba4b4 100644 --- a/src/wpool/mongoose_wpool.erl +++ b/src/wpool/mongoose_wpool.erl @@ -90,10 +90,10 @@ -type callback_fun() :: init | start | is_supported_strategy | stop. -callback init() -> ok | {error, term()}. --callback start(scope(), tag(), WPoolOpts :: pool_opts(), ConnOpts :: conn_opts()) -> +-callback start(host_type_or_global(), tag(), WPoolOpts :: pool_opts(), ConnOpts :: conn_opts()) -> {ok, {pid(), proplists:proplist()}} | {ok, pid()} | {error, Reason :: term()}. -callback is_supported_strategy(Strategy :: wpool:strategy()) -> boolean(). --callback stop(scope(), tag()) -> ok. +-callback stop(host_type_or_global(), tag()) -> ok. -optional_callbacks([is_supported_strategy/1]). @@ -118,7 +118,7 @@ ensure_started() -> end. start_configured_pools() -> - Pools = mongoose_config:get_opt(outgoing_pools, []), + Pools = mongoose_config:get_opt(outgoing_pools), start_configured_pools(Pools). start_configured_pools(PoolsIn) -> @@ -227,7 +227,7 @@ stop(PoolType, HostType, Tag) -> -spec is_configured(pool_type()) -> boolean(). is_configured(PoolType) -> - Pools = mongoose_config:get_opt(outgoing_pools, []), + Pools = mongoose_config:get_opt(outgoing_pools), lists:any(fun(#{type := Type}) -> Type =:= PoolType end, Pools). -spec get_worker(pool_type()) -> worker_result(). diff --git a/src/wpool/mongoose_wpool_cassandra.erl b/src/wpool/mongoose_wpool_cassandra.erl index 3c0a762e35..9e9dfa2450 100644 --- a/src/wpool/mongoose_wpool_cassandra.erl +++ b/src/wpool/mongoose_wpool_cassandra.erl @@ -7,17 +7,20 @@ %% -------------------------------------------------------------- %% mongoose_wpool callbacks +-spec init() -> ok. init() -> {ok, []} = application:ensure_all_started(cqerl), application:set_env(cqerl, maps, true). -start(HostType, Tag, WpoolOptsIn, CqerlOpts) -> +-spec start(mongooseim:host_type_or_global(), mongoose_wpool:tag(), + mongoose_wpool:pool_opts(), mongoose_wpool:conn_opts()) -> {ok, pid()} | {error, any()}. +start(HostType, Tag, WpoolOptsIn, ConnOpts) -> PoolSize = proplists:get_value(workers, WpoolOptsIn), application:set_env(cqerl, num_clients, PoolSize), - ExtConfig = extend_config(CqerlOpts), - Servers = proplists:get_value(servers, ExtConfig), - set_cluster_config(Tag, Servers, ExtConfig), - Res = cqerl_cluster:add_nodes(Tag, Servers, ExtConfig), + Servers = prepare_cqerl_servers(ConnOpts), + CqerlOpts = prepare_cqerl_opts(ConnOpts), + set_cluster_config(Tag, Servers, CqerlOpts), + Res = cqerl_cluster:add_nodes(Tag, Servers, CqerlOpts), case lists:keyfind(error, 1, Res) of false -> ok; @@ -29,14 +32,31 @@ start(HostType, Tag, WpoolOptsIn, CqerlOpts) -> WpoolOpts = [{worker, Worker} | WpoolOptsIn], mongoose_wpool:start_sup_pool(cassandra, Name, WpoolOpts). +-spec stop(mongooseim:host_type_or_global(), mongoose_wpool:tag()) -> ok. stop(_, _) -> ok. %% -------------------------------------------------------------- %% Internal functions -extend_config(PoolConfig) -> - ConfigMap = maps:merge(#{tcp_opts => [{keepalive, true}]}, PoolConfig), - maps:to_list(ConfigMap). +prepare_cqerl_servers(#{servers := Servers}) -> + [cqerl_server(Server) || Server <- Servers]. + +cqerl_server(#{host := Host, port := Port}) -> {Host, Port}; +cqerl_server(#{host := Host}) -> Host. + +prepare_cqerl_opts(ConnOpts) -> + lists:flatmap(fun(Opt) -> cqerl_opts(Opt, ConnOpts) end, [keyspace, auth, tcp, tls]). + +cqerl_opts(keyspace, #{keyspace := Keyspace}) -> + [{keyspace, Keyspace}]; +cqerl_opts(auth, #{auth := #{plain := #{username := UserName, password := Password}}}) -> + [{cqerl_auth_plain_handler, [{UserName, Password}]}]; +cqerl_opts(tcp, #{}) -> + [{tcp_opts, [{keepalive, true}]}]; % always set +cqerl_opts(tls, #{tls := TLSOpts}) -> + [{ssl, TLSOpts}]; +cqerl_opts(_Opt, #{}) -> + []. %% make the config survive the restart of 'cqerl_cluster' in case of a network failure set_cluster_config(Tag, Servers, ExtConfig) -> diff --git a/src/wpool/mongoose_wpool_elastic.erl b/src/wpool/mongoose_wpool_elastic.erl index 0ea3218324..8a15b6a84a 100644 --- a/src/wpool/mongoose_wpool_elastic.erl +++ b/src/wpool/mongoose_wpool_elastic.erl @@ -7,19 +7,21 @@ %% -------------------------------------------------------------- %% mongoose_wpool callbacks +-spec init() -> ok. init() -> tirerl:start(), ok. -start(HostType, Tag, WpoolOptsIn, #{host := ElasticHost, port := Port}) -> +-spec start(mongooseim:host_type_or_global(), mongoose_wpool:tag(), + mongoose_wpool:pool_opts(), mongoose_wpool:conn_opts()) -> {ok, pid()} | {error, any()}. +start(HostType, Tag, WpoolOptsIn, ConnOpts) -> ProcName = mongoose_wpool:make_pool_name(elastic, HostType, Tag), - Opts = [{host, list_to_binary(ElasticHost)}, {port, Port}], WPoolOptions = [{overrun_warning, infinity}, {overrun_handler, {error_logger, warning_report}}, - {worker, {tirerl_worker, Opts}} + {worker, {tirerl_worker, maps:to_list(ConnOpts)}} | WpoolOptsIn], mongoose_wpool:start_sup_pool(elastic, ProcName, WPoolOptions). +-spec stop(mongooseim:host_type_or_global(), mongoose_wpool:tag()) -> ok. stop(_, _) -> ok. - diff --git a/src/wpool/mongoose_wpool_http.erl b/src/wpool/mongoose_wpool_http.erl index 3556367dfe..d84a3c31da 100644 --- a/src/wpool/mongoose_wpool_http.erl +++ b/src/wpool/mongoose_wpool_http.erl @@ -18,6 +18,7 @@ %% -------------------------------------------------------------- %% mongoose_wpool callbacks +-spec init() -> ok. init() -> case ets:info(?MODULE) of undefined -> @@ -32,11 +33,12 @@ init() -> ok end. +-spec start(mongooseim:host_type_or_global(), mongoose_wpool:tag(), + mongoose_wpool:pool_opts(), mongoose_wpool:conn_opts()) -> {ok, pid()} | {error, any()}. start(HostType, Tag, WpoolOptsIn, ConnOpts) -> Name = mongoose_wpool:make_pool_name(http, HostType, Tag), WpoolOpts = wpool_spec(WpoolOptsIn, ConnOpts), - PathPrefix = list_to_binary(maps:get(path_prefix, ConnOpts)), - RequestTimeout = maps:get(request_timeout, ConnOpts), + #{path_prefix := PathPrefix, request_timeout := RequestTimeout} = ConnOpts, case mongoose_wpool:start_sup_pool(http, Name, WpoolOpts) of {ok, Pid} -> ets:insert(?MODULE, {{HostType, Tag}, PathPrefix, RequestTimeout}), @@ -45,6 +47,7 @@ start(HostType, Tag, WpoolOptsIn, ConnOpts) -> Other end. +-spec stop(mongooseim:host_type_or_global(), mongoose_wpool:tag()) -> ok. stop(HostType, Tag) -> true = ets:delete(?MODULE, {HostType, Tag}), ok. @@ -65,9 +68,7 @@ get_params(HostType, Tag) -> %% -------------------------------------------------------------- %% Internal functions -wpool_spec(WpoolOptsIn, ConnOpts) -> - TargetServer = maps:get(server, ConnOpts), - HttpOpts = maps:get(http_opts, ConnOpts, []), - Worker = {fusco, {TargetServer, [{connect_options, HttpOpts}]}}, +wpool_spec(WpoolOptsIn, #{host := Host} = ConnOpts) -> + HTTPOpts = maps:get(tls, ConnOpts, []), + Worker = {fusco, {Host, [{connect_options, HTTPOpts}]}}, [{worker, Worker} | WpoolOptsIn]. - diff --git a/src/wpool/mongoose_wpool_ldap.erl b/src/wpool/mongoose_wpool_ldap.erl index 675d37b1f0..7985b8ab54 100644 --- a/src/wpool/mongoose_wpool_ldap.erl +++ b/src/wpool/mongoose_wpool_ldap.erl @@ -6,13 +6,17 @@ %% -------------------------------------------------------------- %% mongoose_wpool callbacks +-spec init() -> ok. init() -> ok. +-spec start(mongooseim:host_type_or_global(), mongoose_wpool:tag(), + mongoose_wpool:pool_opts(), mongoose_wpool:conn_opts()) -> {ok, pid()} | {error, any()}. start(HostType, Tag, WpoolOpts, ConnOpts) -> WorkerSpec = {mongoose_ldap_worker, ConnOpts}, ProcName = mongoose_wpool:make_pool_name(ldap, HostType, Tag), mongoose_wpool:start_sup_pool(ldap, ProcName, [{worker, WorkerSpec} | WpoolOpts]). +-spec stop(mongooseim:host_type_or_global(), mongoose_wpool:tag()) -> ok. stop(_HostType, _Tag) -> ok. diff --git a/src/wpool/mongoose_wpool_rabbit.erl b/src/wpool/mongoose_wpool_rabbit.erl index b710d20d95..cc0e337ec7 100644 --- a/src/wpool/mongoose_wpool_rabbit.erl +++ b/src/wpool/mongoose_wpool_rabbit.erl @@ -5,25 +5,24 @@ -export([start/4]). -export([stop/2]). +-spec init() -> ok | {error, any()}. init() -> application:ensure_all_started(amqp_client). -start(Host, Tag, WpoolOptsIn, - AMQPOpts = #{confirms_enabled := Confirms, max_worker_queue_len := MaxQueueLen}) -> - PoolName = mongoose_wpool:make_pool_name(rabbit, Host, Tag), +-spec start(mongooseim:host_type_or_global(), mongoose_wpool:tag(), + mongoose_wpool:pool_opts(), mongoose_wpool:conn_opts()) -> {ok, pid()} | {error, any()}. +start(HostType, Tag, WpoolOptsIn, ConnOpts) -> + #{confirms_enabled := Confirms, max_worker_queue_len := MaxQueueLen} = ConnOpts, + PoolName = mongoose_wpool:make_pool_name(rabbit, HostType, Tag), Worker = {mongoose_rabbit_worker, - [{amqp_client_opts, amqp_client_opts(AMQPOpts)}, - {host, Host}, + [{amqp_client_opts, mongoose_amqp:network_params(ConnOpts)}, + {host_type, HostType}, {pool_tag, Tag}, {confirms, Confirms}, {max_queue_len, MaxQueueLen}]}, WpoolOpts = [{worker, Worker} | WpoolOptsIn], mongoose_wpool:start_sup_pool(rabbit, PoolName, WpoolOpts). +-spec stop(mongooseim:host_type_or_global(), mongoose_wpool:tag()) -> ok. stop(_, _) -> ok. - -amqp_client_opts(AMQPOpts) -> - AMQPNetworkOpts = maps:with([amqp_host, amqp_port, amqp_username, amqp_password], AMQPOpts), - KVOpts = maps:to_list(AMQPNetworkOpts), - mongoose_amqp:network_params(KVOpts). diff --git a/src/wpool/mongoose_wpool_rdbms.erl b/src/wpool/mongoose_wpool_rdbms.erl index aff2b22d47..050004e745 100644 --- a/src/wpool/mongoose_wpool_rdbms.erl +++ b/src/wpool/mongoose_wpool_rdbms.erl @@ -7,6 +7,7 @@ %% -------------------------------------------------------------- %% mongoose_wpool callbacks +-spec init() -> ok. init() -> case ets:info(prepared_statements) of undefined -> @@ -21,34 +22,28 @@ init() -> ok end. +-spec start(mongooseim:host_type_or_global(), mongoose_wpool:tag(), + mongoose_wpool:pool_opts(), mongoose_wpool:conn_opts()) -> {ok, pid()} | {error, any()}. start(HostType, Tag, WpoolOpts, RdbmsOpts) -> try do_start(HostType, Tag, WpoolOpts, RdbmsOpts) catch Err -> {error, Err} end. +-spec stop(mongooseim:host_type_or_global(), mongoose_wpool:tag()) -> ok. stop(_, _) -> ok. %% -------------------------------------------------------------- %% Helper functions do_start(HostType, Tag, WpoolOpts0, RdbmsOpts) when is_list(WpoolOpts0) and is_map(RdbmsOpts) -> - BackendName = backend_name(RdbmsOpts), - KVRdbmsOpts = maps:to_list(RdbmsOpts), + #{driver := BackendName} = RdbmsOpts, mongoose_backend:init(global, mongoose_rdbms, [query, execute], #{backend => BackendName}), - mongoose_metrics:ensure_db_pool_metric({rdbms, HostType, Tag}), - WpoolOpts = make_wpool_opts(WpoolOpts0, KVRdbmsOpts), + WpoolOpts = make_wpool_opts(WpoolOpts0, RdbmsOpts), ProcName = mongoose_wpool:make_pool_name(rdbms, HostType, Tag), mongoose_wpool:start_sup_pool(rdbms, ProcName, WpoolOpts). make_wpool_opts(WpoolOpts0, RdbmsOpts) -> Worker = {mongoose_rdbms, RdbmsOpts}, [{worker, Worker}, {pool_sup_shutdown, infinity} | WpoolOpts0]. - --spec backend_name(map()) -> odbc | pgsql | mysql. -backend_name(RdbmsOpts) -> - case maps:get(server, RdbmsOpts) of - ConnStr when is_list(ConnStr) -> odbc; - Tuple when is_tuple(Tuple) -> element(1, Tuple) - end. diff --git a/src/wpool/mongoose_wpool_redis.erl b/src/wpool/mongoose_wpool_redis.erl index abae497297..243dfe838e 100644 --- a/src/wpool/mongoose_wpool_redis.erl +++ b/src/wpool/mongoose_wpool_redis.erl @@ -8,14 +8,18 @@ %% -------------------------------------------------------------- %% mongoose_wpool callbacks +-spec init() -> ok. init() -> ok. +-spec start(mongooseim:host_type_or_global(), mongoose_wpool:tag(), + mongoose_wpool:pool_opts(), mongoose_wpool:conn_opts()) -> {ok, pid()} | {error, any()}. start(HostType, Tag, WpoolOptsIn, ConnOpts) -> ProcName = mongoose_wpool:make_pool_name(redis, HostType, Tag), WpoolOpts = wpool_spec(WpoolOptsIn, ConnOpts), mongoose_wpool:start_sup_pool(redis, ProcName, WpoolOpts). +-spec stop(mongooseim:host_type_or_global(), mongoose_wpool:tag()) -> ok. stop(_, _) -> ok. diff --git a/src/wpool/mongoose_wpool_riak.erl b/src/wpool/mongoose_wpool_riak.erl index 9239455837..f0f0ad5ebd 100644 --- a/src/wpool/mongoose_wpool_riak.erl +++ b/src/wpool/mongoose_wpool_riak.erl @@ -8,14 +8,18 @@ %% -------------------------------------------------------------- %% mongoose_wpool callbacks +-spec init() -> ok. init() -> ok. +-spec start(mongooseim:host_type_or_global(), mongoose_wpool:tag(), + mongoose_wpool:pool_opts(), mongoose_wpool:conn_opts()) -> {ok, pid()} | {error, any()}. start(HostType, Tag, WpoolOptsIn, ConnOpts) -> ProcName = mongoose_wpool:make_pool_name(riak, HostType, Tag), WpoolOpts = wpool_spec(WpoolOptsIn, ConnOpts), mongoose_wpool:start_sup_pool(riak, ProcName, WpoolOpts). +-spec stop(mongooseim:host_type_or_global(), mongoose_wpool:tag()) -> ok. stop(_, _) -> ok. @@ -33,12 +37,11 @@ wpool_spec(WpoolOptsIn, ConnOpts = #{address := RiakAddr, port := RiakPort}) -> [{worker, Worker} | WpoolOptsIn]. prepare_sec_opts(ConnOpts) -> - SecurityOptsKeys = [credentials, cacertfile, ssl_opts], - SecurityOpts = maps:with(SecurityOptsKeys, ConnOpts), - ListOpts = maps:to_list(SecurityOpts), - lists:map(fun prepare_credentials/1, ListOpts). - -prepare_credentials({credentials, {User, Password}}) -> - {credentials, User, Password}; -prepare_credentials(Opt) -> - Opt. + lists:flatmap(fun(Opt) -> sec_opts(Opt, ConnOpts) end, [credentials, tls]). + +sec_opts(credentials, #{credentials := #{user := User, password := Password}}) -> + [{credentials, User, Password}]; +sec_opts(tls, #{tls := TLSOpts}) -> + TLSOpts; +sec_opts(_Opt, #{}) -> + []. diff --git a/test/auth_http_SUITE.erl b/test/auth_http_SUITE.erl index f98c1fdf3e..f66f9385f5 100644 --- a/test/auth_http_SUITE.erl +++ b/test/auth_http_SUITE.erl @@ -18,13 +18,13 @@ -compile([export_all, nowarn_export_all]). -author('piotr.nosek@erlang-solutions.com'). --include_lib("common_test/include/ct.hrl"). - -define(DOMAIN, <<"localhost">>). -define(HOST_TYPE, <<"test host type">>). -define(AUTH_HOST, "http://localhost:12000"). -define(BASIC_AUTH, "softkitty:purrpurrpurr"). +-import(config_parser_helper, [config/2]). + %%-------------------------------------------------------------------- %% Suite configuration %%-------------------------------------------------------------------- @@ -75,10 +75,7 @@ init_per_suite(Config) -> mim_ct_sup:start_link(ejabberd_sup), mongoose_wpool:ensure_started(), % This would be started via outgoing_pools in normal case - Pool = #{type => http, scope => host, tag => auth, - opts => #{strategy => random_worker, call_timeout => 5000, workers => 20}, - conn_opts => #{path_prefix => "/auth/", http_opts => [], - server => ?AUTH_HOST, request_timeout => 2000}}, + Pool = config([outgoing_pools, http, auth], pool_opts()), HostTypes = [?HOST_TYPE, <<"another host type">>], mongoose_wpool:start_configured_pools([Pool], HostTypes), mongoose_wpool_http:init(), @@ -86,6 +83,11 @@ init_per_suite(Config) -> end), Config. +pool_opts() -> + #{scope => host, + opts => #{strategy => random_worker, call_timeout => 5000, workers => 20}, + conn_opts => #{host => ?AUTH_HOST, path_prefix => <<"/auth/">>}}. + end_per_suite(Config) -> ejabberd_auth_http:stop(?HOST_TYPE), ok = mim_ct_rest:stop(), diff --git a/test/common/config_parser_helper.erl b/test/common/config_parser_helper.erl index 318a88ffdc..82e9502764 100644 --- a/test/common/config_parser_helper.erl +++ b/test/common/config_parser_helper.erl @@ -18,6 +18,7 @@ options("host_types") -> {listen, []}, {loglevel, warning}, {mongooseimctl_access_commands, []}, + {outgoing_pools, []}, {rdbms_server_type, generic}, {registration_timeout, 600}, {routing_modules, mongoose_router:default_routing_modules()}, @@ -78,6 +79,7 @@ options("miscellaneous") -> {loglevel, warning}, {mongooseimctl_access_commands, [{local, ["join_cluster"], [{node, "mongooseim@prime"}]}]}, + {outgoing_pools, []}, {rdbms_server_type, mssql}, {registration_timeout, 600}, {routing_modules, @@ -108,6 +110,7 @@ options("modules") -> {listen, []}, {loglevel, warning}, {mongooseimctl_access_commands, []}, + {outgoing_pools, []}, {rdbms_server_type, generic}, {registration_timeout, 600}, {routing_modules, mongoose_router:default_routing_modules()}, @@ -239,18 +242,21 @@ options("mongooseim-pgsql") -> {max_fsm_queue, 1000}, {mongooseimctl_access_commands, []}, {outgoing_pools, - lists:map(fun merge_with_default_pool_config/1, - [#{type => rdbms, scope => global, tag => default, - opts => #{workers => 5}, - conn_opts => #{server => - {pgsql, "localhost", "ejabberd", "ejabberd", "mongooseim_secret", - [{ssl, required}, - {ssl_opts, - [{cacertfile, "priv/ca.pem"}, - {server_name_indication, disable}, - {verify, verify_peer}]}]}}}, - #{type => redis, scope => <<"localhost">>, tag => global_distrib, - opts => #{workers => 10}, conn_opts => #{}}])}, + lists:map( + fun pool_config/1, + [#{type => rdbms, + opts => #{workers => 5}, + conn_opts => #{driver => pgsql, host => "localhost", port => 5432, database => "ejabberd", + username => "ejabberd", password => "mongooseim_secret", + tls => [{required, true}, + {cacertfile, "priv/ca.pem"}, + {server_name_indication, disable}, + {verify, verify_peer}] + } + }, + #{type => redis, scope => <<"localhost">>, tag => global_distrib, + opts => #{workers => 10}, conn_opts => #{}} + ])}, {rdbms_server_type, generic}, {registration_timeout, infinity}, {routing_modules, mongoose_router:default_routing_modules()}, @@ -315,51 +321,52 @@ options("outgoing_pools") -> {loglevel, warning}, {mongooseimctl_access_commands, []}, {outgoing_pools, - lists:map(fun merge_with_default_pool_config/1, - [#{type => cassandra, scope => global, tag => default, opts => #{}, - conn_opts => #{keyspace => big_mongooseim, - servers => [{"cassandra_server1.example.com", 9042}, - {"cassandra_server2.example.com", 9042}]}}, - #{type => elastic, scope => global, tag => default, opts => #{}, - conn_opts => #{host => "localhost"}}, - #{type => http, scope => global, tag => mongoose_push_http, - opts => #{workers => 50}, - conn_opts => #{server => "https://localhost:8443", - path_prefix => "/", - request_timeout => 2000}}, - #{type => ldap, scope => host, tag => default, - opts => #{workers => 5}, - conn_opts => #{password => <<"ldap-admin-password">>, - rootdn => <<"cn=admin,dc=example,dc=com">>, - servers => ["ldap-server.example.com"]}}, - #{type => rabbit, scope => host, tag => event_pusher, - opts => #{workers => 20}, - conn_opts => #{amqp_host => "localhost", - amqp_password => <<"guest">>, - amqp_port => 5672, - amqp_username => <<"guest">>, - confirms_enabled => true, - max_worker_queue_len => 100}}, - #{type => rdbms, scope => global, tag => default, - opts => #{workers => 5}, - conn_opts => #{server => - {pgsql, "localhost", "ejabberd", "ejabberd", "mongooseim_secret", - [{ssl, required}, - {ssl_opts, [{cacertfile, "priv/ca.pem"}, - {server_name_indication, disable}, - {verify, verify_peer}]}]}, - keepalive_interval => 30}}, - #{type => redis, scope => <<"localhost">>, tag => global_distrib, - opts => #{workers => 10}, conn_opts => #{}}, - #{type => riak, scope => global, tag => default, - opts => #{strategy => next_worker, workers => 20}, - conn_opts => #{address => "127.0.0.1", - credentials => {"username", "pass"}, - port => 8087, - ssl_opts => [{certfile, "path/to/cert.pem"}, - {keyfile, "path/to/key.pem"}, - {verify, verify_peer}], - cacertfile => "path/to/cacert.pem"}}])}, + lists:map( + fun pool_config/1, + [#{type => cassandra, scope => global, tag => default, opts => #{}, + conn_opts => #{keyspace => big_mongooseim, + servers => [#{host => "cassandra_server1.example.com", port => 9042}, + #{host => "cassandra_server2.example.com", port => 9042}]}}, + #{type => elastic, scope => global, tag => default, opts => #{}, conn_opts => #{}}, + #{type => http, scope => global, tag => mongoose_push_http, + opts => #{workers => 50}, + conn_opts => #{host => "https://localhost:8443", + request_timeout => 2000}}, + #{type => ldap, scope => host, tag => default, + opts => #{workers => 5}, + conn_opts => #{password => <<"ldap-admin-password">>, + root_dn => <<"cn=admin,dc=example,dc=com">>, + servers => ["ldap-server.example.com"]}}, + #{type => rabbit, scope => host, tag => event_pusher, + opts => #{workers => 20}, + conn_opts => #{confirms_enabled => true, + max_worker_queue_len => 100}}, + #{type => rdbms, + opts => #{workers => 5}, + conn_opts => #{keepalive_interval => 30, + driver => pgsql, host => "localhost", port => 5432, database => "ejabberd", + username => "ejabberd", password => "mongooseim_secret", + tls => [{required, true}, + {cacertfile, "priv/ca.pem"}, + {server_name_indication, disable}, + {verify, verify_peer}] + } + }, + #{type => redis, scope => <<"localhost">>, tag => global_distrib, + opts => #{workers => 10}, conn_opts => #{}}, + #{type => riak, scope => global, tag => default, + opts => #{strategy => next_worker, workers => 20}, + conn_opts => #{address => "127.0.0.1", + credentials => #{user => "username", password => "pass"}, + port => 8087, + tls => [{cacertfile, "path/to/cacert.pem"}, + {ssl_opts, [{certfile, "path/to/cert.pem"}, + {keyfile, "path/to/key.pem"}, + {verify, verify_peer}] + }] + } + } + ])}, {rdbms_server_type, generic}, {registration_timeout, 600}, {routing_modules, mongoose_router:default_routing_modules()}, @@ -387,6 +394,7 @@ options("s2s_only") -> {listen, []}, {loglevel, warning}, {mongooseimctl_access_commands, []}, + {outgoing_pools, []}, {rdbms_server_type, generic}, {registration_timeout, 600}, {routing_modules, mongoose_router:default_routing_modules()}, @@ -784,18 +792,8 @@ pgsql_access() -> register => [#{acl => all, value => allow}], s2s_shaper => [#{acl => all, value => fast}]}. -merge_with_default_pool_config(PoolIn = #{type := Type}) -> - DefaultConfig = #{opts := DefaultOpts, conn_opts := DefaultConnOpts} = default_pool_config(Type), - WpoolOptsWithDefaults = maps:merge(DefaultOpts, maps:get(opts, PoolIn, #{})), - ConnOptsWithDefaults = maps:merge(DefaultConnOpts, maps:get(conn_opts, PoolIn, #{})), - maps:merge(DefaultConfig, PoolIn#{opts => WpoolOptsWithDefaults, - conn_opts => ConnOptsWithDefaults}). - -default_pool_config(Type) -> - #{scope => global, - tag => default, - opts => default_pool_wpool_opts(Type), - conn_opts => default_pool_conn_opts(Type)}. +pool_config(PoolIn = #{type := Type}) -> + config([outgoing_pools, Type, maps:get(tag, PoolIn, default)], PoolIn). default_pool_wpool_opts(cassandra) -> #{workers => 20, @@ -814,22 +812,25 @@ default_wpool_opts() -> call_timeout => 5000}. default_pool_conn_opts(cassandra) -> - #{servers => [{"localhost", 9042}], + #{servers => [#{host => "localhost", port => 9042}], keyspace => mongooseim}; default_pool_conn_opts(elastic) -> - #{host => "localhost", + #{host => <<"localhost">>, port => 9200}; default_pool_conn_opts(http) -> - #{path_prefix => "/", + #{path_prefix => <<"/">>, request_timeout => 2000}; default_pool_conn_opts(ldap) -> - #{rootdn => <<"">>, - password => <<"">>, - encrypt => none, + #{root_dn => <<>>, + password => <<>>, + port => 389, servers => ["localhost"], connect_interval => 10000}; default_pool_conn_opts(rabbit) -> - #{amqp_port => 5672, + #{host => "localhost", + port => 5672, + username => <<"guest">>, + password => <<"guest">>, confirms_enabled => false, max_worker_queue_len => 1000}; default_pool_conn_opts(redis) -> @@ -837,6 +838,8 @@ default_pool_conn_opts(redis) -> port => 6379, database => 0, password => ""}; +default_pool_conn_opts(rdbms) -> + #{max_start_interval => 30}; default_pool_conn_opts(_Type) -> #{}. @@ -1234,6 +1237,16 @@ default_config([modules, mod_vcard, ldap]) -> % included when backend => ldap {<<"Organization Unit">>, <<"ORGUNIT">>}], search_operator => 'and', binary_search_fields => []}; +default_config([outgoing_pools, Type, Tag] = P) -> + #{type => Type, + tag => Tag, + scope => global, + opts => default_config(P ++ [opts]), + conn_opts => default_config(P ++ [conn_opts])}; +default_config([outgoing_pools, Type, _Tag, opts]) -> + default_pool_wpool_opts(Type); +default_config([outgoing_pools, Type, _Tag, conn_opts]) -> + default_pool_conn_opts(Type); default_config([services, service_admin_extra]) -> #{submods => [node, accounts, sessions, vcard, roster, last, private, stanza, stats, gdpr, upload, domain]}; @@ -1243,7 +1256,9 @@ default_config([services, service_domain_db]) -> db_pool => global}; default_config([services, service_mongoose_system_metrics]) -> #{initial_report => timer:minutes(5), - periodic_report => timer:hours(3)}. + periodic_report => timer:hours(3)}; +default_config(Path) when is_list(Path) -> + #{}. common_mam_config() -> #{no_stanzaid_element => false, @@ -1262,4 +1277,12 @@ mod_event_pusher_http_handler() -> callback_module => mod_event_pusher_http_defaults}. config(Path, Opts) -> + config(Path, Opts, deep). + +config(Path, Opts, deep) -> + Opts1 = maps:map(fun(Key, Value) when is_map(Value) -> config(Path ++ [Key], Value, deep); + (_Key, Value) -> Value + end, Opts), + config(Path, Opts1, shallow); +config(Path, Opts, shallow) -> maps:merge(default_config(Path), Opts). diff --git a/test/config_parser_SUITE.erl b/test/config_parser_SUITE.erl index 7041a96499..f55dbad058 100644 --- a/test/config_parser_SUITE.erl +++ b/test/config_parser_SUITE.erl @@ -34,7 +34,7 @@ end). -import(mongoose_config_parser_toml, [extract_errors/1]). --import(config_parser_helper, [merge_with_default_pool_config/1, default_s2s/0, +-import(config_parser_helper, [default_s2s/0, extra_service_listener_config/0, mod_event_pusher_http_handler/0, mod_config/2, default_mod_config/1, @@ -120,41 +120,31 @@ groups() -> auth_riak_bucket_type, auth_rdbms_users_number_estimate, auth_dummy]}, - {pool, [parallel], [pool_type, - pool_tag, + {pool, [parallel], [pool_basics, pool_scope, - pool_workers, - pool_strategy, - pool_call_timeout, - pool_rdbms_settings, - pool_rdbms_keepalive_interval, - pool_rdbms_server, - pool_rdbms_port, - pool_rdbms_tls, - pool_http_host, - pool_http_path_prefix, - pool_http_request_timeout, - pool_http_tls, - pool_redis_host, - pool_redis_port, - pool_redis_database, - pool_redis_password, - pool_riak_address, - pool_riak_port, - pool_riak_credentials, - pool_riak_cacertfile, - pool_riak_tls, - pool_cassandra_servers, - pool_cassandra_keyspace, - pool_cassandra_auth, - pool_cassandra_tls, - pool_ldap_port, - pool_ldap_servers, - pool_ldap_encrypt, - pool_ldap_rootdn, - pool_ldap_password, - pool_ldap_connect_interval, - pool_ldap_tls]}, + pool_rdbms, + pool_rdbms_connection_odbc, + pool_rdbms_connection_pgsql, + pool_rdbms_connection_mysql, + pool_http, + pool_http_connection, + pool_http_connection_tls, + pool_redis, + pool_redis_connection, + pool_riak, + pool_riak_connection, + pool_riak_connection_credentials, + pool_riak_connection_tls, + pool_cassandra, + pool_cassandra_connection, + pool_cassandra_connection_auth_plain, + pool_cassandra_connection_servers, + pool_elastic, + pool_elastic_connection, + pool_rabbit, + pool_rabbit_connection, + pool_ldap, + pool_ldap_connection]}, {shaper_acl_access, [parallel], [shaper, acl, acl_merge_host_and_global, @@ -926,351 +916,314 @@ auth_dummy(_Config) -> %% tests: outgoing_pools -pool_type(_Config) -> - ?cfg(pool_config(#{type => http}), - pool_raw(<<"http">>, <<"default">>, #{})), - ?err(pool_raw(<<"swimming_pool">>, <<"default">>, #{})). - -pool_tag(_Config) -> - ?cfg(pool_config(#{type => http, tag => my_pool}), - pool_raw(<<"http">>, <<"my_pool">>, #{})), - ?err(pool_raw(<<"http">>, 1000, #{})). +pool_basics(_Config) -> + P = [outgoing_pools, 1], + Required = #{<<"connection">> => #{<<"host">> => <<"http://localhost">>}}, + ?cfg(P ++ [type], http, pool_raw(<<"http">>, <<"default">>, Required)), + ?cfg(P ++ [tag], default, pool_raw(<<"http">>, <<"default">>, Required)), + ?err(pool_raw(<<"swimming_pool">>, <<"default">>, Required)), + ?err(pool_raw(<<"http">>, 1000, Required)). pool_scope(_Config) -> - ?cfg(pool_config(#{type => http, scope => host}), - pool_raw(<<"http">>, <<"default">>, #{<<"scope">> => <<"host">>})), - ?cfg(pool_config(#{type => http, scope => <<"localhost">>}), - pool_raw(<<"http">>, <<"default">>, #{<<"scope">> => <<"single_host">>, - <<"host">> => <<"localhost">>})), - ?err(pool_raw(<<"http">>, <<"default">>, #{<<"scope">> => <<"whatever">>})), - ?err(pool_raw(<<"http">>, <<"default">>, #{<<"scope">> => <<"single_host">>})). - -pool_workers(_Config) -> - ?cfg(pool_config(#{type => http, opts => #{workers => 11}}), - pool_raw(<<"http">>, <<"default">>, #{<<"workers">> => 11})), - ?err(pool_raw(<<"http">>, <<"default">>, #{<<"workers">> => 0})). - -pool_strategy(_Config) -> - ?cfg(pool_config(#{type => http, opts => #{strategy => random_worker}}), - pool_raw(<<"http">>, <<"default">>, #{<<"strategy">> => <<"random_worker">>})), - ?err(pool_raw(<<"http">>, <<"default">>, #{<<"strategy">> => <<"worst_worker">>})). - -pool_call_timeout(_Config) -> - ?cfg(pool_config(#{type => http, opts => #{call_timeout => 999}}), - pool_raw(<<"http">>, <<"default">>, #{<<"call_timeout">> => 999})), - ?err(pool_raw(<<"http">>, <<"default">>, #{<<"call_timeout">> => 0})). - -pool_rdbms_settings(_Config) -> - ?cfg(pool_config(#{type => rdbms, conn_opts => #{server => "DSN=mydb"}}), - pool_conn_raw(<<"rdbms">>, #{<<"driver">> => <<"odbc">>, - <<"settings">> => <<"DSN=mydb">>})), - ?err(pool_conn_raw(<<"rdbms">>, #{<<"driver">> => <<"mysql">>, - <<"settings">> => <<"DSN=mydb">>})), - ?err(pool_conn_raw(<<"rdbms">>, #{<<"driver">> => <<"odbc">>, - <<"settings">> => true})), - ?err(pool_conn_raw(<<"rdbms">>, #{<<"driver">> => <<"odbc">>})). - -pool_rdbms_keepalive_interval(_Config) -> - ?cfg(pool_config(#{type => rdbms, conn_opts => #{server => "DSN=mydb", - keepalive_interval => 1000}}), - pool_conn_raw(<<"rdbms">>, #{<<"driver">> => <<"odbc">>, - <<"settings">> => <<"DSN=mydb">>, - <<"keepalive_interval">> => 1000})), - ?err(pool_conn_raw(<<"rdbms">>, #{<<"driver">> => <<"odbc">>, - <<"settings">> => <<"DSN=mydb">>, - <<"keepalive_interval">> => false})). - -pool_rdbms_server(_Config) -> - ServerOpts = rdbms_opts(), - ?cfg(pool_config(#{type => rdbms, - conn_opts => #{server => {pgsql, "localhost", "db", "dbuser", "secret"}}}), - pool_conn_raw(<<"rdbms">>, ServerOpts)), - ?err(pool_conn_raw(<<"rdbms">>, ServerOpts#{<<"driver">> := <<"odbc">>})), - [?err(pool_conn_raw(<<"rdbms">>, maps:without([K], ServerOpts))) || - K <- maps:keys(ServerOpts)], - [?err(pool_conn_raw(<<"rdbms">>, ServerOpts#{K := 123})) || - K <- maps:keys(ServerOpts)]. - -pool_rdbms_port(_Config) -> - ServerOpts = rdbms_opts(), - ?cfg(pool_config(#{type => rdbms, - conn_opts => #{server => {pgsql, "localhost", 1234, "db", "dbuser", "secret"}}}), - pool_conn_raw(<<"rdbms">>, ServerOpts#{<<"port">> => 1234})), - ?err(pool_conn_raw(<<"rdbms">>, ServerOpts#{<<"port">> => <<"airport">>})). - -pool_rdbms_tls(_Config) -> - ServerOpts = rdbms_opts(), - ?cfg(pool_config(#{type => rdbms, - conn_opts => #{server => {pgsql, "localhost", "db", "dbuser", "secret", - [{ssl, required}]}}}), - pool_conn_raw(<<"rdbms">>, ServerOpts#{<<"tls">> => #{<<"required">> => true}})), - ?cfg(pool_config(#{type => rdbms, - conn_opts => #{server => {pgsql, "localhost", "db", "dbuser", "secret", - [{ssl, true}]}}}), - pool_conn_raw(<<"rdbms">>, ServerOpts#{<<"tls">> => #{}})), - ?cfg(pool_config(#{type => rdbms, - conn_opts => #{server => {mysql, "localhost", "db", "dbuser", "secret", []}}}), - pool_conn_raw(<<"rdbms">>, ServerOpts#{<<"driver">> => <<"mysql">>, - <<"tls">> => #{}})), - ?cfg(pool_config(#{type => rdbms, - conn_opts => #{server => {pgsql, "localhost", 1234, "db", "dbuser", "secret", - [{ssl, true}]}}}), - pool_conn_raw(<<"rdbms">>, ServerOpts#{<<"tls">> => #{}, - <<"port">> => 1234})), - + P = [outgoing_pools, 1, scope], + Required = #{<<"connection">> => #{<<"host">> => <<"http://localhost">>}}, + T = fun(Opts) -> pool_raw(<<"http">>, <<"default">>, maps:merge(Required, Opts)) end, + ?cfg(P, host, T(#{<<"scope">> => <<"host">>})), + ?cfg(P, <<"localhost">>, T(#{<<"scope">> => <<"single_host">>, <<"host">> => <<"localhost">>})), + ?err(T(#{<<"host">> => <<"localhost">>})), % missing scope + ?err(T(#{<<"scope">> => <<"single_host">>})), % missing host + ?err(T(#{<<"scope">> => <<"whatever">>})). + +pool_rdbms(_Config) -> + test_pool_opts(rdbms, #{<<"connection">> => raw_sql_opts(pgsql)}). + +pool_rdbms_connection_odbc(_Config) -> + P = [outgoing_pools, 1, conn_opts], + Required = #{<<"driver">> => <<"odbc">>, <<"settings">> => <<"DSN=mydb">>}, + T = fun(Opts) -> pool_conn_raw(<<"rdbms">>, Opts) end, + test_pool_rdbms_connection_common_opts(P, T, Required), + ?cfg(P, config([outgoing_pools, rdbms, default, conn_opts], + #{driver => odbc, settings => "DSN=mydb"}), T(Required)), + ?err(T(Required#{<<"settings">> => true})), + [?err(T(maps:remove(K, Required))) || K <- maps:keys(Required)]. + +pool_rdbms_connection_pgsql(_Config) -> + P = [outgoing_pools, 1, conn_opts], + T = fun(Opts) -> pool_conn_raw(<<"rdbms">>, Opts) end, + Required = raw_sql_opts(pgsql), + test_pool_rdbms_connection_common_opts(P, T, Required), + test_pool_rdbms_connection_sql_opts(P, T, Required, sql_opts(pgsql, 5432)), + ?cfg(P ++ [tls], [{required, true}], T(Required#{<<"tls">> => #{<<"required">> => true}})), + ?cfg(P ++ [tls], [], T(Required#{<<"tls">> => #{}})), %% one option tested here as they are all checked by 'listen_tls_*' tests - ?cfg(pool_config(#{type => rdbms, - conn_opts => #{server => {pgsql, "localhost", "db", "dbuser", "secret", - [{ssl, true}, {ssl_opts, [{certfile, "cert.pem"}]}]}}}), - pool_conn_raw(<<"rdbms">>, ServerOpts#{<<"tls">> => - #{<<"certfile">> => <<"cert.pem">>}})), - ?err(pool_conn_raw(<<"rdbms">>, ServerOpts#{<<"tls">> => - #{<<"certfile">> => true}})), - ?err(pool_conn_raw(<<"rdbms">>, ServerOpts#{<<"tls">> => <<"secure">>})). - -pool_http_host(_Config) -> - ?cfg(pool_config(#{type => http, conn_opts => #{server => "https://localhost:8443"}}), - pool_conn_raw(<<"http">>, #{<<"host">> => <<"https://localhost:8443">>})), - ?err(pool_conn_raw(<<"http">>, #{<<"host">> => 8443})), - ?err(pool_conn_raw(<<"http">>, #{<<"host">> => ""})). - -pool_http_path_prefix(_Config) -> - ?cfg(pool_config(#{type => http, conn_opts => #{path_prefix => "/my_path/"}}), - pool_conn_raw(<<"http">>, #{<<"path_prefix">> => <<"/my_path/">>})), - ?err(pool_conn_raw(<<"http">>, #{<<"path_prefix">> => 8443})), - ?err(pool_conn_raw(<<"http">>, #{<<"path_prefix">> => ""})). - -pool_http_request_timeout(_Config) -> - ?cfg(pool_config(#{type => http, conn_opts => #{request_timeout => 999}}), - pool_conn_raw(<<"http">>, #{<<"request_timeout">> => 999})), - ?err(pool_conn_raw(<<"http">>, #{<<"request_timeout">> => -1000})), - ?err(pool_conn_raw(<<"http">>, #{<<"request_timeout">> => <<"infinity">>})). - -pool_http_tls(_Config) -> - ?cfg(pool_config(#{type => http, conn_opts => #{http_opts => [{certfile, "cert.pem"}]}}), - pool_conn_raw(<<"http">>, #{<<"tls">> => #{<<"certfile">> => <<"cert.pem">>}})), - ?cfg(pool_config(#{type => http, - conn_opts => #{http_opts => [{certfile, "cert.pem"}, - {verify, verify_peer}, - {cacertfile, "priv/ca.pem"}, - {server_name_indication, disable}]}}), - pool_conn_raw(<<"http">>, #{<<"tls">> => #{<<"certfile">> => <<"cert.pem">>, - <<"verify_peer">> => true, - <<"cacertfile">> => <<"priv/ca.pem">>, - <<"server_name_indication">> => false}})), - ?cfg(pool_config(#{type => http, - conn_opts => #{http_opts => [{certfile, "cert.pem"}, - {verify, verify_peer}, - {cacertfile, "priv/ca.pem"}, - {server_name_indication, "domain.com"}]}}), - pool_conn_raw(<<"http">>, #{<<"tls">> => #{<<"certfile">> => <<"cert.pem">>, - <<"verify_peer">> => true, - <<"cacertfile">> => <<"priv/ca.pem">>, - <<"server_name_indication">> => true, - <<"server_name_indication_host">> => <<"domain.com">>}})), - ?cfg(pool_config(#{type => http, - conn_opts => #{http_opts => [{certfile, "cert.pem"}, - {verify, verify_peer}, - {cacertfile, "priv/ca.pem"}, - {server_name_indication, "domain.com"}, - {customize_hostname_check, - [{match_fun, public_key:pkix_verify_hostname_match_fun(https)}]}]}}), - pool_conn_raw(<<"http">>, #{<<"tls">> => #{<<"certfile">> => <<"cert.pem">>, - <<"verify_peer">> => true, - <<"cacertfile">> => <<"priv/ca.pem">>, - <<"server_name_indication">> => true, - <<"server_name_indication_host">> => <<"domain.com">>, - <<"server_name_indication_protocol">> => <<"https">>}})), - ?cfg(pool_config(#{type => http, - conn_opts => #{http_opts => [{verify, verify_peer}, - {cacertfile, "priv/ca.pem"}]}}), - pool_conn_raw(<<"http">>, #{<<"tls">> => #{<<"verify_peer">> => true, - <<"cacertfile">> => <<"priv/ca.pem">>}})), - ?err(pool_conn_raw(<<"http">>, #{<<"tls">> => #{<<"verify_peer">> => true, - <<"cacertfile">> => <<"priv/ca.pem">>, - <<"server_name_indication">> => <<"domain.com">>, - <<"server_name_indication_host">> => <<"domain.com">>}})), - ?err(pool_conn_raw(<<"http">>, #{<<"tls">> => #{<<"verify_peer">> => true, - <<"cacertfile">> => <<"priv/ca.pem">>, - <<"server_name_indication">> => <<"true">>, - <<"server_name_indication_host">> => <<"domain.com">>, - <<"server_name_indication_protocol">> => <<"non_value">>}})), - ?err(pool_conn_raw(<<"http">>, #{<<"tls">> => #{<<"certfile">> => true}})), - ?err(pool_conn_raw(<<"http">>, #{<<"tls">> => <<"secure">>})). - -pool_redis_host(_Config) -> - ?cfg(pool_config(#{type => redis, conn_opts => #{host => "my_host"}}), - pool_conn_raw(<<"redis">>, #{<<"host">> => <<"my_host">>})), - ?err(pool_conn_raw(<<"redis">>, #{<<"host">> => 8443})), - ?err(pool_conn_raw(<<"redis">>, #{<<"host">> => ""})). - -pool_redis_port(_Config) -> - ?cfg(pool_config(#{type => redis, conn_opts => #{port => 9999}}), - pool_conn_raw(<<"redis">>, #{<<"port">> => 9999})), - ?err(pool_conn_raw(<<"redis">>, #{<<"port">> => 666666})), - ?err(pool_conn_raw(<<"redis">>, #{<<"port">> => <<"airport">>})). - -pool_redis_database(_Config) -> - ?cfg(pool_config(#{type => redis, conn_opts => #{database => 1}}), - pool_conn_raw(<<"redis">>, #{<<"database">> => 1})), - ?err(pool_conn_raw(<<"redis">>, #{<<"database">> => -1})), - ?err(pool_conn_raw(<<"redis">>, #{<<"database">> => <<"my_database">>})). - -pool_redis_password(_Config) -> - ?cfg(pool_config(#{type => redis, conn_opts => #{password => "password1"}}), - pool_conn_raw(<<"redis">>, #{<<"password">> => <<"password1">>})), - ?err(pool_conn_raw(<<"redis">>, #{<<"password">> => 0})). - -pool_riak_address(_Config) -> - ?cfg(pool_config(#{type => riak, conn_opts => #{address => "127.0.0.1"}}), - pool_conn_raw(<<"riak">>, #{<<"address">> => <<"127.0.0.1">>})), - ?err(pool_conn_raw(<<"riak">>, #{<<"address">> => 66})), - ?err(pool_conn_raw(<<"riak">>, #{<<"address">> => <<"">>})). - -pool_riak_port(_Config) -> - ?cfg(pool_config(#{type => riak, conn_opts => #{port => 8087}}), - pool_conn_raw(<<"riak">>, #{<<"port">> => 8087})), - ?err(pool_conn_raw(<<"riak">>, #{<<"port">> => 666666})), - ?err(pool_conn_raw(<<"riak">>, #{<<"port">> => <<"airport">>})). - -pool_riak_credentials(_Config) -> - ?cfg(pool_config(#{type => riak, conn_opts => #{credentials => {"user", "pass"}}}), - pool_conn_raw(<<"riak">>, #{<<"credentials">> => - #{<<"user">> => <<"user">>, <<"password">> => <<"pass">>}})), - ?err(pool_conn_raw(<<"riak">>, #{<<"credentials">> => #{<<"user">> => <<"user">>}})), - ?err(pool_conn_raw(<<"riak">>, #{<<"credentials">> => - #{<<"user">> => <<"">>, <<"password">> => 011001}})). - -pool_riak_cacertfile(_Config) -> - ?cfg(pool_config(#{type => riak, conn_opts => #{cacertfile => "cacert.pem"}}), - pool_conn_raw(<<"riak">>, #{<<"tls">> => #{<<"cacertfile">> => <<"cacert.pem">>}})), - ?err(pool_conn_raw(<<"riak">>, #{<<"cacertfile">> => <<"">>})). - -pool_riak_tls(_Config) -> - %% make sure these options are not extracted out of 'ssl_opts' + ?cfg(P ++ [tls], [{certfile, "cert.pem"}], + T(Required#{<<"tls">> => #{<<"certfile">> => <<"cert.pem">>}})), + ?err(T(Required#{<<"tls">> => #{<<"required">> => <<"maybe">>}})). + +pool_rdbms_connection_mysql(_Config) -> + P = [outgoing_pools, 1, conn_opts], + T = fun(Opts) -> pool_conn_raw(<<"rdbms">>, Opts) end, + Required = raw_sql_opts(mysql), + test_pool_rdbms_connection_common_opts(P, T, Required), + test_pool_rdbms_connection_sql_opts(P, T, Required, sql_opts(mysql, 3306)), + ?cfg(P ++ [tls], [], T(Required#{<<"tls">> => #{}})). + +test_pool_rdbms_connection_sql_opts(P, T, Required, Expected) -> + ?cfg(P, config([outgoing_pools, rdbms, default, conn_opts], Expected), T(Required)), + ?cfg(P ++ [port], 1234, T(Required#{<<"port">> => 1234})), + ?err(T(Required#{<<"host">> => <<>>})), + ?err(T(Required#{<<"port">> => -1})), + ?err(T(Required#{<<"database">> => <<>>})), + ?err(T(Required#{<<"username">> => <<>>})), + ?err(T(Required#{<<"password">> => <<>>})). + +test_pool_rdbms_connection_common_opts(P, T, Required) -> + ?cfg(P ++ [keepalive_interval], 100, T(Required#{<<"keepalive_interval">> => 100})), + ?cfg(P ++ [max_start_interval], 200, T(Required#{<<"max_start_interval">> => 200})), + ?err(T(Required#{<<"keepalive_interval">> => 0})), + ?err(T(Required#{<<"max_start_interval">> => 0})), + [?err(T(maps:remove(K, Required))) || K <- maps:keys(Required)]. + +raw_sql_opts(Driver) -> + #{<<"driver">> => atom_to_binary(Driver), + <<"host">> => <<"localhost">>, + <<"database">> => <<"db">>, + <<"username">> => <<"dbuser">>, + <<"password">> => <<"secret">>}. + +sql_opts(Driver, Port) -> + #{driver => Driver, + host => "localhost", + port => Port, + database => "db", + username => "dbuser", + password => "secret"}. + +pool_http(_Config) -> + test_pool_opts(http, #{<<"connection">> => #{<<"host">> => <<"https://localhost:8443">>}}). + +pool_http_connection(_Config) -> + P = [outgoing_pools, 1, conn_opts], + T = fun(Opts) -> pool_conn_raw(<<"http">>, Opts) end, + Required = #{<<"host">> => <<"https://localhost:8443">>}, + ?cfg(P, config([outgoing_pools, http, default, conn_opts], #{host => "https://localhost:8443"}), + T(Required)), + ?cfg(P ++ [path_prefix], <<"/my_path/">>, T(Required#{<<"path_prefix">> => <<"/my_path/">>})), + ?cfg(P ++ [request_timeout], 999, T(Required#{<<"request_timeout">> => 999})), + ?err(T(#{})), + ?err(T(#{<<"host">> => <<>>})), + ?err(T(Required#{<<"path_prefix">> => <<>>})), + ?err(T(Required#{<<"request_timeout">> => -1000})). + +pool_http_connection_tls(_Config) -> + P = [outgoing_pools, 1, conn_opts, tls], + T = fun(Opts) -> pool_conn_raw(<<"http">>, #{<<"host">> => <<"http://localhost">>, + <<"tls">> => Opts}) + end, + ?cfg(P, [{certfile, "cert.pem"}], T(#{<<"certfile">> => <<"cert.pem">>})), + ?cfg(P, [{certfile, "cert.pem"}, + {verify, verify_peer}, + {cacertfile, "priv/ca.pem"}, + {server_name_indication, disable}], + T(#{<<"certfile">> => <<"cert.pem">>, + <<"verify_peer">> => true, + <<"cacertfile">> => <<"priv/ca.pem">>, + <<"server_name_indication">> => false})), + ?cfg(P, [{certfile, "cert.pem"}, + {verify, verify_peer}, + {cacertfile, "priv/ca.pem"}, + {server_name_indication, "domain.com"}], + T(#{<<"certfile">> => <<"cert.pem">>, + <<"verify_peer">> => true, + <<"cacertfile">> => <<"priv/ca.pem">>, + <<"server_name_indication">> => true, + <<"server_name_indication_host">> => <<"domain.com">>})), + ?cfg(P, [{certfile, "cert.pem"}, + {verify, verify_peer}, + {cacertfile, "priv/ca.pem"}, + {server_name_indication, "domain.com"}, + {customize_hostname_check, + [{match_fun, public_key:pkix_verify_hostname_match_fun(https)}]}], + T(#{<<"certfile">> => <<"cert.pem">>, + <<"verify_peer">> => true, + <<"cacertfile">> => <<"priv/ca.pem">>, + <<"server_name_indication">> => true, + <<"server_name_indication_host">> => <<"domain.com">>, + <<"server_name_indication_protocol">> => <<"https">>})), + ?cfg(P, [{verify, verify_peer}, + {cacertfile, "priv/ca.pem"}], + T(#{<<"verify_peer">> => true, + <<"cacertfile">> => <<"priv/ca.pem">>})), + ?err(T(#{<<"verify_peer">> => true, + <<"cacertfile">> => <<"priv/ca.pem">>, + <<"server_name_indication">> => <<"domain.com">>, + <<"server_name_indication_host">> => <<"domain.com">>})), + ?err(T(#{<<"verify_peer">> => true, + <<"cacertfile">> => <<"priv/ca.pem">>, + <<"server_name_indication">> => <<"true">>, + <<"server_name_indication_host">> => <<"domain.com">>, + <<"server_name_indication_protocol">> => <<"non_value">>})), + ?err(T(#{<<"certfile">> => true})), + ?err(T(<<"secure">>)). + +pool_redis(_Config) -> + test_pool_opts(redis, #{}). + +pool_redis_connection(_Config) -> + P = [outgoing_pools, 1, conn_opts], + T = fun(Opts) -> pool_conn_raw(<<"redis">>, Opts) end, + ?cfg(P, default_config([outgoing_pools, redis, default, conn_opts]), T(#{})), + ?cfg(P ++ [host], "my_host", T(#{<<"host">> => <<"my_host">>})), + ?cfg(P ++ [port], 9999, T(#{<<"port">> => 9999})), + ?cfg(P ++ [database], 1, T(#{<<"database">> => 1})), + ?cfg(P ++ [password], "password1", T(#{<<"password">> => <<"password1">>})), + ?err(T(#{<<"host">> => 8443})), + ?err(T(#{<<"port">> => 666666})), + ?err(T(#{<<"database">> => -1})), + ?err(T(#{<<"password">> => 0})). + +pool_riak(_Config) -> + test_pool_opts(riak, #{<<"connection">> => required_riak_connection_opts()}). + +pool_riak_connection(_Config) -> + P = [outgoing_pools, 1, conn_opts], + T = fun(Opts) -> pool_conn_raw(<<"riak">>, Opts) end, + Required = required_riak_connection_opts(), + ?cfg(P ++ [address], "127.0.0.1", T(Required)), + ?cfg(P ++ [port], 8087, T(Required)), + [?err(T(maps:remove(K, Required))) || K <- maps:keys(Required)], + ?err(T(Required#{<<"address">> => 8443})), + ?err(T(Required#{<<"port">> => 666666})). + +pool_riak_connection_credentials(_Config) -> + P = [outgoing_pools, 1, conn_opts, credentials], + T = fun(Opts) -> pool_conn_raw(<<"riak">>, + (required_riak_connection_opts())#{<<"credentials">> => Opts}) + end, + Required = #{<<"user">> => <<"user">>, <<"password">> => <<"pass">>}, + ?cfg(P, #{user => "user", password => "pass"}, T(Required)), + [?err(T(maps:remove(K, Required))) || K <- maps:keys(Required)], + [?err(T(Required#{K => <<>>})) || K <- maps:keys(Required)]. + +pool_riak_connection_tls(_Config) -> + P = [outgoing_pools, 1, conn_opts, tls], + T = fun(Opts) -> pool_conn_raw(<<"riak">>, + (required_riak_connection_opts())#{<<"tls">> => Opts}) + end, + %% make sure 'certfile' is extracted out of 'ssl_opts' %% all the TLS options are checked by 'listen_tls_*' tests - ?cfg(pool_config(#{type => riak, - conn_opts => #{ssl_opts => [{certfile, "path/to/cert.pem"}, - {dhfile, "cert.pem"}, - {keyfile, "path/to/key.pem"}]}}), - pool_conn_raw(<<"riak">>, #{<<"tls">> => #{<<"certfile">> => <<"path/to/cert.pem">>, - <<"dhfile">> => <<"cert.pem">>, - <<"keyfile">> => <<"path/to/key.pem">>}})), - ?err(pool_conn_raw(<<"riak">>, #{<<"tls">> => #{<<"dhfile">> => true}})), - ?err(pool_conn_raw(<<"riak">>, #{<<"tls">> => <<"secure">>})). - -pool_cassandra_servers(_Config) -> - ?cfg(pool_config(#{type => cassandra, - conn_opts => #{servers => [{"cassandra_server1.example.com", 9042}, - {"cassandra_server2.example.com", 9042}]}}), - pool_conn_raw(<<"cassandra">>, - #{<<"servers">> => [#{<<"ip_address">> => <<"cassandra_server1.example.com">>, - <<"port">> => 9042}, - #{<<"ip_address">> => <<"cassandra_server2.example.com">>, - <<"port">> => 9042}]})), - ?err(pool_conn_raw(<<"cassandra">>, - #{<<"servers">> => #{<<"ip_address">> => <<"cassandra_server1.example.com">>, - <<"port">> => 9042}})). - -pool_cassandra_keyspace(_Config) -> - ?cfg(pool_config(#{type => cassandra, conn_opts => #{keyspace => big_mongooseim}}), - pool_conn_raw(<<"cassandra">>, #{<<"keyspace">> => <<"big_mongooseim">>})), - ?err(pool_conn_raw(<<"cassandra">>, #{<<"keyspace">> => <<"">>})). - -pool_cassandra_auth(_Config) -> - ?cfg(pool_config(#{type => cassandra, - conn_opts => #{auth => {cqerl_auth_plain_handler, - [{<<"auser">>, <<"secretpass">>}]}}}), - pool_conn_raw(<<"cassandra">>, - #{<<"auth">> => #{<<"plain">> => #{<<"username">> => <<"auser">>, - <<"password">> => <<"secretpass">>}}})), - ?err(pool_conn_raw(<<"cassandra">>, #{<<"tls">> => #{<<"verify">> => <<"verify_none">>}})). - -pool_cassandra_tls(_Config) -> - %% one option tested here as they are all checked by 'listen_tls_*' tests - ?cfg(pool_config(#{type => cassandra, conn_opts => #{ssl => [{verify, verify_none}]}}), - pool_conn_raw(<<"cassandra">>, #{<<"tls">> => #{<<"verify_peer">> => false}})), - ?err(pool_conn_raw(<<"cassandra">>, #{<<"tls">> => #{<<"verify">> => <<"verify_none">>}})). - -pool_elastic_host(_Config) -> - ?cfg(pool_config(#{type => elastic, conn_opts => #{host => "my_host"}}), - pool_conn_raw(<<"elastic">>, #{<<"host">> => <<"my_host">>})), - ?err(pool_conn_raw(<<"elastic">>, #{<<"host">> => <<"">>})). - -pool_elastic_port(_Config) -> - ?cfg(pool_config(#{type => elastic, conn_opts => #{port => 9999}}), - pool_conn_raw(<<"elastic">>, #{<<"port">> => 9999})), - ?err(pool_conn_raw(<<"elastic">>, #{<<"port">> => 122333})), - ?err(pool_conn_raw(<<"elastic">>, #{<<"port">> => <<"airport">>})). - -pool_rabbit_amqp_host(_Config) -> - ?cfg(pool_config(#{type => rabbit, conn_opts => #{amqp_host => "localhost"}}), - pool_conn_raw(<<"rabbit">>, #{<<"amqp_host">> => <<"localhost">>})), - ?err(pool_conn_raw(<<"rabbit">>, #{<<"amqp_host">> => <<"">>})). - -pool_rabbit_amqp_port(_Config) -> - ?cfg(pool_config(#{type => rabbit, conn_opts => #{amqp_port => 5672}}), - pool_conn_raw(<<"rabbit">>, #{<<"amqp_port">> => 5672})), - ?err(pool_conn_raw(<<"rabbit">>, #{<<"amqp_port">> => <<"airport">>})). - -pool_rabbit_amqp_username(_Config) -> - ?cfg(pool_config(#{type => rabbit, conn_opts => #{amqp_username => "guest"}}), - pool_conn_raw(<<"rabbit">>, #{<<"amqp_username">> => <<"guest">>})), - ?err(pool_conn_raw(<<"rabbit">>, #{<<"amqp_username">> => <<"">>})). - -pool_rabbit_amqp_password(_Config) -> - ?cfg(pool_config(#{type => rabbit, conn_opts => #{amqp_password => "guest"}}), - pool_conn_raw(<<"rabbit">>, #{<<"amqp_password">> => <<"guest">>})), - ?err(pool_conn_raw(<<"rabbit">>, #{<<"amqp_password">> => <<"">>})). - -pool_rabbit_amqp_confirms_enabled(_Config) -> - ?cfg(pool_config(#{type => rabbit, conn_opts => #{confirms_enabled => true}}), - pool_conn_raw(<<"rabbit">>, #{<<"confirms_enabled">> => true})), - ?err(pool_conn_raw(<<"rabbit">>, #{<<"confirms_enabled">> => <<"yes">>})). - -pool_rabbit_amqp_max_worker_queue_len(_Config) -> - ?cfg(pool_config(#{type => rabbit, conn_opts => #{max_worker_queue_len => 100}}), - pool_conn_raw(<<"rabbit">>, #{<<"max_worker_queue_len">> => 100})), - ?err(pool_conn_raw(<<"rabbit">>, #{<<"max_worker_queue_len">> => 0})). - -pool_ldap_port(_Config) -> - ?cfg(pool_config(#{type => ldap, conn_opts => #{port => 389}}), - pool_conn_raw(<<"ldap">>, #{<<"port">> => 389})), - ?err(pool_conn_raw(<<"ldap">>, #{<<"port">> => <<"airport">>})). - -pool_ldap_servers(_Config) -> - ?cfg(pool_config(#{type => ldap, - conn_opts => #{servers => ["primary-ldap-server.example.com", - "secondary-ldap-server.example.com"]}}), - pool_conn_raw(<<"ldap">>, #{<<"servers">> => [<<"primary-ldap-server.example.com">>, - <<"secondary-ldap-server.example.com">>]})), - ?err(pool_conn_raw(<<"ldap">>, #{<<"servers">> => #{<<"server">> => <<"example.com">>}})). - -pool_ldap_encrypt(_Config) -> - ?cfg(pool_config(#{type => ldap, conn_opts => #{encrypt => tls}}), - pool_conn_raw(<<"ldap">>, #{<<"encrypt">> => <<"tls">>})), - ?err(pool_conn_raw(<<"ldap">>, #{<<"encrypt">> => true})). - -pool_ldap_rootdn(_Config) -> - ?cfg(pool_config(#{type => ldap, conn_opts => #{rootdn => <<"my_rootdn">>}}), - pool_conn_raw(<<"ldap">>, #{<<"rootdn">> => <<"my_rootdn">>})), - ?err(pool_conn_raw(<<"ldap">>, #{<<"rootdn">> => false})). - -pool_ldap_password(_Config) -> - ?cfg(pool_config(#{type => ldap, conn_opts => #{password => <<"pass">>}}), - pool_conn_raw(<<"ldap">>, #{<<"password">> => <<"pass">>})), - ?err(pool_conn_raw(<<"ldap">>, #{<<"password">> => true})). - -pool_ldap_connect_interval(_Config) -> - ?cfg(pool_config(#{type => ldap, conn_opts => #{connect_interval => 9999}}), - pool_conn_raw(<<"ldap">>, #{<<"connect_interval">> => 9999})), - ?err(pool_conn_raw(<<"ldap">>, #{<<"connect_interval">> => <<"infinity">>})). - -pool_ldap_tls(_Config) -> + ?cfg(P, [{cacertfile, "cacert.pem"}], T(#{<<"cacertfile">> => <<"cacert.pem">>})), + ?cfg(P, [{ssl_opts, [{certfile, "path/to/cert.pem"}, + {dhfile, "cert.pem"}, + {keyfile, "path/to/key.pem"}]}], + T(#{<<"certfile">> => <<"path/to/cert.pem">>, + <<"dhfile">> => <<"cert.pem">>, + <<"keyfile">> => <<"path/to/key.pem">>})), + ?err(T(#{<<"cacertfile">> => <<>>})), + ?err(T(#{<<"certfile">> => <<>>})). + +required_riak_connection_opts() -> + #{<<"address">> => <<"127.0.0.1">>, <<"port">> => 8087}. + +pool_cassandra(_Config) -> + test_pool_opts(cassandra, #{<<"connection">> => #{}}). + +pool_cassandra_connection(_Config) -> + P = [outgoing_pools, 1, conn_opts], + T = fun(Opts) -> pool_conn_raw(<<"cassandra">>, Opts) end, + ?cfg(P, default_config([outgoing_pools, cassandra, default, conn_opts]), T(#{})), + ?cfg(P ++ [keyspace], big_mongooseim, T(#{<<"keyspace">> => <<"big_mongooseim">>})), + %% only one tls option tested here as they are all checked by 'listen_tls_*' tests + ?cfg(P ++ [tls], [{verify, verify_none}], T(#{<<"tls">> => #{<<"verify_peer">> => false}})), + ?err(T(#{<<"keyspace">> => <<>>})), + ?err(T(#{<<"tls">> => #{<<"verify_peer">> => <<"verify_pear">>}})). + +pool_cassandra_connection_auth_plain(_Config) -> + P = [outgoing_pools, 1, conn_opts, auth, plain], + T = fun(Opts) -> pool_conn_raw(<<"cassandra">>, #{<<"auth">> => #{<<"plain">> => Opts}}) end, + Required = #{<<"username">> => <<"user">>, <<"password">> => <<"pass">>}, + ?cfg(P, #{username => <<"user">>, password => <<"pass">>}, T(Required)), + [?err(T(maps:remove(K, Required))) || K <- maps:keys(Required)], + [?err(T(Required#{K => false})) || K <- maps:keys(Required)]. + +pool_cassandra_connection_servers(_Config) -> + P = [outgoing_pools, 1, conn_opts, servers], + T = fun(Servers) -> pool_conn_raw(<<"cassandra">>, #{<<"servers">> => Servers}) end, + Required = #{<<"host">> => <<"example.com">>}, + ?cfg(P, [#{host => "example.com", port => 9042}, % default port + #{host => "example.com", port => 9043}], + T([Required, Required#{<<"port">> => 9043}])), + ?err(T([Required, Required#{<<"port">> => 9042}])), % same port for both servers + ?err(T([#{}])), % missing host + ?err(T([])). % no servers + +pool_elastic(_Config) -> + test_pool_opts(elastic, #{<<"connection">> => #{}}). + +pool_elastic_connection(_Config) -> + P = [outgoing_pools, 1, conn_opts], + T = fun(Opts) -> pool_conn_raw(<<"elastic">>, Opts) end, + ?cfg(P, default_config([outgoing_pools, elastic, default, conn_opts]), T(#{})), + ?cfg(P ++ [host], <<"my_host">>, T(#{<<"host">> => <<"my_host">>})), + ?cfg(P ++ [port], 9999, T(#{<<"port">> => 9999})), + ?err(T(#{<<"host">> => <<>>})), + ?err(T(#{<<"port">> => 123456})). + +pool_rabbit(_Config) -> + test_pool_opts(rabbit, #{<<"connection">> => #{}}). + +pool_rabbit_connection(_Config) -> + P = [outgoing_pools, 1, conn_opts], + T = fun(Opts) -> pool_conn_raw(<<"rabbit">>, Opts) end, + ?cfg(P, default_config([outgoing_pools, rabbit, default, conn_opts]), T(#{})), + ?cfg(P ++ [host], "my_host", T(#{<<"host">> => <<"my_host">>})), + ?cfg(P ++ [port], 9999, T(#{<<"port">> => 9999})), + ?cfg(P ++ [username], <<"user">>, T(#{<<"username">> => <<"user">>})), + ?cfg(P ++ [password], <<"pass">>, T(#{<<"password">> => <<"pass">>})), + ?cfg(P ++ [confirms_enabled], true, T(#{<<"confirms_enabled">> => true})), + ?cfg(P ++ [max_worker_queue_len], 100, T(#{<<"max_worker_queue_len">> => 100})), + ?err(T(#{<<"host">> => <<>>})), + ?err(T(#{<<"port">> => 123456})), + ?err(T(#{<<"username">> => <<>>})), + ?err(T(#{<<"password">> => <<>>})), + ?err(T(#{<<"confirms_enabled">> => <<"yes">>})), + ?err(T(#{<<"max_worker_queue_len">> => -1})). + +pool_ldap(_Config) -> + test_pool_opts(ldap, #{<<"connection">> => #{}}). + +pool_ldap_connection(_Config) -> + P = [outgoing_pools, 1, conn_opts], + T = fun(Opts) -> pool_conn_raw(<<"ldap">>, Opts) end, + ?cfg(P, default_config([outgoing_pools, ldap, default, conn_opts]), T(#{})), + ?cfg(P ++ [servers], ["server1.example.com", "server2.example.com"], + T(#{<<"servers">> => [<<"server1.example.com">>, <<"server2.example.com">>]})), + ?cfg(P ++ [port], 999, T(#{<<"port">> => 999})), + ?cfg(P ++ [root_dn], <<"my_rootdn">>, T(#{<<"root_dn">> => <<"my_rootdn">>})), + ?cfg(P ++ [password], <<"pass">>, T(#{<<"password">> => <<"pass">>})), + ?cfg(P ++ [connect_interval], 5000, T(#{<<"connect_interval">> => 5000})), + ?cfg(P ++ [tls], [], T(#{<<"tls">> => #{}})), + ?cfg(P ++ [port], 636, T(#{<<"tls">> => #{}})), % default TLS port is different %% one option tested here as they are all checked by 'listen_tls_*' tests - ?cfg(pool_config(#{type => ldap, conn_opts => #{tls_options => [{verify, verify_peer}]}}), - pool_conn_raw(<<"ldap">>, #{<<"tls">> => #{<<"verify_peer">> => true}})), - ?err(pool_conn_raw(<<"ldap">>, #{<<"tls">> => #{<<"verify">> => <<"verify_none">>}})). + ?cfg(P ++ [tls], [{verify, verify_peer}], T(#{<<"tls">> => #{<<"verify_peer">> => true}})), + ?err(T(#{<<"servers">> => [<<"server1.example.com">>, <<"server1.example.com">>]})), + ?err(T(#{<<"servers">> => []})), + ?err(T(#{<<"port">> => 123456})), + ?err(T(#{<<"root_dn">> => 1})), + ?err(T(#{<<"password">> => true})), + ?err(T(#{<<"connect_interval">> => <<"infinity">>})), + ?err(T(#{<<"tls">> => #{<<"verify">> => <<"verify_none">>}})). + +test_pool_opts(Type, Required) -> + P = [outgoing_pools, 1, opts], + T = fun(Opts) -> pool_raw(atom_to_binary(Type), <<"default">>, Opts) end, + ?cfg(P, default_config([outgoing_pools, Type, default, opts]), T(Required)), + ?cfg(P ++ [workers], 11, T(Required#{<<"workers">> => 11})), + ?cfg(P ++ [strategy], random_worker, T(Required#{<<"strategy">> => <<"random_worker">>})), + ?cfg(P ++ [call_timeout], 999, T(Required#{<<"call_timeout">> => 999})), + ?err(T(Required#{<<"workers">> => 0})), + ?err(T(Required#{<<"strategy">> => <<"worst_worker">>})), + ?err(T(Required#{<<"call_timeout">> => 0})). %% tests: shaper, acl, access shaper(_Config) -> @@ -3073,23 +3026,12 @@ auth_raw(Method, Opts) -> %% helpers for 'pool' tests -pool_config(PoolIn) -> - Pool = merge_with_default_pool_config(PoolIn), - [{outgoing_pools, [Pool]}]. - pool_raw(Type, Tag, Opts) -> #{<<"outgoing_pools">> => #{Type => #{Tag => Opts}}}. pool_conn_raw(Type, Opts) -> #{<<"outgoing_pools">> => #{Type => #{<<"default">> => #{<<"connection">> => Opts}}}}. -rdbms_opts() -> - #{<<"driver">> => <<"pgsql">>, - <<"host">> => <<"localhost">>, - <<"database">> => <<"db">>, - <<"username">> => <<"dbuser">>, - <<"password">> => <<"secret">>}. - %% helpers for 'access' tests access_raw(RuleName, RuleSpec) -> @@ -3195,14 +3137,17 @@ handle_config_option(Opt1, Opt2) -> -spec compare_nodes(mongoose_config:key_path(), mongoose_config:value(), mongoose_config:value()) -> any(). -compare_nodes([listen], V1, V2) -> - compare_ordered_lists(V1, V2, fun handle_listener/2); -compare_nodes([outgoing_pools], V1, V2) -> - compare_unordered_lists(V1, V2, fun handle_conn_pool/2); -compare_nodes([{auth_method, _}], V1, V2) when is_atom(V1) -> - ?eq([V1], V2); -compare_nodes([{s2s_addr, _}], {_, _, _, _} = IP1, IP2) -> - ?eq(inet:ntoa(IP1), IP2); +compare_nodes([listen] = P, V1, V2) -> + compare_ordered_lists_of_nodes(P, V1, V2); +compare_nodes([listen, I, tls], V1, V2) when is_integer(I) -> + compare_unordered_lists(V1, V2); +compare_nodes([listen, I, handlers] = P, V1, V2) when is_integer(I) -> + compare_ordered_lists_of_nodes(P, V1, V2); +compare_nodes([outgoing_pools] = P, V1, V2) -> + compare_ordered_lists_of_nodes(P, V1, V2); +compare_nodes([outgoing_pools, I, conn_opts, K], V1, V2) + when is_integer(I), K =:= http_opts orelse K =:= tls -> + compare_unordered_lists(V1, V2); compare_nodes(Node, V1, V2) when is_map(V1), is_map(V2) -> compare_maps(V1, V2, fun({K1, MV1}, {K2, MV2}) -> ?eq(K1, K2), @@ -3211,45 +3156,9 @@ compare_nodes(Node, V1, V2) when is_map(V1), is_map(V2) -> compare_nodes(Node, V1, V2) -> ?eq({Node, V1}, {Node, V2}). -%% Comparisons of internal config option parts - -handle_listener(V1, V2) -> - ct:pal("Listeners: ~p~n~p", [V1,V2]), - compare_maps(V1, V2, fun handle_listener_option/2). - -handle_listener_option({tls, O1}, {tls, O2}) -> - compare_unordered_lists(O1, O2); -handle_listener_option({handlers, M1}, {handlers, M2}) -> - compare_ordered_lists(M1, M2, fun compare_maps/2); -handle_listener_option(V1, V2) -> ?eq(V1, V2). - -handle_item_with_opts({M1, O1}, {M2, O2}) -> - ?eq(M1, M2), - compare_unordered_lists(O1, O2). - -handle_conn_pool(#{type := Type1, scope := Scope1, tag := Tag1, opts := POpts1, conn_opts := COpts1}, - #{type := Type2, scope := Scope2, tag := Tag2, opts := POpts2, conn_opts := COpts2}) -> - ?eq(Type1, Type2), - ?eq(Scope1, Scope2), - ?eq(Tag1, Tag2), - compare_maps(POpts1, POpts2), - compare_maps(COpts1, COpts2, fun handle_conn_opt/2). - -handle_conn_opt({server, {D1, H1, DB1, U1, P1, O1}}, - {server, {D2, H2, DB2, U2, P2, O2}}) -> - ?eq(D1, D2), - ?eq(H1, H2), - ?eq(DB1, DB2), - ?eq(U1, U2), - ?eq(P1, P2), - compare_unordered_lists(O1, O2, fun handle_db_server_opt/2); -handle_conn_opt({http_opts, O1}, {http_opts, O2}) -> - compare_unordered_lists(O1, O2); -handle_conn_opt(V1, V2) -> ?eq(V1, V2). - -handle_db_server_opt({ssl_opts, O1}, {ssl_opts, O2}) -> - compare_unordered_lists(O1, O2); -handle_db_server_opt(V1, V2) -> ?eq(V1, V2). +compare_ordered_lists_of_nodes(Path, L1, L2) when length(L1) =:= length(L2) -> + lists:foreach(fun({I, V1, V2}) -> compare_nodes(Path ++ [I], V1, V2) end, + lists:zip3(lists:seq(1, length(L1)), L1, L2)). %% Generic assertions, use the 'F' handler for any custom cases compare_unordered_lists(L1, L2) when is_list(L1), is_list(L2) -> diff --git a/test/config_parser_SUITE_data/outgoing_pools.toml b/test/config_parser_SUITE_data/outgoing_pools.toml index bcb86b2825..55a7d81b22 100644 --- a/test/config_parser_SUITE_data/outgoing_pools.toml +++ b/test/config_parser_SUITE_data/outgoing_pools.toml @@ -55,8 +55,8 @@ [outgoing_pools.cassandra.default.connection] servers = [ - {ip_address = "cassandra_server1.example.com", port = 9042}, - {ip_address = "cassandra_server2.example.com", port = 9042} + {host = "cassandra_server1.example.com", port = 9042}, + {host = "cassandra_server2.example.com", port = 9042} ] keyspace = "big_mongooseim" @@ -69,10 +69,10 @@ workers = 20 [outgoing_pools.rabbit.event_pusher.connection] - amqp_host = "localhost" - amqp_port = 5672 - amqp_username = "guest" - amqp_password = "guest" + host = "localhost" + port = 5672 + username = "guest" + password = "guest" confirms_enabled = true max_worker_queue_len = 100 @@ -82,5 +82,5 @@ [outgoing_pools.ldap.default.connection] servers = ["ldap-server.example.com"] - rootdn = "cn=admin,dc=example,dc=com" + root_dn = "cn=admin,dc=example,dc=com" password = "ldap-admin-password" diff --git a/test/ejabberd_sm_SUITE.erl b/test/ejabberd_sm_SUITE.erl index d357b309d0..fd09010654 100644 --- a/test/ejabberd_sm_SUITE.erl +++ b/test/ejabberd_sm_SUITE.erl @@ -2,20 +2,16 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). --include("ejabberd_c2s.hrl"). --include("mongoose.hrl"). -include_lib("jid/include/jid.hrl"). -include_lib("session.hrl"). -compile([export_all, nowarn_export_all]). - --define(_eq(E, I), ?_assertEqual(E, I)). -define(eq(E, I), ?assertEqual(E, I)). --define(ne(E, I), ?assert(E =/= I)). -define(B(C), (proplists:get_value(backend, C))). -define(MAX_USER_SESSIONS, 2). +-import(config_parser_helper, [default_config/1]). all() -> [{group, mnesia}, {group, redis}]. @@ -76,8 +72,7 @@ init_redis_group(true, Config) -> register(test_helper, self()), mongoose_wpool:ensure_started(), % This would be started via outgoing_pools in normal case - PoolConf = #{type => redis, scope => global, tag => default}, - Pool = config_parser_helper:merge_with_default_pool_config(PoolConf), + Pool = default_config([outgoing_pools, redis, default]), mongoose_wpool:start_configured_pools([Pool], []), Self ! ready, receive stop -> ok end diff --git a/test/http_client_SUITE.erl b/test/http_client_SUITE.erl index b2fa36a955..5c1d030e83 100644 --- a/test/http_client_SUITE.erl +++ b/test/http_client_SUITE.erl @@ -17,7 +17,8 @@ -compile([export_all, nowarn_export_all]). --include_lib("common_test/include/ct.hrl"). +-import(config_parser_helper, [config/2]). + -include_lib("eunit/include/eunit.hrl"). all() -> @@ -54,34 +55,19 @@ end_per_suite(_Config) -> exit(whereis(ejabberd_sup), shutdown), whereis(test_helper) ! stop. -init_per_testcase(request_timeout_test, Config) -> - mongoose_wpool:start_configured_pools([#{type => http, scope => global, tag => pool(), - opts => #{}, - conn_opts => #{server => "http://localhost:8080", - request_timeout => 10, - path_prefix => "/"}}], - [<<"a.com">>]), - Config; -init_per_testcase(pool_timeout_test, Config) -> - mongoose_wpool:start_configured_pools([#{type => http, scope => global, tag => pool(), - opts => #{workers => 1, - max_overflow => 0, - strategy => available_worker, - call_timeout => 10}, - conn_opts => #{server => "http://localhost:8080", - request_timeout => 5000, - path_prefix => "/"}}], - [<<"a.com">>]), - Config; -init_per_testcase(_TC, Config) -> - mongoose_wpool:start_configured_pools([#{type => http, scope => global, tag => pool(), - opts => #{}, - conn_opts => #{server => "http://localhost:8080", - request_timeout => 1000, - path_prefix => "/"}}], - [<<"a.com">>]), +init_per_testcase(TC, Config) -> + Pool = config([outgoing_pools, http, pool()], pool_opts(TC)), + mongoose_wpool:start_configured_pools([Pool], [<<"a.com">>]), Config. +pool_opts(request_timeout_test) -> + #{conn_opts => #{host => "http://localhost:8080", request_timeout => 10}}; +pool_opts(pool_timeout_test) -> + #{opts => #{workers => 1, max_overflow => 0, strategy => available_worker, call_timeout => 10}, + conn_opts => #{host => "http://localhost:8080", request_timeout => 5000}}; +pool_opts(_TC) -> + #{conn_opts => #{host => "http://localhost:8080", request_timeout => 1000}}. + end_per_testcase(_TC, _Config) -> mongoose_wpool:stop(http, global, pool()). diff --git a/test/mongoose_config_SUITE.erl b/test/mongoose_config_SUITE.erl index c3e4b7be30..c269d134fa 100644 --- a/test/mongoose_config_SUITE.erl +++ b/test/mongoose_config_SUITE.erl @@ -181,6 +181,7 @@ minimal_config_opts() -> {listen, []}, {loglevel, warning}, {mongooseimctl_access_commands, []}, + {outgoing_pools, []}, {rdbms_server_type, generic}, {registration_timeout, 600}, {routing_modules, mongoose_router:default_routing_modules()}, diff --git a/test/mongoose_rdbms_SUITE.erl b/test/mongoose_rdbms_SUITE.erl index e513208426..d522484533 100644 --- a/test/mongoose_rdbms_SUITE.erl +++ b/test/mongoose_rdbms_SUITE.erl @@ -59,13 +59,15 @@ init_per_testcase(does_backoff_increase_to_a_point, Config) -> meck_db(DbType), meck_connection_error(DbType), meck_rand(), - [{db_opts, [{server, server(DbType)}, {keepalive_interval, 2}, {start_interval, 10}]} | Config]; + ServerOpts = server_opts(DbType), + [{db_opts, ServerOpts#{keepalive_interval => 2, max_start_interval => 10}} | Config]; init_per_testcase(_, Config) -> DbType = ?config(db_type, Config), set_opts(), meck_db(DbType), - [{db_opts, [{server, server(DbType)}, {keepalive_interval, ?KEEPALIVE_INTERVAL}, - {start_interval, ?MAX_INTERVAL}]} | Config]. + ServerOpts = server_opts(DbType), + [{db_opts, ServerOpts#{keepalive_interval => ?KEEPALIVE_INTERVAL, + max_start_interval => ?MAX_INTERVAL}} | Config]. end_per_testcase(does_backoff_increase_to_a_point, Config) -> meck_unload_rand(), @@ -211,9 +213,11 @@ a(mysql) -> a(pgsql) -> ['_', [?KEEPALIVE_QUERY]]. -server(odbc) -> - "fake-connection-string"; -server(mysql) -> - {mysql, "fake-host", "fake-db", "fake-user", "fake-pass"}; -server(pgsql) -> - {pgsql, "fake-host", "fake-db", "fake-user", "fake-pass"}. +server_opts(odbc) -> + #{driver => odbc, settings => "fake-connection-string"}; +server_opts(mysql) -> + #{driver => mysql, host => "fake-host", port => 3306, + database => "fake-db", username => "fake-user", password => "fake-pass"}; +server_opts(pgsql) -> + #{driver => pgsql, host => "fake-host", port => 5432, + database => "fake-db", username => "fake-user", password => "fake-pass"}.