Merge pull request #3420 from esl/config-with-maps

Add an option to format a section/list as a map

big_tests/tests/accounts_SUITE.erl

@@ -92,16 +92,15 @@ init_per_group(change_account_details, Config) ->
     [{escalus_user_db,  {module, escalus_ejabberd}} |Config];
 init_per_group(change_account_details_store_plain, Config) ->
     AuthOpts = mongoose_helper:auth_opts_with_password_format(plain),
-    Config1 = mongoose_helper:backup_and_set_config_option(Config, auth_opts, AuthOpts),
+    Config1 = mongoose_helper:backup_and_set_config_option(Config, auth, AuthOpts),
     [{escalus_user_db,  {module, escalus_ejabberd}} |Config1];
 init_per_group(registration_timeout, Config) ->
     set_registration_timeout(Config);
 init_per_group(utilities, Config) ->
     escalus:create_users(Config, escalus:get_users([alice, bob]));
 init_per_group(users_number_estimate, Config) ->
     AuthOpts = get_auth_opts(),
-    Key = rdbms_users_number_estimate,
-    NewAuthOpts = lists:keystore(Key, 1, AuthOpts, {Key, true}),
+    NewAuthOpts = AuthOpts#{rdbms => #{users_number_estimate => true}},
     set_auth_opts(Config, NewAuthOpts);
 init_per_group(_GroupName, Config) ->
     Config.
@@ -125,12 +124,11 @@ end_per_group(_GroupName, Config) ->
     Config.
 
 get_auth_opts() ->
-    rpc(mim(), mongoose_config, get_opt, [{auth_opts, host_type()}]).
+    rpc(mim(), mongoose_config, get_opt, [{auth, host_type()}]).
 
 set_auth_opts(Config, AuthOpts) ->
     rpc(mim(), ejabberd_auth, stop, [host_type()]),
-    Config1 = mongoose_helper:backup_and_set_config_option(Config, {auth_opts, host_type()},
-                                                           AuthOpts),
+    Config1 = mongoose_helper:backup_and_set_config_option(Config, {auth, host_type()}, AuthOpts),
     rpc(mim(), ejabberd_auth, start, [host_type()]),
     Config1.
 

big_tests/tests/login_SUITE.erl

@@ -165,16 +165,15 @@ init_per_group(GroupName, ConfigIn) ->
     escalus:create_users(Config, escalus:get_users([alice, bob])).
 
 backup_and_set_options(GroupName, Config) ->
-    Options = config_options(GroupName),
-    mongoose_helper:backup_and_set_config(Config, Options).
+    mongoose_helper:backup_and_set_config_option(Config, {auth, host_type()}, auth_opts(GroupName)).
 
-config_options(login_digest) ->
-    #{{auth_opts, host_type()} => mongoose_helper:auth_opts_with_password_format(plain),
-      {sasl_mechanisms, host_type()} => [cyrsasl_digest]};
-config_options(login_scram_store_plain) ->
-    #{{auth_opts, host_type()} => mongoose_helper:auth_opts_with_password_format(plain)};
-config_options(_GroupName) ->
-    #{{auth_opts, host_type()} => mongoose_helper:auth_opts_with_password_format(scram)}.
+auth_opts(login_digest) ->
+    AuthOpts = mongoose_helper:auth_opts_with_password_format(plain),
+    AuthOpts#{sasl_mechanisms => [cyrsasl_digest]};
+auth_opts(login_scram_store_plain) ->
+    mongoose_helper:auth_opts_with_password_format(plain);
+auth_opts(_GroupName) ->
+    mongoose_helper:auth_opts_with_password_format(scram).
 
 end_per_group(login_digest, Config) ->
     mongoose_helper:restore_config(Config),
@@ -481,7 +480,7 @@ configure_scram_plus_and_fail_log_scram(Config, Sha, Mech) ->
 
 set_scram_sha(Config, Sha) ->
     NewAuthOpts = mongoose_helper:auth_opts_with_password_format({scram, [Sha]}),
-    mongoose_helper:change_config_option(Config, {auth_opts, host_type()}, NewAuthOpts),
+    mongoose_helper:change_config_option(Config, {auth, host_type()}, NewAuthOpts),
     assert_password_format({scram, Sha}, Config).
 
 fail_log_one(Config) ->

big_tests/tests/metrics_register_SUITE.erl

@@ -67,10 +67,10 @@ init_per_testcase(unregister, Config) ->
     escalus_users:create_user(Config, Alice),
     Config;
 init_per_testcase(registered_users, Config) ->
-    case rpc(mim(), mongoose_config, lookup_opt, [{auth_method, host_type()}]) of
-        {ok, external} ->
+    case rpc(mim(), mongoose_config, get_opt, [[{auth, host_type()}, methods], []]) of
+        [external] ->
             {skip, "counter not supported with ejabberd_auth_external"};
-        {ok, anonymous} ->
+        [anonymous] ->
             {skip, "counter not supported with anonymous authentication"};
         _ ->
             Config

big_tests/tests/mongoose_helper.erl

@@ -472,14 +472,13 @@ supports_sasl_module(Module) ->
 
 auth_opts_with_password_format(Type) ->
     HostType = domain_helper:host_type(mim),
-    AuthOpts = rpc(mim(), mongoose_config, get_opt, [{auth_opts, HostType}]),
+    AuthOpts = rpc(mim(), mongoose_config, get_opt, [{auth, HostType}]),
     build_new_auth_opts(Type, AuthOpts).
 
 build_new_auth_opts(scram, AuthOpts) ->
-    NewAuthOpts0 = lists:keystore(password_format, 1, AuthOpts, {password_format, scram}),
-    lists:keystore(scram_iterations, 1, NewAuthOpts0, {scram_iterations, 64});
+    AuthOpts#{password_format => scram, scram_iterations => 64};
 build_new_auth_opts(Type, AuthOpts) ->
-    lists:keystore(password_format, 1, AuthOpts, {password_format, Type}).
+    AuthOpts#{password_format => Type}.
 
 get_listener_opts(#{} = Spec, Port) ->
     Listeners = rpc(Spec, mongoose_config, get_opt, [listen]),

big_tests/tests/mongooseimctl_SUITE.erl

@@ -293,7 +293,7 @@ init_per_testcase(check_password_hash, Config) ->
             {skip, not_fully_supported_with_ldap};
         false ->
             AuthOpts = mongoose_helper:auth_opts_with_password_format(plain),
-            Config1 = mongoose_helper:backup_and_set_config_option(Config, {auth_opts, host_type()},
+            Config1 = mongoose_helper:backup_and_set_config_option(Config, {auth, host_type()},
                                                                    AuthOpts),
             Config2 = escalus:create_users(Config1, escalus:get_users([carol])),
             escalus:init_per_testcase(check_password_hash, Config2)

big_tests/tests/oauth_SUITE.erl

@@ -115,7 +115,7 @@ init_per_group(GroupName, Config0) ->
              end,
     AuthOpts = mongoose_helper:auth_opts_with_password_format(password_format(GroupName)),
     HostType = domain_helper:host_type(),
-    Config1 = mongoose_helper:backup_and_set_config_option(Config, {auth_opts, HostType}, AuthOpts),
+    Config1 = mongoose_helper:backup_and_set_config_option(Config, {auth, HostType}, AuthOpts),
     Config2 = escalus:create_users(Config1, escalus:get_users([bob, alice])),
     assert_password_format(GroupName, Config2).
 

big_tests/tests/sasl_SUITE.erl

@@ -29,6 +29,7 @@
 -import(distributed_helper, [mim/0,
                              require_rpc_nodes/1,
                              rpc/4]).
+-import(domain_helper, [host_type/0]).
 
 %%--------------------------------------------------------------------
 %% Suite configuration
@@ -55,12 +56,12 @@ init_per_suite(Config) ->
 end_per_suite(Config) ->
     escalus:end_per_suite(Config).
 
-init_per_group(GroupName, Config) ->
-    Config1 = set_sasl_mechanisms(mech_option_key(GroupName), Config),
+init_per_group(_GroupName, Config) ->
+    Config1 = set_sasl_mechanisms(Config),
     escalus:create_users(Config1, escalus:get_users([alice])).
 
-end_per_group(GroupName, Config) ->
-    reset_sasl_mechanisms(mech_option_key(GroupName), Config),
+end_per_group(_GroupName, Config) ->
+    reset_sasl_mechanisms(Config),
     escalus:delete_users(Config, escalus:get_users([alice])).
 
 init_per_testcase(CaseName, Config) ->
@@ -86,21 +87,20 @@ text_response(Config) ->
 %% Helpers
 %%--------------------------------------------------------------------
 
-mech_option_key(host_type_config) ->
-    HostType = domain_helper:host_type(),
-    {sasl_mechanisms, HostType}.
-
-set_sasl_mechanisms(Key, Config) ->
+set_sasl_mechanisms(Config) ->
     %% pretend that an auth module is set for this mechanism
     rpc(mim(), meck, new, [ejabberd_auth, [no_link, passthrough]]),
     rpc(mim(), meck, expect, [ejabberd_auth, supports_sasl_module,
                               fun(_, M) -> M =:= ?MODULE end]),
 
     %% configure the mechanism
-    mongoose_helper:backup_and_set_config_option(Config, Key, [?MODULE]).
+    Key = {auth, host_type()},
+    AuthOpts = rpc(mim(), mongoose_config, get_opt, [Key]),
+    NewAuthOpts = AuthOpts#{sasl_mechanisms => [?MODULE]},
+    mongoose_helper:backup_and_set_config_option(Config, Key, NewAuthOpts).
 
-reset_sasl_mechanisms(Key, Config) ->
-    mongoose_helper:restore_config_option(Config, Key),
+reset_sasl_mechanisms(Config) ->
+    mongoose_helper:restore_config_option(Config, {auth, host_type()}),
     rpc(mim(), meck, unload, [ejabberd_auth]).
 
 assert_is_failure_with_text(#xmlel{name = <<"failure">>,

big_tests/tests/shared_roster_SUITE.erl

@@ -165,7 +165,7 @@ stop_roster_module(_) ->
 
 get_auth_method() ->
     XMPPDomain = domain(),
-    case rpc(mim(), mongoose_config, get_opt, [{auth_method, XMPPDomain}]) of
+    case rpc(mim(), mongoose_config, get_opt, [[{auth, XMPPDomain}, methods], []]) of
         [Method|_] ->
             Method;
         _ ->

big_tests/tests/vcard_simple_SUITE.erl

@@ -456,7 +456,7 @@ configure_mod_vcard(Config) ->
 ldap_opts() ->
     [{backend,ldap}, {host, subhost_pattern("vjud.@HOST@")},
      {ldap_uids, [{<<"uid">>}]}, %% equivalent to {<<"uid">>, <<"%u">>}
-     {ldap_filter,"(objectClass=inetOrgPerson)"},
+     {ldap_filter,<<"(objectClass=inetOrgPerson)">>},
      {ldap_base,"ou=Users,dc=esl,dc=com"},
      {ldap_search_fields, [{"Full Name","cn"},{"User","uid"}]},
      {ldap_vcard_map,[{"FN","%s",["cn"]}]}].

doc/authentication-methods/dummy.md

@@ -16,13 +16,13 @@ where `Base` is `base_time` and `Variance` is `variance`, as configured below.
 
 ### `auth.dummy.base_time`
 * **Scope:** local
-* **Syntax:** integer
+* **Syntax:** non-negative integer
 * **Default:** 50
 * **Example:** `base_time = 5`
 
 ### `auth.dummy.variance`
 * **Scope:** local
-* **Syntax:** integer
+* **Syntax:** positive integer
 * **Default:** 450
 * **Example:** `variance = 10`
 
@@ -34,4 +34,3 @@ where `Base` is `base_time` and `Variance` is `variance`, as configured below.
   dummy.base = 5
   dummy.variance = 10
 ```
-

doc/configuration/host_config.md

@@ -59,30 +59,12 @@ The `replaced_wait_timeout` option is set to `2000` only for `domain2.com`.
 
 ### `host_config.auth`
 
-This section overrides the top-level [`auth`](auth.md) section, all options are allowed.
-It is recommended to repeat all top-level options in the domain-specific section as the rule is quite complicated:
-
-- If you specify any of the following options, **all** of the following options will be overridden:
-    - [`sasl_external`](auth.md#authsasl_external)
-    - [`password.*`](auth.md#password-related-options)
-    - [`scram_iterations`](auth.md#authscram_iterations)
-    - [`external.program`](../authentication-methods/external.md#authexternalprogram)
-    - [`rdbms.*`](../authentication-methods/rdbms.md)
-    - [`ldap.*`](../authentication-methods/ldap.md)
-    - [`jwt.*`](../authentication-methods/jwt.md)
-    - [`riak.*`](../authentication-methods/riak.md)
-    - [`http.*`](../authentication-methods/http.md)
-- If you specify any of the following options, only these options will be overridden:
-    - [`methods`](auth.md#authmethods)
-    - [`sasl_mechanisms`](auth.md#authsasl_mechanisms)
-    - [`external.instances`](../authentication-methods/external.md#authexternalinstances)
-    - [`anonymous.*`](../authentication-methods/anonymous.md)
+This section completely overrides the top-level [`auth`](auth.md) section, all options are allowed.
 
 #### Example
 
 In the example below the number of `scram_iterations` is increased for `domain2`.
-It is necessary to put the `password.hash` there as well, as otherwise it would be replaced with the default setting.
-However, specifying `methods` is not necessary as this value will not be changed.
+It is necessary to put `methods` and `password.hash` and there as well, as otherwise they would not be set for `domain2`.
 
 ```toml
 [general]
@@ -101,14 +83,6 @@ However, specifying `methods` is not necessary as this value will not be changed
     scram_iterations = 40_000
 ```
 
-The last section would work the same without `methods`:
-
-```toml
-  [host_config.auth]
-    password.hash = ["sha256"]
-    scram_iterations = 40_000
-```
-
 ### `host_config.modules`
 
 This section completely overrides the top-level [`modules`](Modules.md) section.
@@ -223,7 +197,7 @@ The options defined here override the ones defined in the top-level [`s2s`](s2s.
 The following options are allowed:
 
 * [`default_policy`](s2s.md#s2sdefault_policy)
-* [`host_policy`](s2s.md#s2shost_policy) - overrides the top-level setting host by host
+* [`host_policy`](s2s.md#s2shost_policy)
 * [`shared`](s2s.md#s2sshared)
 * [`max_retry_delay`](s2s.md#s2smax_retry_delay)
 
@@ -253,14 +227,4 @@ The `host_policy` option is changed for `domain2.com`:
     ]
 ```
 
-The resulting `host_policy` for `domain2.com` is the following:
-
-```toml
-host_policy = [
-  {host = "good-xmpp.org", policy = "allow"},
-  {host = "bad-xmpp.org", policy = "allow"},
-  {host = "evil-xmpp.org", policy = "deny"}
-]
-```
-
 The `default_policy` is still `deny`.

doc/configuration/s2s.md

@@ -94,7 +94,7 @@ The options listed below affect only the outgoing S2S connections.
     * `host` - string, mandatory, host name
     * `ip_address` - string, mandatory, IP address
     * `port` - integer, optional, port number
-* **Default:** `"allow"`
+* **Default:** not set
 * **Example:**
 
 ```toml

include/mongoose_config_spec.hrl

@@ -6,19 +6,21 @@
                   validate_keys = any :: mongoose_config_validator:validator(),
                   required = [] :: [mongoose_config_parser_toml:toml_key()] | all,
                   validate = any :: mongoose_config_validator:section_validator(),
+                  format_items = none :: mongoose_config_spec:format_items(),
                   process :: undefined | mongoose_config_parser_toml:list_processor(),
-                  format = default :: mongoose_config_spec:format(),
+                  wrap = default :: mongoose_config_spec:wrapper(),
                   defaults = #{} :: #{mongoose_config_parser_toml:toml_key() =>
                                          mongoose_config_parser_toml:config_part()}}).
 
 -record(list, {items :: mongoose_config_spec:config_node(),
                validate = any :: mongoose_config_validator:list_validator(),
+               format_items = none :: mongoose_config_spec:format_items(),
                process :: undefined | mongoose_config_parser_toml:list_processor(),
-               format = default :: mongoose_config_spec:format()}).
+               wrap = default :: mongoose_config_spec:wrapper()}).
 
 -record(option, {type :: mongoose_config_spec:option_type(),
                  validate = any :: mongoose_config_validator:validator(),
                  process :: undefined | mongoose_config_parser_toml:processor(),
-                 format = default :: mongoose_config_spec:format()}).
+                 wrap = default :: mongoose_config_spec:wrapper()}).
 
 -endif.

src/auth/ejabberd_auth.erl

@@ -29,8 +29,6 @@
 -export([start/0,
          start/1,
          stop/1,
-         get_opt/3,
-         get_opt/2,
          authorize/1,
          set_password/2,
          check_password/2,
@@ -120,25 +118,6 @@ hooks(HostType) ->
      {remove_domain, HostType, ?MODULE, remove_domain, 10}
     ].
 
--spec get_opt(HostType :: mongooseim:host_type(),
-              Opt :: atom(),
-              Default :: ejabberd:value()) -> undefined | ejabberd:value().
-get_opt(HostType, Opt, Default) ->
-    case mongoose_config:lookup_opt({auth_opts, HostType}) of
-        {error, not_found} ->
-            Default;
-        {ok, Opts} ->
-            case lists:keyfind(Opt, 1, Opts) of
-                {Opt, Value} ->
-                    Value;
-                false ->
-                    Default
-            end
-    end.
-
-get_opt(HostType, Opt) ->
-    get_opt(HostType, Opt, undefined).
-
 -spec supports_sasl_module(mongooseim:host_type(), cyrsasl:sasl_module()) -> boolean().
 supports_sasl_module(HostType, SASLModule) ->
     F = fun(Mod) ->
@@ -433,7 +412,7 @@ auth_modules_for_host_type(HostType) ->
 
 -spec auth_methods(mongooseim:host_type()) -> [atom()].
 auth_methods(HostType) ->
-    mongoose_config:get_opt({auth_method, HostType}, []).
+    mongoose_config:get_opt([{auth, HostType}, methods], []).
 
 -spec auth_method_to_module(atom()) -> authmodule().
 auth_method_to_module(Method) ->

src/auth/ejabberd_auth_anonymous.erl

@@ -86,7 +86,7 @@ stop(HostType) ->
 %% defaults to false
 -spec allow_multiple_connections(mongooseim:host_type()) -> boolean().
 allow_multiple_connections(HostType) ->
-    mongoose_config:get_opt({allow_multiple_connections, HostType}, false).
+    mongoose_config:get_opt([{auth, HostType}, anonymous, allow_multiple_connections], false).
 
 does_user_exist(_, LUser, LServer) ->
     does_anonymous_user_exist(LUser, LServer).
@@ -274,7 +274,7 @@ is_protocol_enabled(HostType, Protocol) ->
 %% @doc Returns the anonymous protocol to use, defaults to sasl_anon
 -spec anonymous_protocol(mongooseim:host_type()) -> sasl_anon | login_anon | both.
 anonymous_protocol(HostType) ->
-    mongoose_config:get_opt({anonymous_protocol, HostType}, sasl_anon).
+    mongoose_config:get_opt([{auth, HostType}, anonymous, protocol], sasl_anon).
 
 -spec supported_features() -> [atom()].
 supported_features() -> [dynamic_domains].

src/auth/ejabberd_auth_dummy.erl

@@ -39,8 +39,9 @@ stop(_HostType) ->
 
 authorize(Creds) ->
     HostType = mongoose_credentials:host_type(Creds),
-    Base = ejabberd_auth:get_opt(HostType, dummy_base_timeout, 50),
-    Variance = ejabberd_auth:get_opt(HostType, dummy_variance, 450),
+    Opts = mongoose_config:get_opt([{auth, HostType}, dummy], #{}),
+    Base = maps:get(base_time, Opts, 50),
+    Variance = maps:get(variance, Opts, 450),
     timer:sleep(Base + rand:uniform(Variance)),
     {ok, mongoose_credentials:set(Creds, auth_module, ?MODULE)}.