From d71d8d2497bba3c6c20bcc7d68e5c12c565c8ed2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chrz=C4=85szcz?= Date: Mon, 25 Apr 2022 16:21:39 +0200 Subject: [PATCH 01/20] Export process/3 to merge processor functions when merging sections --- src/config/mongoose_config_parser_toml.erl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/config/mongoose_config_parser_toml.erl b/src/config/mongoose_config_parser_toml.erl index 544a889b5a6..188fb5a7d45 100644 --- a/src/config/mongoose_config_parser_toml.erl +++ b/src/config/mongoose_config_parser_toml.erl @@ -5,6 +5,9 @@ -export([parse_file/1]). +%% Utilities for section manipulation +-export([process/3]). + -ifdef(TEST). -export([process/1, extract_errors/1]). From 1a4b8ff423b75dfc7d799857a4529eb93e8b1893 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chrz=C4=85szcz?= Date: Mon, 25 Apr 2022 16:22:37 +0200 Subject: [PATCH 02/20] Move http handler config specs out of mongoose_config_spec It will be handled by mongoose_config_handler. --- src/config/mongoose_config_spec.erl | 119 ++-------------------------- 1 file changed, 6 insertions(+), 113 deletions(-) diff --git a/src/config/mongoose_config_spec.erl b/src/config/mongoose_config_spec.erl index 95a03c33b76..d7ab6e2a2fb 100644 --- a/src/config/mongoose_config_spec.erl +++ b/src/config/mongoose_config_spec.erl @@ -3,10 +3,11 @@ %% entry point - returns the entire spec -export([root/0]). -%% spec parts used by modules and services +%% spec parts used by http handlers, modules and services -export([wpool_items/0, wpool_defaults/0, - iqdisc/0]). + iqdisc/0, + xmpp_listener_extra/1]). %% callbacks for the 'process' step -export([process_root/1, @@ -18,7 +19,6 @@ process_tls_sni/1, process_xmpp_tls/1, process_fast_tls/1, - process_http_handler/2, process_sasl_external/1, process_sasl_mechanism/1, process_auth/1, @@ -262,7 +262,7 @@ listen() -> %% path: listen.*[] listener(Type) -> - merge_sections(listener_common(), listener_extra(Type)). + mongoose_config_utils:merge_sections(listener_common(), listener_extra(Type)). listener_common() -> #section{items = #{<<"port">> => #option{type = integer, @@ -284,9 +284,9 @@ listener_extra(http) -> #section{items = #{<<"tls">> => http_listener_tls(), <<"transport">> => http_transport(), <<"protocol">> => http_protocol(), - <<"handlers">> => http_handlers()}}; + <<"handlers">> => mongoose_http_handler:config_spec()}}; listener_extra(Type) -> - merge_sections(xmpp_listener_common(), xmpp_listener_extra(Type)). + mongoose_config_utils:merge_sections(xmpp_listener_common(), xmpp_listener_extra(Type)). xmpp_listener_common() -> #section{items = #{<<"backlog">> => #option{type = integer, @@ -438,71 +438,6 @@ http_protocol() -> include = always }. -%% path: listen.http[].handlers -http_handlers() -> - Keys = [<<"mod_websockets">>, - <<"lasse_handler">>, - <<"cowboy_static">>, - <<"mongoose_api">>, - <<"mongoose_api_admin">>, - <<"mongoose_domain_handler">>, - default], - #section{ - items = maps:from_list([{Key, #list{items = http_handler(Key), - wrap = none}} || Key <- Keys]), - validate_keys = module, - include = always - }. - -%% path: listen.http[].handlers.*[] -http_handler(Key) -> - ExtraItems = http_handler_items(Key), - RequiredKeys = case http_handler_required(Key) of - all -> all; - Keys -> Keys ++ [<<"host">>, <<"path">>] - end, - #section{ - items = ExtraItems#{<<"host">> => #option{type = string, - validate = non_empty}, - <<"path">> => #option{type = string} - }, - required = RequiredKeys, - process = fun ?MODULE:process_http_handler/2 - }. - -http_handler_items(<<"mod_websockets">>) -> - #{<<"timeout">> => #option{type = int_or_infinity, - validate = non_negative}, - <<"ping_rate">> => #option{type = integer, - validate = positive}, - <<"max_stanza_size">> => #option{type = int_or_infinity, - validate = positive}, - <<"service">> => xmpp_listener_extra(service) - }; -http_handler_items(<<"lasse_handler">>) -> - #{<<"module">> => #option{type = atom, - validate = module}}; -http_handler_items(<<"cowboy_static">>) -> - #{<<"type">> => #option{type = atom}, - <<"app">> => #option{type = atom}, - <<"content_path">> => #option{type = string}}; -http_handler_items(<<"mongoose_api">>) -> - #{<<"handlers">> => #list{items = #option{type = atom, - validate = module}}}; -http_handler_items(<<"mongoose_api_admin">>) -> - #{<<"username">> => #option{type = binary}, - <<"password">> => #option{type = binary}}; -http_handler_items(<<"mongoose_domain_handler">>) -> - #{<<"username">> => #option{type = binary}, - <<"password">> => #option{type = binary}}; -http_handler_items(_) -> - #{}. - -http_handler_required(<<"lasse_handler">>) -> all; -http_handler_required(<<"cowboy_static">>) -> [<<"type">>, <<"content_path">>]; -http_handler_required(<<"mongoose_api">>) -> all; -http_handler_required(_) -> []. - %% path: (host_config[].)auth auth() -> Items = maps:from_list([{a2b(Method), ejabberd_auth:config_spec(Method)} || @@ -1167,39 +1102,6 @@ listener_module(<<"c2s">>) -> ejabberd_c2s; listener_module(<<"s2s">>) -> ejabberd_s2s_in; listener_module(<<"service">>) -> ejabberd_service. -process_http_handler([item, Type | _], KVs) -> - {[[{host, Host}], [{path, Path}]], Opts} = proplists:split(KVs, [host, path]), - HandlerOpts = process_http_handler_opts(Type, Opts), - {Host, Path, binary_to_atom(Type, utf8), HandlerOpts}. - -process_http_handler_opts(<<"lasse_handler">>, [{module, Module}]) -> - [Module]; -process_http_handler_opts(<<"cowboy_static">>, Opts) -> - case proplists:split(Opts, [type, app, content_path]) of - {[[{type, Type}], [{app, App}], [{content_path, Path}]], []} -> - {Type, App, Path, [{mimetypes, cow_mimetypes, all}]}; - {[[{type, Type}], [], [{content_path, Path}]], []} -> - {Type, Path, [{mimetypes, cow_mimetypes, all}]} - end; -process_http_handler_opts(<<"mongoose_api_admin">>, Opts) -> - {[UserOpts, PassOpts], []} = proplists:split(Opts, [username, password]), - case {UserOpts, PassOpts} of - {[], []} -> []; - {[{username, User}], [{password, Pass}]} -> [{auth, {User, Pass}}] - end; -process_http_handler_opts(<<"mongoose_domain_handler">>, Opts) -> - {[UserOpts, PassOpts], []} = proplists:split(Opts, [username, password]), - case {UserOpts, PassOpts} of - {[], []} -> ok; - {[{username, _User}], [{password, _Pass}]} -> ok; - _ -> error(#{what => both_username_and_password_required, - handler => mongoose_domain_handler, opts => Opts}) - end, - Opts; -process_http_handler_opts(<<"cowboy_swagger_redirect_handler">>, []) -> #{}; -process_http_handler_opts(<<"cowboy_swagger_json_handler">>, []) -> #{}; -process_http_handler_opts(_, Opts) -> Opts. - process_sasl_external(V) when V =:= standard; V =:= common_name; V =:= auth_id -> @@ -1340,12 +1242,3 @@ process_s2s_address(M) -> process_domain_cert(#{domain := Domain, certfile := Certfile}) -> {Domain, Certfile}. - -%% Helpers - -merge_sections(BasicSection, ExtraSection) -> - #section{items = Items1, required = Required1, defaults = Defaults1} = BasicSection, - #section{items = Items2, required = Required2, defaults = Defaults2} = ExtraSection, - BasicSection#section{items = maps:merge(Items1, Items2), - required = Required1 ++ Required2, - defaults = maps:merge(Defaults1, Defaults2)}. From 62a55f9ef200a67a21cfc9a1970e6a017f3b48f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chrz=C4=85szcz?= Date: Mon, 25 Apr 2022 16:23:41 +0200 Subject: [PATCH 03/20] Move merge_sections/2 to mongoose_config_utils The goal is better separation between specs and utils --- src/config/mongoose_config_utils.erl | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/config/mongoose_config_utils.erl b/src/config/mongoose_config_utils.erl index 3d278fae145..364c2642b76 100644 --- a/src/config/mongoose_config_utils.erl +++ b/src/config/mongoose_config_utils.erl @@ -2,8 +2,8 @@ %% This stuff can be pure, but most likely not. %% It's for generic functions. -module(mongoose_config_utils). --export([exit_or_halt/1]). --export([section_to_defaults/1]). +-export([exit_or_halt/1, section_to_defaults/1, merge_sections/2]). + -ignore_xref([section_to_defaults/1]). -include("mongoose_config_spec.hrl"). @@ -21,3 +21,22 @@ exit_or_halt(ExitText) -> section_to_defaults(#section{defaults = Defaults}) -> Defaults. + +-spec merge_sections(mongoose_config_spec:config_section(), + mongoose_config_spec:config_section()) -> + mongoose_config_spec:config_section(). +merge_sections(BasicSection, ExtraSection) -> + #section{items = Items1, required = Required1, defaults = Defaults1, + process = Process1} = BasicSection, + #section{items = Items2, required = Required2, defaults = Defaults2, + process = Process2} = ExtraSection, + BasicSection#section{items = maps:merge(Items1, Items2), + required = Required1 ++ Required2, + defaults = maps:merge(Defaults1, Defaults2), + process = merge_process_functions(Process1, Process2)}. + +merge_process_functions(Process1, Process2) -> + fun(Path, V) -> + V1 = mongoose_config_parser_toml:process(Path, V, Process1), + mongoose_config_parser_toml:process(Path, V1, Process2) + end. From 0a5ccd0ebec8ded4384454357704ebeaaa658756 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chrz=C4=85szcz?= Date: Mon, 25 Apr 2022 16:25:23 +0200 Subject: [PATCH 04/20] Remove unused include --- src/config/mongoose_config_validator.erl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/config/mongoose_config_validator.erl b/src/config/mongoose_config_validator.erl index 1d0a832c052..241255cefcf 100644 --- a/src/config/mongoose_config_validator.erl +++ b/src/config/mongoose_config_validator.erl @@ -5,7 +5,6 @@ validate_list/2]). -include("mongoose.hrl"). --include("mongoose_config_spec.hrl"). -include_lib("jid/include/jid.hrl"). -type validator() :: From 1706de12d9a68cb786533fec265f1aa527c6317d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chrz=C4=85szcz?= Date: Mon, 25 Apr 2022 16:37:45 +0200 Subject: [PATCH 05/20] Add a new behaviour: mongoose_http_handler API: - config_spec/0 returns the config section spec for all possible HTTP handlers - get_routes/1 returns Cowboy routes for a list of configured handlers There are two optional callbacks: - config_spec/0 returning config specification for the handler, if it has any options - routes/1 returning Cowboy routes, if different from the default All handlers which have options should be listed in configurable_handler_modules/0 --- src/mongoose_http_handler.erl | 88 +++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 src/mongoose_http_handler.erl diff --git a/src/mongoose_http_handler.erl b/src/mongoose_http_handler.erl new file mode 100644 index 00000000000..d5a05d77ebd --- /dev/null +++ b/src/mongoose_http_handler.erl @@ -0,0 +1,88 @@ +%% @doc Manage the configuration and initialization of HTTP handlers + +-module(mongoose_http_handler). + +-export([config_spec/0, process_config/2, cowboy_host/1, get_routes/1]). + +-type options() :: #{host := '_' | string(), + path := string(), + module := module(), + atom() => any()}. + +-type path() :: iodata(). +-type routes() :: [{path(), module(), options()}]. + +-export_type([options/0, path/0, routes/0]). + +-callback config_spec() -> mongoose_config_spec:config_section(). +-callback routes(options()) -> routes(). + +-optional_callbacks([config_spec/0, routes/1]). + +-include("mongoose.hrl"). +-include("mongoose_config_spec.hrl"). + +%% @doc Return a config section with a list of sections for each handler type +-spec config_spec() -> mongoose_config_spec:config_section(). +config_spec() -> + Items = maps:from_list([{atom_to_binary(Module), + #list{items = handler_config_spec(Module), + wrap = none}} + || Module <- configurable_handler_modules()]), + #section{items = Items#{default => #list{items = common_handler_config_spec(), + wrap = none}}, + validate_keys = module, + include = always}. + +%% @doc Return a config section with handler options +-spec handler_config_spec(module()) -> mongoose_config_spec:config_section(). +handler_config_spec(Module) -> + mongoose_config_utils:merge_sections(common_handler_config_spec(), Module:config_spec()). + +common_handler_config_spec() -> + #section{items = #{<<"host">> => #option{type = string, + validate = non_empty, + process = fun ?MODULE:cowboy_host/1}, + <<"path">> => #option{type = string} + }, + required = [<<"host">>, <<"path">>], + format_items = map, + process = fun ?MODULE:process_config/2}. + +process_config([item, HandlerType | _], Opts) -> + Opts#{module => binary_to_atom(HandlerType)}. + +%% @doc Return the list of Cowboy routes for the specified handler configuration. +%% Cowboy will search for a matching Host, then for a matching Path. If no Path matches, +%% Cowboy will not search for another matching Host. So, we must merge all Paths for each Host, +%% add any wildcard Paths to each Host, and ensure that the wildcard Host '_' is listed last. +%% A proplist ensures that the user can influence Host ordering if wildcards like "[...]" are used. +-spec get_routes([options()]) -> cowboy_router:routes(). +get_routes(Handlers) -> + Routes = lists:foldl(fun add_handler_routes/2, [], Handlers), + WildcardPaths = proplists:get_value('_', Routes, []), + Merge = fun(Paths) -> Paths ++ WildcardPaths end, + Merged = lists:keymap(Merge, 2, proplists:delete('_', Routes)), + Final = Merged ++ [{'_', WildcardPaths}], + ?LOG_DEBUG(#{what => configured_cowboy_routes, routes => Final}), + Final. + +add_handler_routes(#{host := Host, path := Path, module := Module} = HandlerOpts, Routes) -> + HandlerRoutes = case backend_module:is_exported(Module, routes, 1) of + true -> Module:routes(HandlerOpts); + false -> [{Path, Module, HandlerOpts}] + end, + HostRoutes = proplists:get_value(Host, Routes, []), + lists:keystore(Host, 1, Routes, {Host, HostRoutes ++ HandlerRoutes}). + +%% @doc Translate "_" used in TOML to '_' expected by COwboy +cowboy_host("_") -> '_'; +cowboy_host(Host) -> Host. + +%% @doc All handlers implementing config_spec/0 are listed here +configurable_handler_modules() -> + [mod_websockets, + mongoose_client_api, + mongoose_api, + mongoose_api_admin, + mongoose_domain_handler]. From 12d72a509d189013465972049e627b765fececa8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chrz=C4=85szcz?= Date: Mon, 25 Apr 2022 16:25:40 +0200 Subject: [PATCH 06/20] Add mongoose_http_handler callbacks to mongoose_domain_handler --- src/domain/mongoose_domain_handler.erl | 43 ++++++++++++++++++++------ 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/src/domain/mongoose_domain_handler.erl b/src/domain/mongoose_domain_handler.erl index a81449b1d78..ba86bcd0ec8 100644 --- a/src/domain/mongoose_domain_handler.erl +++ b/src/domain/mongoose_domain_handler.erl @@ -1,9 +1,14 @@ %% REST API for domain actions. -module(mongoose_domain_handler). + +-behaviour(mongoose_http_handler). -behaviour(cowboy_rest). -%% ejabberd_cowboy exports --export([cowboy_router_paths/2]). +%% mongoose_http_handler callbacks +-export([config_spec/0, routes/1]). + +%% config processing callbacks +-export([process_config/1]). %% Standard cowboy_rest callbacks. -export([init/2, @@ -20,17 +25,37 @@ -ignore_xref([cowboy_router_paths/2, handle_domain/2, to_json/2]). -include("mongoose_logger.hrl"). +-include("mongoose_config_spec.hrl"). -type state() :: map(). --spec cowboy_router_paths(ejabberd_cowboy:path(), ejabberd_cowboy:options()) -> - ejabberd_cowboy:implemented_result(). -cowboy_router_paths(Base, Opts) -> - [{[Base, "/domains/:domain"], ?MODULE, Opts}]. +-type handler_options() :: #{path := string(), username => binary(), password => binary(), + atom() => any()}. + +%% mongoose_http_handler callbacks + +-spec config_spec() -> mongoose_config_spec:config_section(). +config_spec() -> + #section{items = #{<<"username">> => #option{type = binary}, + <<"password">> => #option{type = binary}}, + format_items = map, + process = fun ?MODULE:process_config/1}. + +process_config(Opts) -> + case maps:is_key(username, Opts) =:= maps:is_key(password, Opts) of + true -> + Opts; + false -> + error(#{what => both_username_and_password_required, opts => Opts}) + end. + +-spec routes(handler_options()) -> mongoose_http_handler:routes(). +routes(Opts = #{path := BasePath}) -> + [{[BasePath, "/domains/:domain"], ?MODULE, Opts}]. + +%% cowboy_rest callbacks -%% cowboy_rest callbacks: -%% Opts could be `[{password, <<\"secret\">>}, {username, <<\"admin\">>}]'. init(Req, Opts) -> - {cowboy_rest, Req, maps:from_list(Opts)}. + {cowboy_rest, Req, Opts}. allowed_methods(Req, State) -> {[<<"GET">>, <<"PUT">>, <<"PATCH">>, <<"DELETE">>], Req, State}. From a84386c9b4b2c2a89cabb8db1e81931b66903fdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chrz=C4=85szcz?= Date: Mon, 25 Apr 2022 16:27:12 +0200 Subject: [PATCH 07/20] Update options type in mod_bosh --- src/mod_bosh.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mod_bosh.erl b/src/mod_bosh.erl index d3c6bce85ab..db657f3aa10 100644 --- a/src/mod_bosh.erl +++ b/src/mod_bosh.erl @@ -9,6 +9,7 @@ -behaviour(mongoose_module_metrics). %% cowboy_loop is a long polling handler -behaviour(cowboy_loop). + -xep([{xep, 206}, {version, "1.4"}]). -xep([{xep, 124}, {version, "1.11"}]). @@ -130,8 +131,7 @@ node_cleanup(Acc, Node) -> %% cowboy_loop_handler callbacks %%-------------------------------------------------------------------- --type option() :: {atom(), any()}. --spec init(req(), _Opts :: [option()]) -> {cowboy_loop, req(), rstate()}. +-spec init(req(), mongoose_http_handler:options()) -> {cowboy_loop, req(), rstate()}. init(Req, _Opts) -> ?LOG_DEBUG(#{what => bosh_init, req => Req}), Msg = init_msg(Req), From 3f703402405b73a0ac33e45fdac8096d3d6b217d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chrz=C4=85szcz?= Date: Mon, 25 Apr 2022 16:27:49 +0200 Subject: [PATCH 08/20] Add mongoose_http_handler callbacks to mod_websockets Use maps with defaults in the config spec --- src/mod_websockets.erl | 42 +++++++++++++++++++++++++++++------------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/src/mod_websockets.erl b/src/mod_websockets.erl index 7e70a579c7f..82f616c62b7 100644 --- a/src/mod_websockets.erl +++ b/src/mod_websockets.erl @@ -5,9 +5,13 @@ %%%=================================================================== -module(mod_websockets). +-behaviour(mongoose_http_handler). -behaviour(cowboy_websocket). -behaviour(mongoose_transport). +%% mongoose_http_handler callbacks +-export([config_spec/0]). + %% cowboy_http_websocket_handler callbacks -export([init/2, websocket_init/1, @@ -33,6 +37,7 @@ get_peer_certificate/1, get_sockmod/1, send/2, set_ping/2]). -include("mongoose.hrl"). +-include("mongoose_config_spec.hrl"). -include("jlib.hrl"). -include_lib("exml/include/exml_stream.hrl"). @@ -49,7 +54,7 @@ fsm_pid :: pid() | undefined, open_tag :: stream | open | undefined, parser :: exml_stream:parser() | undefined, - opts :: proplists:proplist() | undefined, + opts :: map(), ping_rate :: integer() | none, max_stanza_size :: integer() | infinity, peercert :: undefined | passed | binary() @@ -58,19 +63,33 @@ -type socket() :: #websocket{}. +%% mongoose_http_handler callbacks + +-spec config_spec() -> mongoose_config_spec:config_section(). +config_spec() -> + #section{items = #{<<"timeout">> => #option{type = int_or_infinity, + validate = non_negative}, + <<"ping_rate">> => #option{type = integer, + validate = positive}, + <<"max_stanza_size">> => #option{type = int_or_infinity, + validate = positive}, + <<"service">> => mongoose_config_spec:xmpp_listener_extra(service)}, + defaults = #{<<"timeout">> => 60000, + <<"max_stanza_size">> => infinity}, + format_items = map + }. + %%-------------------------------------------------------------------- %% Common callbacks for all cowboy behaviours %%-------------------------------------------------------------------- -init(Req, Opts) -> +init(Req, Opts = #{timeout := Timeout}) -> Peer = cowboy_req:peer(Req), PeerCert = cowboy_req:cert(Req), Req1 = add_sec_websocket_protocol_header(Req), ?LOG_DEBUG(#{what => ws_init, text => <<"New websockets request">>, req => Req, opts => Opts}), - Timeout = proplists:get_value(timeout, Opts, 60000), - - AllModOpts = [{peer, Peer}, {peercert, PeerCert} | Opts], + AllModOpts = Opts#{peer => Peer, peer_cert => PeerCert}, %% upgrade protocol {cowboy_websocket, Req1, AllModOpts, #{idle_timeout => Timeout}}. @@ -82,11 +101,8 @@ terminate(_Reason, _Req, _State) -> %%-------------------------------------------------------------------- % Called for every new websocket connection. -websocket_init(Opts) -> - PingRate = proplists:get_value(ping_rate, Opts, none), - MaxStanzaSize = proplists:get_value(max_stanza_size, Opts, infinity), - Peer = proplists:get_value(peer, Opts), - PeerCert = proplists:get_value(peercert, Opts), +websocket_init(Opts = #{peer := Peer, peer_cert := PeerCert, max_stanza_size := MaxStanzaSize}) -> + PingRate = maps:get(ping_rate, Opts, none), maybe_send_ping_request(PingRate), ?LOG_DEBUG(#{what => ws_init, text => <<"New websockets connection">>, peer => Peer, opts => Opts}), @@ -194,9 +210,9 @@ send_to_fsm(FSM, StreamElement) -> maybe_start_fsm([#xmlstreamstart{ name = <<"stream", _/binary>>, attrs = Attrs} | _], - #ws_state{fsm_pid = undefined, opts = Opts}=State) -> - case {lists:keyfind(<<"xmlns">>, 1, Attrs), proplists:get_value(service, Opts)} of - {{<<"xmlns">>, ?NS_COMPONENT}, #{} = ServiceOpts} -> + #ws_state{fsm_pid = undefined, opts = Opts} = State) -> + case {lists:keyfind(<<"xmlns">>, 1, Attrs), Opts} of + {{<<"xmlns">>, ?NS_COMPONENT}, #{service := ServiceOpts}} -> do_start_fsm(ejabberd_service, ServiceOpts, State); _ -> {stop, State} From 3d4bfd99d7adeecb9f16e04aa53505d3e934cb90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chrz=C4=85szcz?= Date: Mon, 25 Apr 2022 16:28:46 +0200 Subject: [PATCH 09/20] Add mongoose_http_handler callbacks to mongoose_api Use maps with defaults, include all handlers by default --- src/mongoose_api.erl | 48 ++++++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/src/mongoose_api.erl b/src/mongoose_api.erl index bb49357cca1..693f9148fe0 100644 --- a/src/mongoose_api.erl +++ b/src/mongoose_api.erl @@ -15,10 +15,11 @@ %%============================================================================== -module(mongoose_api). +-behaviour(mongoose_http_handler). -behaviour(cowboy_rest). -%% ejabberd_cowboy callbacks --export([cowboy_router_paths/2]). +%% mongoose_http_handler callbacks +-export([config_spec/0, routes/1]). %% cowboy_rest callbacks -export([init/2, @@ -54,29 +55,38 @@ -callback handle_put(term(), bindings(), options()) -> response(). -callback handle_delete(bindings(), options()) -> response(). +-include("mongoose_config_spec.hrl"). + +-type handler_options() :: #{path := string(), handlers := [module()], atom() => any()}. + %%-------------------------------------------------------------------- -%% ejabberd_cowboy callbacks +%% mongoose_http_handler callbacks %%-------------------------------------------------------------------- -cowboy_router_paths(Base, Opts) -> - Handlers = proplists:get_value(handlers, Opts, []), - lists:flatmap(pa:bind(fun register_handler/2, Base), Handlers). - -register_handler(Base, Handler) -> - [{[Base, Handler:prefix(), Path], ?MODULE, [{handler, Handler}|Opts]} - || {Path, Opts} <- Handler:routes()]. +-spec config_spec() -> mongoose_config_spec:config_section(). +config_spec() -> + HandlerModules = [mongoose_api_metrics, mongoose_api_users], + #section{items = #{<<"handlers">> => #list{items = #option{type = atom, + validate = {enum, HandlerModules}}, + validate = unique_non_empty} + }, + defaults = #{<<"handlers">> => HandlerModules}, + format_items = map}. + +-spec routes(handler_options()) -> mongoose_http_handler:routes(). +routes(#{path := BasePath, handlers := HandlerModules}) -> + lists:flatmap(fun(Module) -> cowboy_routes(BasePath, Module) end, HandlerModules). + +cowboy_routes(BasePath, Module) -> + [{[BasePath, Module:prefix(), Path], ?MODULE, #{module => Module, opts => Opts}} + || {Path, Opts} <- Module:routes()]. %%-------------------------------------------------------------------- %% cowboy_rest callbacks %%-------------------------------------------------------------------- -init(Req, Opts) -> - case lists:keytake(handler, 1, Opts) of - {value, {handler, Handler}, Opts1} -> - Bindings = maps:to_list(cowboy_req:bindings(Req)), - State = #state{handler=Handler, opts=Opts1, bindings=Bindings}, - {cowboy_rest, Req, State}; % upgrade protocol - false -> - erlang:throw(no_handler_defined) - end. +init(Req, #{module := Module, opts := Opts}) -> + Bindings = maps:to_list(cowboy_req:bindings(Req)), + State = #state{handler = Module, opts = Opts, bindings = Bindings}, + {cowboy_rest, Req, State}. % upgrade protocol terminate(_Reason, _Req, _State) -> ok. From b24ffda17525e032dbb6df2eeff323cd6fcd9558 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chrz=C4=85szcz?= Date: Mon, 25 Apr 2022 16:30:14 +0200 Subject: [PATCH 10/20] Add mongoose_http_handler callbacks to mongoose_api_admin - Use maps with defaults for options --- src/mongoose_api_admin.erl | 60 ++++++++++++++++++++++++++------------ 1 file changed, 41 insertions(+), 19 deletions(-) diff --git a/src/mongoose_api_admin.erl b/src/mongoose_api_admin.erl index 1338f44c570..c68dc44e3b4 100644 --- a/src/mongoose_api_admin.erl +++ b/src/mongoose_api_admin.erl @@ -10,10 +10,15 @@ %% @end -module(mongoose_api_admin). -author("ludwikbukowski"). + +-behaviour(mongoose_http_handler). -behaviour(cowboy_rest). -%% ejabberd_cowboy exports --export([cowboy_router_paths/2]). +%% mongoose_http_handler callbacks +-export([config_spec/0, routes/1]). + +%% config processing callbacks +-export([process_config/1]). %% cowboy_rest exports -export([allowed_methods/2, @@ -31,6 +36,7 @@ -include("mongoose_api.hrl"). -include("mongoose.hrl"). +-include("mongoose_config_spec.hrl"). -import(mongoose_api_common, [error_response/4, action_to_method/1, @@ -41,20 +47,36 @@ -type credentials() :: {Username :: binary(), Password :: binary()} | any. +-type handler_options() :: #{path := string(), username => binary(), password => binary(), + atom() => any()}. + %%-------------------------------------------------------------------- -%% ejabberd_cowboy callbacks +%% mongoose_http_handler callbacks %%-------------------------------------------------------------------- -%% @doc This is implementation of ejabberd_cowboy callback. -%% Returns list of all available http paths. --spec cowboy_router_paths(ejabberd_cowboy:path(), ejabberd_cowboy:options()) -> - ejabberd_cowboy:implemented_result(). -cowboy_router_paths(Base, Opts) -> +-spec config_spec() -> mongoose_config_spec:config_section(). +config_spec() -> + #section{items = #{<<"username">> => #option{type = binary}, + <<"password">> => #option{type = binary}}, + format_items = map, + process = fun ?MODULE:process_config/1}. + +-spec process_config(handler_options()) -> handler_options(). +process_config(Opts) -> + case maps:is_key(username, Opts) =:= maps:is_key(password, Opts) of + true -> + Opts; + false -> + error(#{what => both_username_and_password_required, opts => Opts}) + end. + +-spec routes(handler_options()) -> mongoose_http_handler:routes(). +routes(Opts = #{path := BasePath}) -> ejabberd_hooks:add(register_command, global, mongoose_api_common, reload_dispatches, 50), ejabberd_hooks:add(unregister_command, global, mongoose_api_common, reload_dispatches, 50), try Commands = mongoose_commands:list(admin), - [handler_path(Base, Command, Opts) || Command <- Commands] + [handler_path(BasePath, Command, Opts) || Command <- Commands] catch Class:Err:StackTrace -> ?LOG_ERROR(#{what => getting_command_list_error, @@ -68,9 +90,8 @@ cowboy_router_paths(Base, Opts) -> init(Req, Opts) -> Bindings = maps:to_list(cowboy_req:bindings(Req)), - CommandCategory = proplists:get_value(command_category, Opts), - CommandSubCategory = proplists:get_value(command_subcategory, Opts), - Auth = proplists:get_value(auth, Opts, any), + #{command_category := CommandCategory, command_subcategory := CommandSubCategory} = Opts, + Auth = auth_opts(Opts), State = #http_api_state{allowed_methods = mongoose_api_common:get_allowed_methods(admin), bindings = Bindings, command_category = CommandCategory, @@ -78,6 +99,9 @@ init(Req, Opts) -> auth = Auth}, {cowboy_rest, Req, State}. +auth_opts(#{username := UserName, password := Password}) -> {UserName, Password}; +auth_opts(#{}) -> any. + options(Req, State) -> Req1 = set_cors_headers(Req), {ok, Req1, State}. @@ -187,11 +211,9 @@ from_json(Req, #http_api_state{command_category = Category, end end. --spec handler_path(ejabberd_cowboy:path(), mongoose_commands:t(), [{atom(), term()}]) -> - ejabberd_cowboy:route(). -handler_path(Base, Command, ExtraOpts) -> +-spec handler_path(ejabberd_cowboy:path(), mongoose_commands:t(), handler_options()) -> + ejabberd_cowboy:route(). +handler_path(Base, Command, CommonOpts) -> {[Base, mongoose_api_common:create_admin_url_path(Command)], - ?MODULE, [{command_category, mongoose_commands:category(Command)}, - {command_subcategory, mongoose_commands:subcategory(Command)} | ExtraOpts]}. - - + ?MODULE, CommonOpts#{command_category => mongoose_commands:category(Command), + command_subcategory => mongoose_commands:subcategory(Command)}}. From f45db8d69ed2fe68671f93883cbadad17e02b9de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chrz=C4=85szcz?= Date: Mon, 25 Apr 2022 16:32:08 +0200 Subject: [PATCH 11/20] Add mongoose_http_handler callbacks to mongoose_api_client Expect options in a map --- src/mongoose_api_client.erl | 34 ++++++++++++++-------------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/src/mongoose_api_client.erl b/src/mongoose_api_client.erl index df0cfe81883..0f26c58dcd5 100644 --- a/src/mongoose_api_client.erl +++ b/src/mongoose_api_client.erl @@ -16,8 +16,12 @@ -include("jlib.hrl"). -include("mongoose.hrl"). -%% ejabberd_cowboy exports --export([cowboy_router_paths/2, to_json/2, from_json/2]). +-behaviour(mongoose_http_handler). + +%% mongoose_http_handler callbacks +-export([routes/1]). + +-export([to_json/2, from_json/2]). %% API -export([is_authorized/2, @@ -39,19 +43,16 @@ parse_request_body/1]). %%-------------------------------------------------------------------- -%% ejabberd_cowboy callbacks +%% mongoose_http_handler callbacks %%-------------------------------------------------------------------- -%% @doc This is implementation of ejabberd_cowboy callback. -%% Returns list of all available http paths. --spec cowboy_router_paths(ejabberd_cowboy:path(), ejabberd_cowboy:options()) -> - ejabberd_cowboy:implemented_result(). -cowboy_router_paths(Base, _Opts) -> +-spec routes(mongoose_http_handler:options()) -> mongoose_http_handler:routes(). +routes(#{path := BasePath}) -> ejabberd_hooks:add(register_command, global, mongoose_api_common, reload_dispatches, 50), ejabberd_hooks:add(unregister_command, global, mongoose_api_common, reload_dispatches, 50), try Commands = mongoose_commands:list(user), - [handler_path(Base, Command) || Command <- Commands] + [handler_path(BasePath, Command) || Command <- Commands] catch Class:Err:Stacktrace -> ?LOG_ERROR(#{what => rest_getting_command_list_failed, @@ -59,23 +60,16 @@ cowboy_router_paths(Base, _Opts) -> [] end. - %%-------------------------------------------------------------------- %% cowboy_rest callbacks %%-------------------------------------------------------------------- -init(Req, Opts) -> +init(Req, #{command_category := CommandCategory}) -> Bindings = maps:to_list(cowboy_req:bindings(Req)), - CommandCategory = - case lists:keytake(command_category, 1, Opts) of - {value, {command_category, Name}, _Opts1} -> - Name; - false -> - undefined - end, State = #http_api_state{allowed_methods = mongoose_api_common:get_allowed_methods(user), - bindings = Bindings, command_category = CommandCategory}, + bindings = Bindings, + command_category = CommandCategory}, {cowboy_rest, Req, State}. allowed_methods(Req, #http_api_state{command_category = Name} = State) -> @@ -171,5 +165,5 @@ make_unauthorized_response(Req, State) -> -spec handler_path(ejabberd_cowboy:path(), mongoose_commands:t()) -> ejabberd_cowboy:route(). handler_path(Base, Command) -> {[Base, mongoose_api_common:create_user_url_path(Command)], - ?MODULE, [{command_category, mongoose_commands:category(Command)}]}. + ?MODULE, #{command_category => mongoose_commands:category(Command)}}. From 19a798ace16e70c0a2455303edee087126d7bc93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chrz=C4=85szcz?= Date: Mon, 25 Apr 2022 16:32:43 +0200 Subject: [PATCH 12/20] Add mongoose_http_handler callbacks to mongoose_client_api - Use maps with defaults - Include all handlers by default New options: - 'handlers' - to choose specific low-level handlers, - 'docs' - to disable docs. --- .../mongoose_client_api.erl | 49 ++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/src/mongoose_client_api/mongoose_client_api.erl b/src/mongoose_client_api/mongoose_client_api.erl index c005d6859bf..95d594de411 100644 --- a/src/mongoose_client_api/mongoose_client_api.erl +++ b/src/mongoose_client_api/mongoose_client_api.erl @@ -1,5 +1,10 @@ -module(mongoose_client_api). +-behaviour(mongoose_http_handler). + +%% mongoose_http_handler callbacks +-export([config_spec/0, routes/1]). + -export([init/2]). -export([content_types_provided/2]). -export([is_authorized/2]). @@ -16,7 +21,49 @@ options/2, to_json/2]). -include("mongoose.hrl"). --include("jlib.hrl"). +-include("mongoose_config_spec.hrl"). + +-type handler_options() :: #{path := string(), handlers := [module()], docs := boolean(), + atom() => any()}. + +%% mongoose_http_handler callbacks + +-spec config_spec() -> mongoose_config_spec:config_section(). +config_spec() -> + HandlerModules = [Module || {_, Module, _} <- api_paths()], + #section{items = #{<<"handlers">> => #list{items = #option{type = atom, + validate = {enum, HandlerModules}}, + validate = unique}, + <<"docs">> => #option{type = boolean}}, + defaults = #{<<"handlers">> => HandlerModules, + <<"docs">> => true}, + format_items = map}. + +-spec routes(handler_options()) -> mongoose_http_handler:routes(). +routes(Opts = #{path := BasePath}) -> + [{[BasePath, Path], Module, ModuleOpts} + || {Path, Module, ModuleOpts} <- api_paths(Opts)] ++ api_doc_paths(Opts). + +api_paths(#{handlers := HandlerModules}) -> + lists:filter(fun({_, Module, _}) -> lists:member(Module, HandlerModules) end, api_paths()). + +api_paths() -> + [{"/sse", lasse_handler, #{module => mongoose_client_api_sse}}, + {"/messages/[:with]", mongoose_client_api_messages, #{}}, + {"/contacts/[:jid]", mongoose_client_api_contacts, #{}}, + {"/rooms/[:id]", mongoose_client_api_rooms, #{}}, + {"/rooms/[:id]/config", mongoose_client_api_rooms_config, #{}}, + {"/rooms/:id/users/[:user]", mongoose_client_api_rooms_users, #{}}, + {"/rooms/[:id]/messages", mongoose_client_api_rooms_messages, #{}}]. + +api_doc_paths(#{docs := true}) -> + [{"/api-docs", cowboy_swagger_redirect_handler, #{}}, + {"/api-docs/swagger.json", cowboy_swagger_json_handler, #{}}, + {"/api-docs/[...]", cowboy_static, {priv_dir, cowboy_swagger, "swagger", + [{mimetypes, cow_mimetypes, all}]} + }]; +api_doc_paths(#{docs := false}) -> + []. init(Req, _Opts) -> State = #{}, From a40f30806626e9c600fde999b037c754596a7ee8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chrz=C4=85szcz?= Date: Mon, 25 Apr 2022 16:35:14 +0200 Subject: [PATCH 13/20] Move handler-related functionality out of ejabberd_cowboy It will be placed in mongoose_http_handler Also: refactor 'store_trails' --- src/ejabberd_cowboy.erl | 99 +++++++---------------------------------- 1 file changed, 17 insertions(+), 82 deletions(-) diff --git a/src/ejabberd_cowboy.erl b/src/ejabberd_cowboy.erl index 9ae5339309a..8bd611f8e3e 100644 --- a/src/ejabberd_cowboy.erl +++ b/src/ejabberd_cowboy.erl @@ -44,11 +44,6 @@ start_cowboy/4, start_link/1, start_listener/2, start_listener/1, stop_cowboy/1]). -include("mongoose.hrl"). --type options() :: [any()]. --type path() :: binary(). --type paths() :: [path()]. --type route() :: {path() | paths(), module(), options()}. --type implemented_result() :: [route()]. -type listener_options() :: #{port := inet:port_number(), ip_tuple := inet:ip_address(), @@ -60,13 +55,6 @@ protocol := cowboy:opts(), atom() => any()}. --export_type([options/0]). --export_type([path/0]). --export_type([route/0]). --export_type([implemented_result/0]). - --callback cowboy_router_paths(path(), options()) -> implemented_result(). - -record(cowboy_state, {ref :: atom(), opts :: listener_options()}). %%-------------------------------------------------------------------- @@ -162,12 +150,13 @@ start_cowboy(Ref, Opts, Retries, SleepTime) -> -spec do_start_cowboy(atom(), listener_options()) -> {ok, pid()} | {error, any()}. do_start_cowboy(Ref, Opts) -> - #{ip_tuple := IPTuple, port := Port, handlers := Modules, + #{ip_tuple := IPTuple, port := Port, handlers := Handlers, transport := TransportOpts0, protocol := ProtocolOpts0} = Opts, - Dispatch = cowboy_router:compile(get_routes(Modules)), + Routes = mongoose_http_handler:get_routes(Handlers), + Dispatch = cowboy_router:compile(Routes), ProtocolOpts = ProtocolOpts0#{env => #{dispatch => Dispatch}}, TransportOpts = TransportOpts0#{socket_opts => [{port, Port}, {ip, IPTuple}]}, - ok = trails_store(Modules), + store_trails(Routes), case catch start_http_or_https(Opts, Ref, TransportOpts, ProtocolOpts) of {error, {{shutdown, {failed_to_start_child, ranch_acceptors_sup, @@ -200,8 +189,8 @@ add_common_middleware(Map = #{ middlewares := Middlewares }) -> add_common_middleware(Map) -> Map#{ middlewares => [cowboy_router, ?MODULE, cowboy_handler] }. -reload_dispatch(Ref, #{handlers := Modules}) -> - Dispatch = cowboy_router:compile(get_routes(Modules)), +reload_dispatch(Ref, #{handlers := Handlers}) -> + Dispatch = cowboy_router:compile(mongoose_http_handler:get_routes(Handlers)), cowboy:set_env(Ref, dispatch, Dispatch). stop_cowboy(Ref) -> @@ -213,49 +202,6 @@ ref(Listener) -> ModRef = [?MODULE_STRING, <<"_">>, Ref], list_to_atom(binary_to_list(iolist_to_binary(ModRef))). -%% @doc Cowboy will search for a matching Host, then for a matching Path. If no -%% Path matches, Cowboy will not search for another matching Host. So, we must -%% merge all Paths for each Host, add any wildcard Paths to each Host, and -%% ensure that the wildcard Host is listed last. A dict would be slightly -%% easier to use here, but a proplist ensures that the user can influence Host -%% ordering if other wildcards like "[...]" are used. -get_routes(Modules) -> - Routes = get_routes(Modules, []), - WildcardPaths = proplists:get_value('_', Routes, []), - Merge = fun(Paths) -> Paths ++ WildcardPaths end, - Merged = lists:keymap(Merge, 2, proplists:delete('_', Routes)), - Final = Merged ++ [{'_', WildcardPaths}], - ?LOG_DEBUG(#{what => configured_cowboy_routes, routes => Final}), - Final. - -get_routes([], Routes) -> - Routes; -get_routes([{Host, BasePath, Module} | Tail], Routes) -> - get_routes([{Host, BasePath, Module, []} | Tail], Routes); -get_routes([{Host, BasePath, Module, Opts} | Tail], Routes) -> - %% "_" is used in TOML and translated to '_' here. - CowboyHost = case Host of - "_" -> '_'; - _ -> Host - end, - ensure_loaded_module(Module), - Paths = proplists:get_value(CowboyHost, Routes, []) ++ - case erlang:function_exported(Module, cowboy_router_paths, 2) of - true -> Module:cowboy_router_paths(BasePath, Opts); - _ -> [{BasePath, Module, Opts}] - end, - get_routes(Tail, lists:keystore(CowboyHost, 1, Routes, {CowboyHost, Paths})). - -ensure_loaded_module(Module) -> - case code:ensure_loaded(Module) of - {module, Module} -> - ok; - Other -> - erlang:error(#{issue => ensure_loaded_module_failed, - modue => Module, - reason => Other}) - end. - maybe_set_verify_fun(SSLOptions) -> case lists:keytake(verify_mode, 1, SSLOptions) of false -> @@ -273,27 +219,16 @@ maybe_set_verify_fun(SSLOptions) -> %% The modules must be added into `mongooseim.toml' in the `swagger' section. %% @end %% ------------------------------------------------------------------- -trails_store(Modules) -> +store_trails(Routes) -> + AllModules = lists:usort(lists:flatmap(fun({_Host, HostRoutes}) -> + [Module || {_Path, Module, _Opts} <- HostRoutes] + end, Routes)), + TrailModules = lists:filter(fun(Module) -> + backend_module:is_exported(Module, trails, 0) + end, AllModules), try - trails:store(trails:trails(collect_trails(Modules, []))) - catch Class:Reason -> - ?LOG_WARNING(#{what => caught_exception, class => Class, reason => Reason}) + trails:store(trails:trails(TrailModules)) + catch Class:Reason:Stacktrace -> + ?LOG_WARNING(#{what => store_trails_failed, + class => Class, reason => Reason, stacktrace => Stacktrace}) end. - -%% ------------------------------------------------------------------- -%% @private -%% @doc -%% Helper of store trails for collect trails modules -%% @end -%% ------------------------------------------------------------------- -collect_trails([], Acc) -> - Acc; -collect_trails([{Host, BasePath, Module} | T], Acc) -> - collect_trails([{Host, BasePath, Module, []} | T], Acc); -collect_trails([{_, _, Module, _} | T], Acc) -> - case erlang:function_exported(Module, trails, 0) of - true -> - collect_trails(T, [Module | Acc]); - _ -> - collect_trails(T, Acc) - end. From f61348a5e53cf9d4755fea7892c4a64d80666c8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chrz=C4=85szcz?= Date: Mon, 25 Apr 2022 16:37:05 +0200 Subject: [PATCH 14/20] Update handler collection for system metrics --- .../mongoose_system_metrics_collector.erl | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/system_metrics/mongoose_system_metrics_collector.erl b/src/system_metrics/mongoose_system_metrics_collector.erl index 06ddc60408a..dad0ee4dbd8 100644 --- a/src/system_metrics/mongoose_system_metrics_collector.erl +++ b/src/system_metrics/mongoose_system_metrics_collector.erl @@ -126,11 +126,8 @@ get_api() -> [#{report_name => http_api, key => Api, value => enabled} || Api <- ApiList]. filter_unknown_api(ApiList) -> - AllowedToReport = [ mongoose_api, mongoose_client_api_rooms_messages, - mongoose_client_api_rooms_users, mongoose_client_api_rooms_config, - mongoose_client_api_rooms, mongoose_client_api_contacts, - mongoose_client_api_messages, lasse_handler, mongoose_api_admin, - mod_bosh, mod_websockets], + AllowedToReport = [mongoose_api, mongoose_client_api, mongoose_api_admin, mongoose_api_client, + mongoose_domain_handler, mod_bosh, mod_websockets], [Api || Api <- ApiList, lists:member(Api, AllowedToReport)]. get_transport_mechanisms() -> @@ -143,15 +140,15 @@ get_transport_mechanisms() -> get_http_handler_modules() -> Listeners = get_listeners(ejabberd_cowboy), - Modules = lists:flatten([Modules || #{handlers := Modules} <- Listeners]), - % Modules Option can have variable number of elements. To be more - % error-proof, extracting 3rd element instead of pattern matching. - lists:usort(lists:map(fun(Module) -> element(3, Module) end, Modules)). + lists:usort(lists:flatmap(fun get_http_handler_modules/1, Listeners)). get_listeners(Module) -> Listeners = mongoose_config:get_opt(listen), lists:filter(fun(#{module := Mod}) -> Mod =:= Module end, Listeners). +get_http_handler_modules(#{handlers := Handlers}) -> + [Module || #{module := Module} <- Handlers]. + get_tls_options() -> TLSOptions = lists:flatmap(fun extract_tls_options/1, get_listeners(ejabberd_c2s)), [#{report_name => tls_option, key => TLSMode, value => TLSModule} || From aa71aef4842dc1772f4060fd37ba8e7ece5cf535 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chrz=C4=85szcz?= Date: Mon, 25 Apr 2022 16:43:33 +0200 Subject: [PATCH 15/20] Use the new, simplified format for mongoose_client_api --- rel/files/mongooseim.toml | 46 +++------------------------------------ 1 file changed, 3 insertions(+), 43 deletions(-) diff --git a/rel/files/mongooseim.toml b/rel/files/mongooseim.toml index 129c0b00ef2..4b702ecee0d 100644 --- a/rel/files/mongooseim.toml +++ b/rel/files/mongooseim.toml @@ -59,6 +59,7 @@ [[listen.http.handlers.mongoose_api_admin]] host = "localhost" path = "/api" + [[listen.http.handlers.mongoose_domain_handler]] host = "localhost" path = "/api" @@ -74,49 +75,9 @@ {{{https_config}}} {{/https_config}} - [[listen.http.handlers.lasse_handler]] - host = "_" - path = "/api/sse" - module = "mongoose_client_api_sse" - - [[listen.http.handlers.mongoose_client_api_messages]] - host = "_" - path = "/api/messages/[:with]" - - [[listen.http.handlers.mongoose_client_api_contacts]] - host = "_" - path = "/api/contacts/[:jid]" - - [[listen.http.handlers.mongoose_client_api_rooms]] - host = "_" - path = "/api/rooms/[:id]" - - [[listen.http.handlers.mongoose_client_api_rooms_config]] - host = "_" - path = "/api/rooms/[:id]/config" - - [[listen.http.handlers.mongoose_client_api_rooms_users]] - host = "_" - path = "/api/rooms/:id/users/[:user]" - - [[listen.http.handlers.mongoose_client_api_rooms_messages]] - host = "_" - path = "/api/rooms/[:id]/messages" - - [[listen.http.handlers.cowboy_swagger_redirect_handler]] + [[listen.http.handlers.mongoose_client_api]] host = "_" - path = "/api-docs" - - [[listen.http.handlers.cowboy_swagger_json_handler]] - host = "_" - path = "/api-docs/swagger.json" - - [[listen.http.handlers.cowboy_static]] - host = "_" - path = "/api-docs/[...]" - type = "priv_dir" - app = "cowboy_swagger" - content_path = "swagger" + path = "/api" [[listen.http]] {{#http_api_old_endpoint}} @@ -128,7 +89,6 @@ [[listen.http.handlers.mongoose_api]] host = "localhost" path = "/api" - handlers = ["mongoose_api_metrics", "mongoose_api_users"] [[listen.c2s]] port = {{{c2s_port}}} From 3be89a93813136c673a220e55a00aae1eaf1ceff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chrz=C4=85szcz?= Date: Mon, 25 Apr 2022 16:45:19 +0200 Subject: [PATCH 16/20] Specify http options in maps in small tests --- test/commands_backend_SUITE.erl | 2 +- test/mod_websockets_SUITE.erl | 11 ++++++----- .../mongooseim.basic.toml | 1 - 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/test/commands_backend_SUITE.erl b/test/commands_backend_SUITE.erl index 98c438cb302..1f2d089e86f 100644 --- a/test/commands_backend_SUITE.erl +++ b/test/commands_backend_SUITE.erl @@ -109,7 +109,7 @@ setup(Module) -> port => ?PORT, ip_tuple => ?IP, proto => tcp, - handlers => [{"localhost", "/api", Module, []}]}, + handlers => [#{host => "localhost", path => "/api", module => Module}]}, ejabberd_cowboy:start_listener(Opts). teardown() -> diff --git a/test/mod_websockets_SUITE.erl b/test/mod_websockets_SUITE.erl index db12a181603..676dd853581 100644 --- a/test/mod_websockets_SUITE.erl +++ b/test/mod_websockets_SUITE.erl @@ -12,7 +12,7 @@ %The 300ms is just an additional overhead -define(IDLE_TIMEOUT, ?NEW_TIMEOUT * 2 + 300). --import(config_parser_helper, [default_config/1]). +-import(config_parser_helper, [config/2, default_config/1]). all() -> ping_tests() ++ subprotocol_header_tests() ++ timeout_tests(). @@ -63,10 +63,11 @@ setup() -> end), %% Start websocket cowboy listening - Handlers = [{"_", "/http-bind", mod_bosh}, - {"_", "/ws-xmpp", mod_websockets, - [{timeout, ?IDLE_TIMEOUT}, - {ping_rate, ?FAST_PING_RATE}]}], + Handlers = [config([listen, http, handlers, mod_bosh], + #{host => '_', path => "/http-bind"}), + config([listen, http, handlers, mod_websockets], + #{host => '_', path => "/ws-xmpp", + timeout => ?IDLE_TIMEOUT, ping_rate => ?FAST_PING_RATE})], ejabberd_cowboy:start_listener(#{port => ?PORT, ip_tuple => ?IP, ip_address => "127.0.0.1", diff --git a/test/mongoose_listener_SUITE_data/mongooseim.basic.toml b/test/mongoose_listener_SUITE_data/mongooseim.basic.toml index eabbdfcf4a5..ccd65946f54 100644 --- a/test/mongoose_listener_SUITE_data/mongooseim.basic.toml +++ b/test/mongoose_listener_SUITE_data/mongooseim.basic.toml @@ -16,7 +16,6 @@ [[listen.http.handlers.mongoose_api]] host = "localhost" path = "/api" - handlers = ["mongoose_api_metrics", "mongoose_api_users"] [[listen.c2s]] port = 5222 From 22ebfd7f906e7b7a8b9084a128d27a4d41b2eccd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chrz=C4=85szcz?= Date: Mon, 25 Apr 2022 16:47:21 +0200 Subject: [PATCH 17/20] Update small tests for HTTP handler config options - Use maps with defaults - Add new options - Rework test layout after reworking mongoose_client_api config - Remove the custom logic for comparing handler tuples --- test/common/config_parser_helper.erl | 86 +++++----- test/config_parser_SUITE.erl | 147 ++++++++---------- .../mongooseim-pgsql.toml | 45 +----- 3 files changed, 118 insertions(+), 160 deletions(-) diff --git a/test/common/config_parser_helper.erl b/test/common/config_parser_helper.erl index f6e7bc26a0c..318a88ffdc1 100644 --- a/test/common/config_parser_helper.erl +++ b/test/common/config_parser_helper.erl @@ -65,12 +65,14 @@ options("miscellaneous") -> {listen, [config([listen, http], #{port => 5280, - handlers => [{"_", "/ws-xmpp", mod_websockets, - [{service, maps:merge(extra_service_listener_config(), + handlers => + [config([listen, http, handlers, mod_websockets], + #{host => '_', path => "/ws-xmpp", + service => maps:merge(extra_service_listener_config(), #{password => "secret", shaper_rule => fast, - max_fsm_queue => 1000})}] - }], + max_fsm_queue => 1000})} + )], transport => #{num_acceptors => 10, max_connections => 1024} })]}, {loglevel, warning}, @@ -145,23 +147,31 @@ options("mongooseim-pgsql") -> }), config([listen, http], #{port => 5280, - handlers => [{"_", "/http-bind", mod_bosh, []}, - {"_", "/ws-xmpp", mod_websockets, - [{service, maps:merge(extra_service_listener_config(), - #{password => "secret", shaper_rule => fast})}] - }], + handlers => + [config([listen, http, handlers, mod_bosh], + #{host => '_', path => "/http-bind"}), + config([listen, http, handlers, mod_websockets], + #{host => '_', path => "/ws-xmpp", + service => maps:merge(extra_service_listener_config(), + #{password => "secret", shaper_rule => fast}) + }) + ], transport => #{num_acceptors => 10, max_connections => 1024} }), config([listen, http], #{port => 5285, - handlers => [{"_", "/http-bind", mod_bosh, []}, - {"_", "/ws-xmpp", mod_websockets, - [{max_stanza_size, 100}, - {ping_rate, 120000}, - {timeout, infinity}]}, - {"localhost", "/api", mongoose_api_admin, - [{auth, {<<"ala">>, <<"makotaipsa">>}}]}, - {"localhost", "/api/contacts/{:jid}", mongoose_api_client, []}], + handlers => + [config([listen, http, handlers, mod_bosh], + #{host => '_', path => "/http-bind"}), + config([listen, http, handlers, mod_websockets], + #{host => '_', path => "/ws-xmpp", max_stanza_size => 100, + ping_rate => 120000, timeout => infinity}), + config([listen, http, handlers, mongoose_api_admin], + #{host => "localhost", path => "/api", + username => <<"ala">>, password => <<"makotaipsa">>}), + config([listen, http, handlers, mongoose_api_client], + #{host => "localhost", path => "/api/contacts/{:jid}"}) + ], transport => #{num_acceptors => 10, max_connections => 1024}, tls => [{certfile, "priv/cert.pem"}, {keyfile, "priv/dc1.pem"}, {password, []}] }), @@ -170,25 +180,15 @@ options("mongooseim-pgsql") -> ip_tuple => {127, 0, 0, 1}, port => 8088, transport => #{num_acceptors => 10, max_connections => 1024}, - handlers => [{"localhost", "/api", mongoose_api_admin, []}] + handlers => + [config([listen, http, handlers, mongoose_api_admin], + #{host => "localhost", path => "/api"})] }), config([listen, http], #{port => 8089, - handlers => [{"_", "/api-docs/[...]", cowboy_static, - {priv_dir, cowboy_swagger, "swagger", - [{mimetypes, cow_mimetypes, all}]}}, - {"_", "/api-docs/swagger.json", cowboy_swagger_json_handler, #{}}, - {"_", "/api-docs", cowboy_swagger_redirect_handler, #{}}, - {"_", "/api/sse", lasse_handler, [mongoose_client_api_sse]}, - {"_", "/api/contacts/[:jid]", mongoose_client_api_contacts, []}, - {"_", "/api/messages/[:with]", mongoose_client_api_messages, []}, - {"_", "/api/rooms/[:id]", mongoose_client_api_rooms, []}, - {"_", "/api/rooms/[:id]/config", - mongoose_client_api_rooms_config, []}, - {"_", "/api/rooms/[:id]/messages", - mongoose_client_api_rooms_messages, []}, - {"_", "/api/rooms/:id/users/[:user]", - mongoose_client_api_rooms_users, []}], + handlers => + [config([listen, http, handlers, mongoose_client_api], + #{host => '_', path => "/api"})], protocol => #{compress => true}, transport => #{num_acceptors => 10, max_connections => 1024}, tls => [{certfile, "priv/cert.pem"}, {keyfile, "priv/dc1.pem"}, {password, []}] @@ -199,8 +199,8 @@ options("mongooseim-pgsql") -> port => 5288, transport => #{num_acceptors => 10, max_connections => 1024}, handlers => - [{"localhost", "/api", mongoose_api, - [{handlers, [mongoose_api_metrics, mongoose_api_users]}]}] + [config([listen, http, handlers, mongoose_api], + #{host => "localhost", path => "/api"})] }), config([listen, s2s], #{port => 5269, @@ -1078,6 +1078,22 @@ default_config([listen, http]) -> transport => default_config([listen, http, transport]), protocol => default_config([listen, http, protocol]), handlers => []}; +default_config([listen, http, handlers, mod_websockets]) -> + #{timeout => 60000, + max_stanza_size => infinity, + module => mod_websockets}; +default_config([listen, http, handlers, mongoose_client_api]) -> + #{handlers => [lasse_handler, mongoose_client_api_messages, + mongoose_client_api_contacts, mongoose_client_api_rooms, + mongoose_client_api_rooms_config, mongoose_client_api_rooms_users, + mongoose_client_api_rooms_messages], + docs => true, + module => mongoose_client_api}; +default_config([listen, http, handlers, mongoose_api]) -> + #{handlers => [mongoose_api_metrics, mongoose_api_users], + module => mongoose_api}; +default_config([listen, http, handlers, Module]) -> + #{module => Module}; default_config([listen, http, transport]) -> #{num_acceptors => 100, max_connections => 1024}; diff --git a/test/config_parser_SUITE.erl b/test/config_parser_SUITE.erl index 1eb959c5bac..7041a964991 100644 --- a/test/config_parser_SUITE.erl +++ b/test/config_parser_SUITE.erl @@ -92,11 +92,13 @@ groups() -> listen_http, listen_http_tls, listen_http_transport, - listen_http_handlers, + listen_http_handlers_invalid, + listen_http_handlers_bosh, listen_http_handlers_websockets, - listen_http_handlers_lasse, - listen_http_handlers_static, + listen_http_handlers_client_api, listen_http_handlers_api, + listen_http_handlers_api_admin, + listen_http_handlers_api_client, listen_http_handlers_domain]}, {auth, [parallel], [auth_methods, auth_password, @@ -633,73 +635,72 @@ listen_http_protocol(_Config) -> ?cfg(P ++ [compress], true, T(#{<<"compress">> => true})), ?err(T(#{<<"compress">> => 1})). -listen_http_handlers(_Config) -> +listen_http_handlers_invalid(_Config) -> T = fun(Opts) -> listen_raw(http, #{<<"port">> => 5280, <<"handlers">> => Opts}) end, - P = [listen, 1, handlers], - ?cfg(P, [{"_", "/http-bind", mod_bosh, []}], - T(#{<<"mod_bosh">> => [#{<<"host">> => <<"_">>, - <<"path">> => <<"/http-bind">>}]})), ?err(T(#{<<"mod_bosch">> => [#{<<"host">> => <<"dishwasher">>, - <<"path">> => <<"/cutlery">>}]})), - ?err(T(#{<<"mod_bosh">> => [#{<<"host">> => <<"pathless">>}]})), - ?err(T(#{<<"mod_bosh">> => [#{<<"host">> => <<>>, <<"path">> => <<"/">>}]})), - ?err(T(#{<<"mod_bosh">> => [#{<<"path">> => <<"hostless">>}]})). + <<"path">> => <<"/cutlery">>}]})). + +listen_http_handlers_bosh(_Config) -> + test_listen_http_handler(mod_bosh). listen_http_handlers_websockets(_Config) -> - T = fun(Opts) -> http_handler_raw(<<"mod_websockets">>, Opts) end, - P = [listen, 1, handlers], - ?cfg(P, [{"localhost", "/api", mod_websockets, []}], T(#{})), - ?cfg(P, [{"localhost", "/api", mod_websockets, - [{service, maps:merge(extra_service_listener_config(), #{password => "secret"})}] - }], + {P, T} = test_listen_http_handler(mod_websockets), + ?cfg(P ++ [timeout], 30000, T(#{<<"timeout">> => 30000})), + ?cfg(P ++ [ping_rate], 20, T(#{<<"ping_rate">> => 20})), + ?cfg(P ++ [max_stanza_size], 10000, T(#{<<"max_stanza_size">> => 10000})), + ?cfg(P ++ [service], maps:merge(extra_service_listener_config(), #{password => "secret"}), T(#{<<"service">> => #{<<"password">> => <<"secret">>}})), + ?err(T(#{<<"timeout">> => -1})), + ?err(T(#{<<"ping_rate">> => 0})), + ?err(T(#{<<"max_stanza_size">> => 0})), ?err(T(#{<<"service">> => #{}})). -listen_http_handlers_lasse(_Config) -> - T = fun(Opts) -> http_handler_raw(<<"lasse_handler">>, Opts) end, - P = [listen, 1, handlers], - ?cfg(P, [{"localhost", "/api", lasse_handler, [mongoose_client_api_sse]}], - T(#{<<"module">> => <<"mongoose_client_api_sse">>})), - ?err(T(#{<<"module">> => <<"mooongooose_api_ssie">>})), - ?err(T(#{})). - -listen_http_handlers_static(_Config) -> - T = fun(Opts) -> http_handler_raw(<<"cowboy_static">>, Opts) end, - P = [listen, 1, handlers], - ?cfg(P, [{"localhost", "/api", cowboy_static, - {priv_dir, cowboy_swagger, "swagger", - [{mimetypes, cow_mimetypes, all}]} - }], - T(#{<<"type">> => <<"priv_dir">>, <<"app">> => <<"cowboy_swagger">>, - <<"content_path">> => <<"swagger">>})), - ?cfg(P, [{"localhost", "/api", cowboy_static, - {file, "swagger", [{mimetypes, cow_mimetypes, all}]} - }], - T(#{<<"type">> => <<"file">>, <<"content_path">> => <<"swagger">>})), - ?err(T(#{<<"type">> => <<"priv_dir">>, <<"app">> => <<"cowboy_swagger">>})). +listen_http_handlers_client_api(_Config) -> + {P, T} = test_listen_http_handler(mongoose_client_api), + ?cfg(P ++ [handlers], [mongoose_client_api_messages], + T(#{<<"handlers">> => [<<"mongoose_client_api_messages">>]})), + ?cfg(P ++ [docs], false, T(#{<<"docs">> => false})), + ?err(T(#{<<"handlers">> => [not_a_module]})), + ?err(T(#{<<"docs">> => <<"maybe">>})). listen_http_handlers_api(_Config) -> - T = fun(Opts) -> http_handler_raw(<<"mongoose_api">>, Opts) end, - P = [listen, 1, handlers], - ?cfg(P, [{"localhost", "/api", mongoose_api, - [{handlers, [mongoose_api_metrics, - mongoose_api_users]}]} - ], - T(#{<<"handlers">> => [<<"mongoose_api_metrics">>, <<"mongoose_api_users">>]})), - ?err(T(#{<<"handlers">> => [<<"not_an_api_module">>]})), - ?err(T(#{})). + {P, T} = test_listen_http_handler(mongoose_api), + ?cfg(P ++ [handlers], [mongoose_api_metrics], + T(#{<<"handlers">> => [<<"mongoose_api_metrics">>]})), + ?err(T(#{<<"handlers">> => [not_a_module]})). + +listen_http_handlers_api_admin(_Config) -> + {P, T} = test_listen_http_handler(mongoose_api_admin), + test_listen_http_handler_creds(P, T). + +listen_http_handlers_api_client(_Config) -> + test_listen_http_handler(mongoose_api_client). listen_http_handlers_domain(_Config) -> - T = fun(Opts) -> http_handler_raw(<<"mongoose_domain_handler">>, Opts) end, - P = [listen, 1, handlers], - ?cfg(P, [{"localhost", "/api", mongoose_domain_handler, - [{password, <<"cool">>}, {username, <<"admin">>}] - }], - T(#{<<"username">> => <<"admin">>, <<"password">> => <<"cool">>})), - ?cfg(P, [{"localhost", "/api", mongoose_domain_handler, []}], T(#{})), + {P, T} = test_listen_http_handler(mongoose_domain_handler), + test_listen_http_handler_creds(P, T). + +test_listen_http_handler_creds(P, T) -> + CredsRaw = #{<<"username">> => <<"user">>, <<"password">> => <<"pass">>}, + ?cfg(P ++ [username], <<"user">>, T(CredsRaw)), + ?cfg(P ++ [password], <<"pass">>, T(CredsRaw)), %% Both username and password required. Or none. - ?err(T(#{<<"username">> => <<"admin">>})), - ?err(T(#{<<"password">> => <<"cool">>})). + [?err(T(maps:remove(Key, CredsRaw))) || Key <- maps:keys(CredsRaw)], + ?err(CredsRaw#{<<"username">> => 1}), + ?err(CredsRaw#{<<"password">> => 1}). + +test_listen_http_handler(Module) -> + T = fun(Opts) -> http_handler_raw(Module, Opts) end, + P = [listen, 1, handlers, 1], + ?cfg(P, config([listen, http, handlers, Module], #{host => "localhost", path => "/api"}), + T(#{})), + ?cfg(P ++ [host], '_', T(#{<<"host">> => <<"_">>})), + ?cfg(P ++ [path], "/my-path", T(#{<<"path">> => <<"/my-path">>})), + ?err(T(#{<<"host">> => <<>>})), + ?err(T(#{<<"host">> => undefined})), + ?err(T(#{<<"path">> => 12})), + ?err(T(#{<<"path">> => undefined})), + {P, T}. test_listen(P, T) -> ?cfg(P ++ [ip_address], "192.168.1.16", T(#{<<"ip_address">> => <<"192.168.1.16">>})), @@ -3051,11 +3052,10 @@ listener(Type, Opts) -> config([listen, Type], Opts). http_handler_raw(Type, Opts) -> + MergedOpts = maps:merge(#{<<"host">> => <<"localhost">>, <<"path">> => <<"/api">>}, Opts), listen_raw(http, #{<<"port">> => 5280, - <<"handlers">> => #{Type => - [Opts#{<<"host">> => <<"localhost">>, - <<"path">> => <<"/api">>}] - }}). + <<"handlers">> => #{atom_to_binary(Type) => [remove_undefined(MergedOpts)]}} + ). listen_raw(Type, Opts) -> #{<<"listen">> => #{atom_to_binary(Type) => [remove_undefined(Opts)]}}. @@ -3196,7 +3196,7 @@ handle_config_option(Opt1, Opt2) -> -spec compare_nodes(mongoose_config:key_path(), mongoose_config:value(), mongoose_config:value()) -> any(). compare_nodes([listen], V1, V2) -> - compare_unordered_lists(V1, V2, fun handle_listener/2); + compare_ordered_lists(V1, V2, fun handle_listener/2); compare_nodes([outgoing_pools], V1, V2) -> compare_unordered_lists(V1, V2, fun handle_conn_pool/2); compare_nodes([{auth_method, _}], V1, V2) when is_atom(V1) -> @@ -3214,32 +3214,15 @@ compare_nodes(Node, V1, V2) -> %% Comparisons of internal config option parts handle_listener(V1, V2) -> + ct:pal("Listeners: ~p~n~p", [V1,V2]), compare_maps(V1, V2, fun handle_listener_option/2). handle_listener_option({tls, O1}, {tls, O2}) -> compare_unordered_lists(O1, O2); handle_listener_option({handlers, M1}, {handlers, M2}) -> - compare_unordered_lists(M1, M2, fun handle_listener_module/2); + compare_ordered_lists(M1, M2, fun compare_maps/2); handle_listener_option(V1, V2) -> ?eq(V1, V2). -handle_listener_module({H1, P1, M1}, M2) -> - handle_listener_module({H1, P1, M1, []}, M2); -handle_listener_module({H1, P1, M1, O1}, {H2, P2, M2, O2}) -> - ?eq(H1, H2), - ?eq(P1, P2), - ?eq(M1, M2), - compare_listener_module_options(M1, O1, O2). - -compare_listener_module_options(mod_websockets, L1, L2) -> - E1 = proplists:get_value(ejabberd_service, L1, []), - E2 = proplists:get_value(ejabberd_service, L2, []), - T1 = proplists:delete(ejabberd_service, L1), - T2 = proplists:delete(ejabberd_service, L2), - compare_unordered_lists(E1, E2), - compare_unordered_lists(T1, T2); -compare_listener_module_options(_, O1, O2) -> - ?eq(O1, O2). - handle_item_with_opts({M1, O1}, {M2, O2}) -> ?eq(M1, M2), compare_unordered_lists(O1, O2). diff --git a/test/config_parser_SUITE_data/mongooseim-pgsql.toml b/test/config_parser_SUITE_data/mongooseim-pgsql.toml index eb25fcdd499..d27e72daa23 100644 --- a/test/config_parser_SUITE_data/mongooseim-pgsql.toml +++ b/test/config_parser_SUITE_data/mongooseim-pgsql.toml @@ -78,49 +78,9 @@ tls.keyfile = "priv/dc1.pem" tls.password = "" - [[listen.http.handlers.lasse_handler]] + [[listen.http.handlers.mongoose_client_api]] host = "_" - path = "/api/sse" - module = "mongoose_client_api_sse" - - [[listen.http.handlers.mongoose_client_api_messages]] - host = "_" - path = "/api/messages/[:with]" - - [[listen.http.handlers.mongoose_client_api_contacts]] - host = "_" - path = "/api/contacts/[:jid]" - - [[listen.http.handlers.mongoose_client_api_rooms]] - host = "_" - path = "/api/rooms/[:id]" - - [[listen.http.handlers.mongoose_client_api_rooms_config]] - host = "_" - path = "/api/rooms/[:id]/config" - - [[listen.http.handlers.mongoose_client_api_rooms_users]] - host = "_" - path = "/api/rooms/:id/users/[:user]" - - [[listen.http.handlers.mongoose_client_api_rooms_messages]] - host = "_" - path = "/api/rooms/[:id]/messages" - - [[listen.http.handlers.cowboy_swagger_redirect_handler]] - host = "_" - path = "/api-docs" - - [[listen.http.handlers.cowboy_swagger_json_handler]] - host = "_" - path = "/api-docs/swagger.json" - - [[listen.http.handlers.cowboy_static]] - host = "_" - path = "/api-docs/[...]" - type = "priv_dir" - app = "cowboy_swagger" - content_path = "swagger" + path = "/api" [[listen.http]] port = 5288 @@ -131,7 +91,6 @@ [[listen.http.handlers.mongoose_api]] host = "localhost" path = "/api" - handlers = ["mongoose_api_metrics", "mongoose_api_users"] [[listen.c2s]] port = 5222 From aac7f26de8c164f2c8b6e5ccf86beb1a0847c42b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chrz=C4=85szcz?= Date: Mon, 25 Apr 2022 16:51:31 +0200 Subject: [PATCH 18/20] Update big test helpers after the changes in handler options --- big_tests/tests/domain_rest_helper.erl | 7 ++-- big_tests/tests/rest_helper.erl | 46 +++++++++++--------------- 2 files changed, 24 insertions(+), 29 deletions(-) diff --git a/big_tests/tests/domain_rest_helper.erl b/big_tests/tests/domain_rest_helper.erl index 93b2eebf6b1..c85268ec11a 100644 --- a/big_tests/tests/domain_rest_helper.erl +++ b/big_tests/tests/domain_rest_helper.erl @@ -114,9 +114,10 @@ listener_opts(Params) -> transport => config([listen, http, transport], #{num_acceptors => 10})}). domain_handler(Params) -> - {"localhost", "/api", mongoose_domain_handler, handler_opts(Params)}. + maps:merge(#{host => "localhost", path => "/api", module => mongoose_domain_handler}, + handler_opts(Params)). handler_opts(#{skip_auth := true}) -> - []; + #{}; handler_opts(_Params) -> - [{password, <<"secret">>}, {username, <<"admin">>}]. + #{password => <<"secret">>, username => <<"admin">>}. diff --git a/big_tests/tests/rest_helper.erl b/big_tests/tests/rest_helper.erl index 7a99b16c987..37fbf3b7229 100644 --- a/big_tests/tests/rest_helper.erl +++ b/big_tests/tests/rest_helper.erl @@ -277,36 +277,30 @@ start_admin_listener(Creds) -> NewOpts = insert_creds(Opts, Creds), rpc(mim(), mongoose_listener, start_listener, [NewOpts]). -insert_creds(Opts = #{handlers := Modules}, Creds) -> - {Host, Path, mongoose_api_admin, PathOpts} = lists:keyfind(mongoose_api_admin, 3, Modules), - NewPathOpts = inject_creds_to_opts(PathOpts, Creds), - NewModules = lists:keyreplace(mongoose_api_admin, 3, Modules, - {Host, Path, mongoose_api_admin, NewPathOpts}), - Opts#{handlers := NewModules}. - -inject_creds_to_opts(PathOpts, any) -> - lists:keydelete(auth, 1, PathOpts); -inject_creds_to_opts(PathOpts, Creds) -> - case lists:keymember(auth, 1, PathOpts) of - true -> - lists:keyreplace(auth, 1, PathOpts, {auth, Creds}); - false -> - lists:append(PathOpts, [{auth, Creds}]) - end. +insert_creds(Opts = #{handlers := Handlers}, Creds) -> + NewHandlers = [inject_creds_to_opts(Handler, Creds) || Handler <- Handlers], + Opts#{handlers := NewHandlers}. + +inject_creds_to_opts(Handler = #{module := mongoose_api_admin}, Creds) -> + case Creds of + {UserName, Password} -> + Handler#{username => UserName, password => Password}; + any -> + maps:without([username, password], Handler) + end; +inject_creds_to_opts(Handler, _Creds) -> + Handler. % @doc Checks whether a config for a port is an admin or client one. -% This is determined based on modules used. If there is any mongoose_api_admin module used, -% it is admin config. If not and there is at least one mongoose_api_client* module used, -% it's clients. -is_roles_config(#{module := ejabberd_cowboy, handlers := Modules}, admin) -> - lists:any(fun({_, _Path, Mod, _Args}) -> Mod == mongoose_api_admin; (_) -> false end, Modules); -is_roles_config(#{module := ejabberd_cowboy, handlers := Modules}, client) -> - ModulesTokens = lists:map(fun({_, _Path, Mod, _}) -> string:tokens(atom_to_list(Mod), "_"); - (_) -> [] - end, Modules), - lists:any(fun(["mongoose", "client", "api" | _T]) -> true; (_) -> false end, ModulesTokens); +% This is determined based on handler modules used. +is_roles_config(#{module := ejabberd_cowboy, handlers := Handlers}, Role) -> + RoleModule = role_to_module(Role), + lists:any(fun(#{module := Module}) -> Module =:= RoleModule end, Handlers); is_roles_config(_, _) -> false. +role_to_module(admin) -> mongoose_api_admin; +role_to_module(client) -> mongoose_client_api. + mapfromlist(L) -> Nl = lists:map(fun({K, {V}}) when is_list(V) -> {binary_to_atom(K, utf8), mapfromlist(V)}; From 836745000136707fc5820213833e0317e1262bf8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chrz=C4=85szcz?= Date: Mon, 25 Apr 2022 17:00:57 +0200 Subject: [PATCH 19/20] Update docs for the changed HTTP handlers - Simplify examples to match the defaults in mongooseim.toml - Document the new options: `handlers` and `docs` - Update the list of available handlers --- doc/configuration/listen.md | 84 +++++++++++++---------------------- doc/migrations/5.0.0_5.1.0.md | 4 ++ 2 files changed, 34 insertions(+), 54 deletions(-) diff --git a/doc/configuration/listen.md b/doc/configuration/listen.md index 4942e3795c2..b4954676912 100644 --- a/doc/configuration/listen.md +++ b/doc/configuration/listen.md @@ -420,16 +420,16 @@ Recommended port number: 5280 for BOSH/WS. There are the following options for each of the HTTP listeners: ### `listen.http.handlers` -* **Syntax:** each handler is specified in a subsection starting with `[[listen.http.handlers.type]]` where `type` is one of the allowed handler types, handling different connection types, e.g. +* **Syntax:** each handler is specified in a subsection starting with `[[listen.http.handlers.type]]` where `type` is one of the allowed handler types, handling different connection types: * `mod_bosh` - for [BOSH](https://xmpp.org/extensions/xep-0124.html) connections, * `mod_websockets` - for [WebSocket](https://tools.ietf.org/html/rfc6455) connections, - * `mongoose_api_*`, `mongoose_client_api_*`, ... - for REST API. + * `mongoose_api_admin`, `mongoose_api_client`(obsolete), `mongoose_client_api`, `mongoose_domain_handler`, `mongoose_api` - for REST API. These types are described below in more detail. The double-bracket syntax is used because there can be multiple handlers of a given type, so for each type there is a TOML array of one or more tables (subsections). -* **Default:** there is no default, all handlers need to be specified explicitly. +* **Default:** `[]` - no handlers enabled, all of them need to be specified explicitly. * **Example:** two handlers, one for BOSH and one for WebSockets ```toml [[listen.http.handlers.mod_bosh]] @@ -459,10 +459,12 @@ Path for this handler. ### Handler types: BOSH - `mod_bosh` +The recommended configuration is shown in [Example 1](#example-1-bosh-and-ws) below. To handle incoming BOSH traffic you need to configure the `mod_bosh` module in the `modules` section as well. ### Handler types: WebSockets - `mod_websockets` +The recommended configuration is shown in [Example 1](#example-1-bosh-and-ws) below. Websocket connections as defined in [RFC 7395](https://tools.ietf.org/html/rfc7395). You can pass the following optional parameters: @@ -490,7 +492,7 @@ Maximum allowed incoming stanza size. This limit is checked **after** the input data parsing, so it does not apply to the input data size itself. #### `listen.http.handlers.mod_websockets.service` -* **Syntax:** an array of `listen.service.*` options +* **Syntax:** a table of `listen.service.*` options * **Default:** not set * **Example:** @@ -506,6 +508,7 @@ See the [service](#xmpp-components-listenservice) listener section for details. ### Handler types: REST API - Admin - `mongoose_api_admin` +The recommended configuration is shown in [Example 2](#example-2-admin-api) below. For more information about the API, see the [REST interface](../rest-api/Administration-backend.md) documentation. The following options are supported for this handler: @@ -523,19 +526,32 @@ When set, enables authentication for the admin API, otherwise it is disabled. Re Required to enable authentication for the admin API. -### Handler types: REST API - Client - -To enable the REST API for clients, several handlers need to be added: - -* `mongoose_client_api_*` - handles individual API endpoints. You can add and remove these to enable particular functionality. -* `lasse_handler` - provides the [SSE handler](https://github.com/inaka/lasse) which is required for the client HTTP API, should not be changed. -* `cowboy_*` - hosts the Swagger web-based documentation, should not be changed, but can be removed to disable the API docs. +### Handler types: REST API - Client - `mongoose_client_api` The recommended configuration is shown in [Example 3](#example-3-client-api) below. Please refer to [REST interface](../rest-api/Client-frontend.md) documentation for more information. +The following options are supported for this handler: + +#### `listen.http.handlers.mongoose_client_api.handlers` +* **Syntax:** array of strings - Erlang modules +* **Default:** all API handler modules enabled +* **Example:** `handlers = ["mongoose_client_api_messages", "mongoose_client_api_sse"]` + +The client API consists of several modules, each of them implementing a subset of the functionality. +By default all modules are enabled, so you don't need to change this option. +For a list of allowed modules, you need to consult the [source code](https://github.com/esl/MongooseIM/blob/master/src/mongoose_client_api/mongoose_client_api.erl). + +#### `listen.http.handlers.mongoose_client_api.docs` +* **Syntax:** boolean +* **Default:** `true` +* **Example:** `docs = "false"` + +The Swagger documentation of the client API is hosted at the `/api-docs` path. +You can disable the hosted documentation by setting this option to `false`. ### Handler types: REST API - Domain management - `mongoose_domain_handler` +The recommended configuration is shown in [Example 4](#example-4-domain-api) below. This handler enables dynamic domain management for different host types. For more information about the API, see the [REST interface](../rest-api/Dynamic-domains.md) documentation. The following options are supported for this handler: @@ -562,7 +578,7 @@ The following option is required: #### `listen.http.handlers.mongoose_api.handlers` * **Syntax:** array of strings - Erlang modules -* **Default:** not set, this is a mandatory option for this handler +* **Default:** all API handler modules enabled * **Example:** `handlers = ["mongoose_api_metrics"]` ### Transport options @@ -720,49 +736,9 @@ REST API for clients. transport.max_connections = 1024 protocol.compress = true - [[listen.http.handlers.lasse_handler]] - host = "_" - path = "/api/sse" - module = "mongoose_client_api_sse" - - [[listen.http.handlers.mongoose_client_api_messages]] + [[listen.http.handlers.mongoose_client_api]] host = "_" - path = "/api/messages/[:with]" - - [[listen.http.handlers.mongoose_client_api_contacts]] - host = "_" - path = "/api/contacts/[:jid]" - - [[listen.http.handlers.mongoose_client_api_rooms]] - host = "_" - path = "/api/rooms/[:id]" - - [[listen.http.handlers.mongoose_client_api_rooms_config]] - host = "_" - path = "/api/rooms/[:id]/config" - - [[listen.http.handlers.mongoose_client_api_rooms_users]] - host = "_" - path = "/api/rooms/:id/users/[:user]" - - [[listen.http.handlers.mongoose_client_api_rooms_messages]] - host = "_" - path = "/api/rooms/[:id]/messages" - - [[listen.http.handlers.cowboy_swagger_redirect_handler]] - host = "_" - path = "/api-docs" - - [[listen.http.handlers.cowboy_swagger_json_handler]] - host = "_" - path = "/api-docs/swagger.json" - - [[listen.http.handlers.cowboy_static]] - host = "_" - path = "/api-docs/[...]" - type = "priv_dir" - app = "cowboy_swagger" - content_path = "swagger" + path = "/api" ``` #### Example 4. Domain API diff --git a/doc/migrations/5.0.0_5.1.0.md b/doc/migrations/5.0.0_5.1.0.md index cdc1d91ca22..b31ec4d1269 100644 --- a/doc/migrations/5.0.0_5.1.0.md +++ b/doc/migrations/5.0.0_5.1.0.md @@ -2,6 +2,10 @@ The configuration format has slightly changed and you might need to amend `mongooseim.toml`. +### Section `listen` + +There is a new, simplified configuration format for `mongoose_client_api`. You need to change the `listen` section unless you have disabled the client API in your configuration file. Consult the [option description](../configuration/listen.md#handler-types-rest-api-client-mongoose_client_api) and the [example configuration](http://localhost:8000/configuration/listen/#example-3-client-api) for details. + ### Section `acl` The implicit check for user's domain in patterns is now configurable and the default behaviour (previously undocumented) is more consistent - the check is always performed unless disabled with `match = "all"`. From eeda6546cec4ae6344acd6f819191b0c741a9a94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chrz=C4=85szcz?= Date: Wed, 27 Apr 2022 10:44:21 +0200 Subject: [PATCH 20/20] Fix indentation --- src/mongoose_client_api/mongoose_client_api.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mongoose_client_api/mongoose_client_api.erl b/src/mongoose_client_api/mongoose_client_api.erl index 95d594de411..bcd9c803355 100644 --- a/src/mongoose_client_api/mongoose_client_api.erl +++ b/src/mongoose_client_api/mongoose_client_api.erl @@ -32,8 +32,8 @@ config_spec() -> HandlerModules = [Module || {_, Module, _} <- api_paths()], #section{items = #{<<"handlers">> => #list{items = #option{type = atom, - validate = {enum, HandlerModules}}, - validate = unique}, + validate = {enum, HandlerModules}}, + validate = unique}, <<"docs">> => #option{type = boolean}}, defaults = #{<<"handlers">> => HandlerModules, <<"docs">> => true},