From bf99f2ae66a51b487e102503fac972b8f849ec34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chrz=C4=85szcz?= Date: Wed, 21 Sep 2022 08:29:28 +0200 Subject: [PATCH 1/2] Include overlay vars from vars-toml.config This way there is only one overlay file, and the overlays work correctly with the latest Rebar3. As we do some templating in our tools, they need to support file inclusion as well. --- big_tests/run_common_test.erl | 25 +++++++++++++------- big_tests/tests/ejabberd_node_utils.erl | 31 ++++++++++++------------- rebar.config | 10 ++++---- rel/fed1.vars-toml.config | 2 ++ rel/mim1.vars-toml.config | 2 ++ rel/mim2.vars-toml.config | 2 ++ rel/mim3.vars-toml.config | 2 ++ rel/reg1.vars-toml.config | 2 ++ tools/test_runner/apply_templates.erl | 19 ++++++++------- 9 files changed, 57 insertions(+), 38 deletions(-) diff --git a/big_tests/run_common_test.erl b/big_tests/run_common_test.erl index 97bc316eb1..b3b6dc6d91 100644 --- a/big_tests/run_common_test.erl +++ b/big_tests/run_common_test.erl @@ -278,22 +278,20 @@ is_test_host_enabled(HostName) -> enable_preset_on_node(Node, PresetVars, HostVarsFilePrefix) -> {ok, Cwd} = call(Node, file, get_cwd, []), TemplatePath = filename:join([repo_dir(), "rel", "files", "mongooseim.toml"]), - DefaultVarsPath = filename:join([repo_dir(), "rel", "vars-toml.config"]), NodeVarsPath = filename:join([repo_dir(), "rel", HostVarsFilePrefix ++ ".vars-toml.config"]), {ok, Template} = handle_file_error(TemplatePath, file:read_file(TemplatePath)), - {ok, DefaultVars} = handle_file_error(DefaultVarsPath, file:consult(DefaultVarsPath)), - {ok, NodeVars} = handle_file_error(NodeVarsPath, file:consult(NodeVarsPath)), + NodeVars = read_vars(NodeVarsPath), - TemplatedConfig = template_config(Template, [DefaultVars, NodeVars, PresetVars]), + TemplatedConfig = template_config(Template, NodeVars ++ PresetVars), CfgPath = filename:join([Cwd, "etc", "mongooseim.toml"]), ok = call(Node, file, write_file, [CfgPath, TemplatedConfig]), call(Node, application, stop, [mongooseim]), call(Node, application, start, [mongooseim]), ok. -template_config(Template, Vars) -> - MergedVars = ensure_binary_strings(merge_vars(Vars)), +template_config(Template, RawVars) -> + MergedVars = ensure_binary_strings(maps:from_list(RawVars)), %% Render twice to replace variables in variables Tmp = bbmustache:render(Template, MergedVars, [{key_type, atom}]), bbmustache:render(Tmp, MergedVars, [{key_type, atom}]). @@ -305,11 +303,20 @@ merge_vars([Vars1, Vars2|Rest]) -> merge_vars([Vars|Rest]); merge_vars([Vars]) -> Vars. +read_vars(File) -> + {ok, Terms} = handle_file_error(File, file:consult(File)), + lists:flatmap(fun({Key, Val}) -> + [{Key, Val}]; + (IncludedFile) when is_list(IncludedFile) -> + Path = filename:join(filename:dirname(File), IncludedFile), + read_vars(Path) + end, Terms). + %% bbmustache tries to iterate over lists, so we need to make them binaries ensure_binary_strings(Vars) -> - lists:map(fun({dbs, V}) -> {dbs, V}; - ({K, V}) when is_list(V) -> {K, list_to_binary(V)}; - ({K, V}) -> {K, V} + maps:map(fun(dbs, V) -> V; + (_K, V) when is_list(V) -> list_to_binary(V); + (_K, V) -> V end, Vars). call(Node, M, F, A) -> diff --git a/big_tests/tests/ejabberd_node_utils.erl b/big_tests/tests/ejabberd_node_utils.erl index eb2b5a1fcb..b3aa0432ef 100644 --- a/big_tests/tests/ejabberd_node_utils.erl +++ b/big_tests/tests/ejabberd_node_utils.erl @@ -154,41 +154,40 @@ modify_config_file(CfgVarsToChange, Config) -> ConfigVariable :: atom(), Value :: string(). modify_config_file(Host, VarsToChange, Config, Format) -> - VarsFile = vars_file(Format), NodeVarsFile = ct:get_config({hosts, Host, vars}, Config) ++ "." ++ vars_file(Format), TemplatePath = config_template_path(Config, Format), - DefaultVarsPath = config_vars_path(VarsFile, Config), NodeVarsPath = config_vars_path(NodeVarsFile, Config), {ok, Template} = file:read_file(TemplatePath), - {ok, DefaultVars} = file:consult(DefaultVarsPath), - {ok, NodeVars} = file:consult(NodeVarsPath), + NodeVars = read_vars(NodeVarsPath), PresetVars = preset_vars(Config, Format), - TemplatedConfig = template_config(Template, [DefaultVars, NodeVars, PresetVars, VarsToChange]), + TemplatedConfig = template_config(Template, NodeVars ++ PresetVars ++ VarsToChange), RPCSpec = distributed_helper:Host(), NewCfgPath = update_config_path(RPCSpec, Format), ok = ejabberd_node_utils:call_fun(RPCSpec, file, write_file, [NewCfgPath, TemplatedConfig]). +read_vars(File) -> + {ok, Terms} = file:consult(File), + lists:flatmap(fun({Key, Val}) -> + [{Key, Val}]; + (IncludedFile) when is_list(IncludedFile) -> + Path = filename:join(filename:dirname(File), IncludedFile), + read_vars(Path) + end, Terms). + template_config(Template, Vars) -> - MergedVars = ensure_binary_strings(merge_vars(Vars)), + MergedVars = ensure_binary_strings(maps:from_list(Vars)), %% Render twice to replace variables in variables Tmp = bbmustache:render(Template, MergedVars, [{key_type, atom}]), bbmustache:render(Tmp, MergedVars, [{key_type, atom}]). -merge_vars([Vars1, Vars2|Rest]) -> - Vars = lists:foldl(fun ({Var, Val}, Acc) -> - lists:keystore(Var, 1, Acc, {Var, Val}) - end, Vars1, Vars2), - merge_vars([Vars|Rest]); -merge_vars([Vars]) -> Vars. - %% bbmustache tries to iterate over lists, so we need to make them binaries ensure_binary_strings(Vars) -> - lists:map(fun({dbs, V}) -> {dbs, V}; - ({K, V}) when is_list(V) -> {K, list_to_binary(V)}; - ({K, V}) -> {K, V} + maps:map(fun(dbs, V) -> V; + (_K, V) when is_list(V) -> list_to_binary(V); + (_K, V) -> V end, Vars). update_config_path(RPCSpec, Format) -> diff --git a/rebar.config b/rebar.config index 2bca5f5351..f1300c1ad5 100644 --- a/rebar.config +++ b/rebar.config @@ -164,15 +164,15 @@ {overlay, [{template, "rel/files/mongooseim.toml", "etc/mongooseim.toml"}]} ]}, {erl_opts, [{d, 'PROD_NODE'}]} ]}, %% development nodes - {mim1, [{relx, [ {overlay_vars, ["rel/vars-toml.config", "rel/mim1.vars-toml.config"]}, + {mim1, [{relx, [ {overlay_vars, "rel/mim1.vars-toml.config"}, {overlay, [{template, "rel/files/mongooseim.toml", "etc/mongooseim.toml"}]} ]}]}, - {mim2, [{relx, [ {overlay_vars, ["rel/vars-toml.config", "rel/mim2.vars-toml.config"]}, + {mim2, [{relx, [ {overlay_vars, "rel/mim2.vars-toml.config"}, {overlay, [{template, "rel/files/mongooseim.toml", "etc/mongooseim.toml"}]} ]}]}, - {mim3, [{relx, [ {overlay_vars, ["rel/vars-toml.config", "rel/mim3.vars-toml.config"]}, + {mim3, [{relx, [ {overlay_vars, "rel/mim3.vars-toml.config"}, {overlay, [{template, "rel/files/mongooseim.toml", "etc/mongooseim.toml"}]} ]}]}, - {fed1, [{relx, [ {overlay_vars, ["rel/vars-toml.config", "rel/fed1.vars-toml.config"]}, + {fed1, [{relx, [ {overlay_vars, "rel/fed1.vars-toml.config"}, {overlay, [{template, "rel/files/mongooseim.toml", "etc/mongooseim.toml"}]} ]}]}, - {reg1, [{relx, [ {overlay_vars, ["rel/vars-toml.config", "rel/reg1.vars-toml.config"]}, + {reg1, [{relx, [ {overlay_vars, "rel/reg1.vars-toml.config"}, {overlay, [{template, "rel/files/mongooseim.toml", "etc/mongooseim.toml"}]} ]}]}, {test, [{extra_src_dirs, [{"test", [{recursive, true}]}]}]} ]}. diff --git a/rel/fed1.vars-toml.config b/rel/fed1.vars-toml.config index 8db4c7981d..78ece4941a 100644 --- a/rel/fed1.vars-toml.config +++ b/rel/fed1.vars-toml.config @@ -1,3 +1,5 @@ +"./vars-toml.config". + {node_name, "fed1@localhost"}. {c2s_port, 5242}. diff --git a/rel/mim1.vars-toml.config b/rel/mim1.vars-toml.config index 2db63aee1d..f85a780b02 100644 --- a/rel/mim1.vars-toml.config +++ b/rel/mim1.vars-toml.config @@ -1,3 +1,5 @@ +"./vars-toml.config". + {c2s_tls_port, 5223}. {outgoing_s2s_port, 5299}. {service_port, 8888}. diff --git a/rel/mim2.vars-toml.config b/rel/mim2.vars-toml.config index a1a08d2dc1..89c144e450 100644 --- a/rel/mim2.vars-toml.config +++ b/rel/mim2.vars-toml.config @@ -1,3 +1,5 @@ +"./vars-toml.config". + {node_name, "ejabberd2@localhost"}. {c2s_port, 5232}. diff --git a/rel/mim3.vars-toml.config b/rel/mim3.vars-toml.config index 422d2de57f..0687b7deac 100644 --- a/rel/mim3.vars-toml.config +++ b/rel/mim3.vars-toml.config @@ -1,3 +1,5 @@ +"./vars-toml.config". + {node_name, "mongooseim3@localhost"}. {c2s_port, 5262}. diff --git a/rel/reg1.vars-toml.config b/rel/reg1.vars-toml.config index 8281f21720..ac14930954 100644 --- a/rel/reg1.vars-toml.config +++ b/rel/reg1.vars-toml.config @@ -1,3 +1,5 @@ +"./vars-toml.config". + {node_name, "reg1@localhost"}. {c2s_port, 5252}. diff --git a/tools/test_runner/apply_templates.erl b/tools/test_runner/apply_templates.erl index 382a21be71..cef57a23d4 100644 --- a/tools/test_runner/apply_templates.erl +++ b/tools/test_runner/apply_templates.erl @@ -20,10 +20,17 @@ main([NodeAtom, BuildDirAtom]) -> overlay_vars(Node) -> - Vars = consult_map("rel/vars-toml.config"), - NodeVars = consult_map("rel/" ++ atom_to_list(Node) ++ ".vars-toml.config"), - %% NodeVars overrides Vars - ensure_binary_strings(maps:merge(Vars, NodeVars)). + File = "rel/" ++ atom_to_list(Node) ++ ".vars-toml.config", + ensure_binary_strings(maps:from_list(read_vars(File))). + +read_vars(File) -> + {ok, Terms} = file:consult(File), + lists:flatmap(fun({Key, Val}) -> + [{Key, Val}]; + (IncludedFile) when is_list(IncludedFile) -> + Path = filename:join(filename:dirname(File), IncludedFile), + read_vars(Path) + end, Terms). %% bbmustache tries to iterate over lists, so we need to make them binaries ensure_binary_strings(Vars) -> @@ -31,10 +38,6 @@ ensure_binary_strings(Vars) -> (_K, V) -> V end, Vars). -consult_map(File) -> - {ok, Vars} = file:consult(File), - maps:from_list(Vars). - %% Based on rebar.config overlay section templates(RelDir) -> simple_templates(RelDir) ++ erts_templates(RelDir). From efee290620055c693240c2c6bf13e36c3a519b60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chrz=C4=85szcz?= Date: Wed, 21 Sep 2022 08:51:49 +0200 Subject: [PATCH 2/2] Get rid of rel/vars-toml.config.in It is easier to modify vars-toml.config directly --- Makefile | 6 +----- rel/{vars-toml.config.in => vars-toml.config} | 4 +++- 2 files changed, 4 insertions(+), 6 deletions(-) rename rel/{vars-toml.config.in => vars-toml.config} (96%) diff --git a/Makefile b/Makefile index 38608568f3..029fe7bea8 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,6 @@ clean: -rm -rf asngen -rm configure.out -rm rel/configure.vars.config - -rm rel/vars-toml.config # REBAR_CT_EXTRA_ARGS comes from a test runner ct: @@ -27,7 +26,7 @@ ct: eunit: @$(RUN) $(REBAR) eunit -rel: certs configure.out rel/vars-toml.config +rel: certs configure.out rel/configure.vars.config . ./configure.out && $(REBAR) as prod release shell: certs etc/mongooseim.cfg @@ -40,9 +39,6 @@ rock: elif [ "$(BRANCH)" ]; then tools/rock_changed.sh $(BRANCH); \ else tools/rock_changed.sh; fi -rel/vars-toml.config: rel/vars-toml.config.in rel/configure.vars.config - cat $^ > $@ - ## Don't allow these files to go out of sync! configure.out rel/configure.vars.config: ./tools/configure with-all without-jingle-sip diff --git a/rel/vars-toml.config.in b/rel/vars-toml.config similarity index 96% rename from rel/vars-toml.config.in rename to rel/vars-toml.config index 62b2d59b2e..8e8d91d8d5 100644 --- a/rel/vars-toml.config.in +++ b/rel/vars-toml.config @@ -55,7 +55,9 @@ {s2s_certfile, "\"priv/ssl/fake_server.pem\""}. {all_metrics_are_global, "false"}. -%% Defined in Makefile by appending configure.vars.config +"./configure.vars.config". + +%% Defined by appending configure.vars.config %% Uncomment for manual release generation. %{mongooseim_runner_user, ""}. %{mongooseim_script_dir, "$(cd ${0%/*} && pwd)"}.