From d4cbade4c6bc119a9c328222492aa19c1e474d8b Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Tue, 14 Nov 2023 14:20:54 +0100 Subject: [PATCH 01/15] Do not wait for CETS to be ready if RDBMS is ready first in mongoose_cluster_id Interrupt other waiting calls in mongoose_cluster_id Report slow wait_for_rdbms call Report slow wait_for_any_backend Use exit with shutdown reason to stop processes in mongoose_cluster_id --- src/mongoose_cluster_id.erl | 50 ++++++++++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 4 deletions(-) diff --git a/src/mongoose_cluster_id.erl b/src/mongoose_cluster_id.erl index 1420314e6fe..82d73997208 100644 --- a/src/mongoose_cluster_id.erl +++ b/src/mongoose_cluster_id.erl @@ -29,6 +29,9 @@ start() -> Backend = which_backend_available(), IntBackend = which_volatile_backend_available(), maybe_prepare_queries(Backend), + cets_long:run_tracked(#{task => wait_for_any_backend, + backend => Backend, volatile_backend => IntBackend}, + fun() -> wait_for_any_backend(Backend, IntBackend) end), CachedRes = get_cached_cluster_id(IntBackend), BackendRes = get_backend_cluster_id(), case {CachedRes, BackendRes} of @@ -47,6 +50,48 @@ start() -> {error, conflict} end. + +%% If RDBMS is available before CETS - it is enough for us to continue +%% the starting procedure +wait_for_any_backend(Backend, IntBackend) -> + Alias = erlang:alias([reply]), + Pids = lists:append([wait_for_backend_promise(B, Alias) || B <- lists:sort([Backend, IntBackend])]), + wait_for_first_reply(Alias), + %% Interrupt other waiting calls to reduce the logging noise + [erlang:exit(Pid, shutdown) || Pid <- Pids], + ok. + +wait_for_first_reply(Alias) -> + receive + {ready, Alias} -> + ok + end. + +wait_for_backend_promise(mnesia, Alias) -> + Alias ! {ready, Alias}, + []; +wait_for_backend_promise(cets, Alias) -> + [spawn(fun() -> + %% We have to do it, because we want to read from across the cluster + %% in the start/0 function. + ok = cets_discovery:wait_for_ready(mongoose_cets_discovery, infinity), + Alias ! {ready, Alias} + end)]; +wait_for_backend_promise(rdbms, Alias) -> + [spawn(fun() -> + cets_long:run_tracked(#{task => wait_for_rdbms}, fun() -> wait_for_rdbms() end), + Alias ! {ready, Alias} + end)]. + +wait_for_rdbms() -> + case get_backend_cluster_id(rdbms) of + {ok, _} -> + ok; + _ -> + timer:sleep(100), + wait_for_rdbms() + end. + %% Get cached version -spec get_cached_cluster_id() -> maybe_cluster_id(). get_cached_cluster_id() -> @@ -102,10 +147,7 @@ init_cache(mnesia) -> ]); init_cache(cets) -> cets:start(cets_cluster_id, #{}), - cets_discovery:add_table(mongoose_cets_discovery, cets_cluster_id), - %% We have to do it, because we want to read from across the cluster - %% in the start/0 function. - ok = cets_discovery:wait_for_ready(mongoose_cets_discovery, infinity). + cets_discovery:add_table(mongoose_cets_discovery, cets_cluster_id). -spec maybe_prepare_queries(mongoose_backend()) -> ok. maybe_prepare_queries(mnesia) -> ok; From 09c8b9d20ca6da19a2a6d2f090f097e92cad5fc1 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Tue, 14 Nov 2023 18:32:24 +0100 Subject: [PATCH 02/15] Put MIM_NODE_IP into RDBMS Fix query specific test for cets Reorder columns to match table, insert, update and select queries Fix graphql_cets_SUITE test --- big_tests/tests/cets_disco_SUITE.erl | 30 ++++++++++---------- big_tests/tests/graphql_cets_SUITE.erl | 7 +++-- priv/mssql2012.sql | 5 ++-- priv/mysql.sql | 5 ++-- priv/pg.sql | 5 ++-- src/mongoose_cets_discovery_rdbms.erl | 38 ++++++++++++++------------ 6 files changed, 49 insertions(+), 41 deletions(-) diff --git a/big_tests/tests/cets_disco_SUITE.erl b/big_tests/tests/cets_disco_SUITE.erl index 8a7acf69074..b3f2a91e138 100644 --- a/big_tests/tests/cets_disco_SUITE.erl +++ b/big_tests/tests/cets_disco_SUITE.erl @@ -144,24 +144,24 @@ rdbms_backend_db_queries(_Config) -> TS2 = TS + 100, %% insertion fails if node name or node num is already added for the cluster - ?assertEqual({updated, 1}, insert_new(CN, <<"test1">>, TS, 1)), - ?assertMatch({error, _}, insert_new(CN, <<"test1">>, TS, 1)), - ?assertMatch({error, _}, insert_new(CN, <<"test1">>, TS, 2)), - ?assertMatch({error, _}, insert_new(CN, <<"test2">>, TS, 1)), - ?assertEqual({updated, 1}, insert_new(CN, <<"test2">>, TS, 2)), + ?assertEqual({updated, 1}, insert_new(CN, <<"test1">>, 1, <<>>, TS)), + ?assertMatch({error, _}, insert_new(CN, <<"test1">>, 1, <<>>, TS)), + ?assertMatch({error, _}, insert_new(CN, <<"test1">>, 2, <<>>, TS)), + ?assertMatch({error, _}, insert_new(CN, <<"test2">>, 1, <<>>, TS)), + ?assertEqual({updated, 1}, insert_new(CN, <<"test2">>, 2, <<>>, TS)), %% update of the timestamp works correctly {selected, SelectedNodes1} = select(CN), - ?assertEqual(lists:sort([{<<"test1">>, 1, TS}, {<<"test2">>, 2, TS}]), + ?assertEqual(lists:sort([{<<"test1">>, 1, <<>>, TS}, {<<"test2">>, 2, <<>>, TS}]), lists:sort(SelectedNodes1)), - ?assertEqual({updated, 1}, update_existing(CN, <<"test1">>, TS2)), + ?assertEqual({updated, 1}, update_existing(CN, <<"test1">>, <<>>, TS2)), {selected, SelectedNodes2} = select(CN), - ?assertEqual(lists:sort([{<<"test1">>, 1, TS2}, {<<"test2">>, 2, TS}]), + ?assertEqual(lists:sort([{<<"test1">>, 1, <<>>, TS2}, {<<"test2">>, 2, <<>>, TS}]), lists:sort(SelectedNodes2)), %% node removal work correctly ?assertEqual({updated, 1}, delete_node_from_db(CN, <<"test1">>)), - ?assertEqual({selected, [{<<"test2">>, 2, TS}]}, select(CN)). + ?assertEqual({selected, [{<<"test2">>, 2, <<>>, TS}]}, select(CN)). %%-------------------------------------------------------------------- %% Helpers @@ -216,9 +216,9 @@ random_cluster_name(CaseName) -> Rand = rpc(mim(), mongoose_bin, gen_from_crypto, []), <<"big_test_", (atom_to_binary(CaseName))/binary, "_", Rand/binary>>. -insert_new(CN, BinNode, TS, NodeNum) -> - Ret = rpc(mim(), mongoose_cets_discovery_rdbms, insert_new, [CN, BinNode, TS, NodeNum]), - ct:log("insert_new(~p, ~p, ~p, ~p) = ~p", [CN, BinNode, TS, NodeNum, Ret]), +insert_new(CN, BinNode, NodeNum, Address, TS) -> + Ret = rpc(mim(), mongoose_cets_discovery_rdbms, insert_new, [CN, BinNode, NodeNum, Address, TS]), + ct:log("insert_new(~p, ~p, ~p, ~p, ~p) = ~p", [CN, BinNode, NodeNum, Address, TS, Ret]), Ret. select(CN) -> @@ -226,9 +226,9 @@ select(CN) -> ct:log("select(~p) = ~p", [CN, Ret]), Ret. -update_existing(CN, BinNode, TS) -> - Ret = rpc(mim(), mongoose_cets_discovery_rdbms, update_existing, [CN, BinNode, TS]), - ct:log("select(~p, ~p, ~p) = ~p", [CN, BinNode, TS, Ret]), +update_existing(CN, BinNode, Address, TS) -> + Ret = rpc(mim(), mongoose_cets_discovery_rdbms, update_existing, [CN, BinNode, Address, TS]), + ct:log("select(~p, ~p, ~p, ~p) = ~p", [CN, BinNode, Address, TS, Ret]), Ret. delete_node_from_db(CN, BinNode) -> diff --git a/big_tests/tests/graphql_cets_SUITE.erl b/big_tests/tests/graphql_cets_SUITE.erl index b22831ec58b..54a30bba692 100644 --- a/big_tests/tests/graphql_cets_SUITE.erl +++ b/big_tests/tests/graphql_cets_SUITE.erl @@ -206,16 +206,17 @@ register_bad_node() -> ClusterName = <<"mim">>, Node = <<"badnode@localhost">>, Num = 100, + Address = <<>>, Timestamp = rpc(mim(), mongoose_rdbms_timestamp, select, []), - InsertArgs = [ClusterName, Node, Num, Timestamp], - {updated, 1} = rpc(mim(), mongoose_rdbms, execute, [global, cets_disco_insert_new, InsertArgs]). + InsertArgs = [ClusterName, Node, Num, Address, Timestamp], + {updated, 1} = rpc(mim(), mongoose_cets_discovery_rdbms, insert_new, InsertArgs). ensure_bad_node_unregistered() -> ClusterName = <<"mim">>, Node = <<"badnode@localhost">>, DeleteArgs = [ClusterName, Node], %% Ensure the node is removed - {updated, _} = rpc(mim(), mongoose_rdbms, execute, [global, cets_delete_node_from_db, DeleteArgs]). + {updated, _} = rpc(mim(), mongoose_cets_discovery_rdbms, delete_node_from_db, DeleteArgs). force_check() -> Pid = rpc(mim(), erlang, whereis, [mongoose_cets_discovery]), diff --git a/priv/mssql2012.sql b/priv/mssql2012.sql index e079ccaec84..a736a694f50 100644 --- a/priv/mssql2012.sql +++ b/priv/mssql2012.sql @@ -754,10 +754,11 @@ CREATE TABLE domain_events ( CREATE INDEX i_domain_events_domain ON domain_events(domain); CREATE TABLE discovery_nodes ( - node_name varchar(250), cluster_name varchar(250), - updated_timestamp BIGINT NOT NULL, -- in seconds + node_name varchar(250), node_num INT NOT NULL, + address varchar(250), + updated_timestamp BIGINT NOT NULL, -- in seconds PRIMARY KEY (cluster_name, node_name) ); CREATE UNIQUE INDEX i_discovery_nodes_node_num ON discovery_nodes(cluster_name, node_num); diff --git a/priv/mysql.sql b/priv/mysql.sql index 2f27ee21faa..841ee188037 100644 --- a/priv/mysql.sql +++ b/priv/mysql.sql @@ -546,10 +546,11 @@ CREATE TABLE domain_events ( CREATE INDEX i_domain_events_domain ON domain_events(domain); CREATE TABLE discovery_nodes ( - node_name varchar(250), cluster_name varchar(250), - updated_timestamp BIGINT NOT NULL, -- in seconds + node_name varchar(250), node_num INT UNSIGNED NOT NULL, + address varchar(250), + updated_timestamp BIGINT NOT NULL, -- in seconds PRIMARY KEY (cluster_name, node_name) ); CREATE UNIQUE INDEX i_discovery_nodes_node_num USING BTREE ON discovery_nodes(cluster_name, node_num); diff --git a/priv/pg.sql b/priv/pg.sql index 646ddd8f032..401d5f26219 100644 --- a/priv/pg.sql +++ b/priv/pg.sql @@ -506,10 +506,11 @@ CREATE TABLE domain_events ( CREATE INDEX i_domain_events_domain ON domain_events(domain); CREATE TABLE discovery_nodes ( - node_name varchar(250), cluster_name varchar(250), - updated_timestamp BIGINT NOT NULL, -- in seconds + node_name varchar(250), node_num INT NOT NULL, + address varchar(250), + updated_timestamp BIGINT NOT NULL, -- in seconds PRIMARY KEY (cluster_name, node_name) ); CREATE UNIQUE INDEX i_discovery_nodes_node_num ON discovery_nodes USING BTREE(cluster_name, node_num); diff --git a/src/mongoose_cets_discovery_rdbms.erl b/src/mongoose_cets_discovery_rdbms.erl index 6708f91277d..6a038328ba0 100644 --- a/src/mongoose_cets_discovery_rdbms.erl +++ b/src/mongoose_cets_discovery_rdbms.erl @@ -4,8 +4,8 @@ -export([init/1, get_nodes/1]). %% these functions are exported for testing purposes only. --export([select/1, insert_new/4, update_existing/3, delete_node_from_db/2]). --ignore_xref([select/1, insert_new/4, update_existing/3, delete_node_from_db/2]). +-export([select/1, insert_new/5, update_existing/4, delete_node_from_db/2]). +-ignore_xref([select/1, insert_new/5, update_existing/4, delete_node_from_db/2]). -include("mongoose_logger.hrl"). @@ -59,19 +59,21 @@ try_register(ClusterName, Node, State) when is_binary(Node), is_binary(ClusterNa prepare(), Timestamp = timestamp(), {selected, Rows} = select(ClusterName), - {Nodes, Nums, _Timestamps} = lists:unzip3(Rows), + Nodes = [element(1, Row) || Row <- Rows], + Nums = [element(2, Row) || Row <- Rows], AlreadyRegistered = lists:member(Node, Nodes), + Address = os:getenv("MIM_NODE_IP", ""), NodeNum = case AlreadyRegistered of true -> - update_existing(ClusterName, Node, Timestamp), - {value, {_, Num, _TS}} = lists:keysearch(Node, 1, Rows), + update_existing(ClusterName, Node, Address, Timestamp), + {value, {_, Num, _Addr, _TS}} = lists:keysearch(Node, 1, Rows), Num; false -> Num = first_free_num(lists:usort(Nums)), %% Could fail with duplicate node_num reason. %% In this case just wait for the next get_nodes call. - case insert_new(ClusterName, Node, Timestamp, Num) of + case insert_new(ClusterName, Node, Num, Address, Timestamp) of {error, _} -> 0; %% return default node num {updated, 1} -> Num end @@ -79,6 +81,7 @@ try_register(ClusterName, Node, State) when is_binary(Node), is_binary(ClusterNa RunCleaningResult = run_cleaning(ClusterName, Timestamp, Rows, State), %% This could be used for debugging Info = #{already_registered => AlreadyRegistered, timestamp => Timestamp, + address => Address, node_num => Num, last_rows => Rows, run_cleaning_result => RunCleaningResult}, {NodeNum, skip_expired_nodes(Nodes, RunCleaningResult), Info}. @@ -87,7 +90,7 @@ skip_expired_nodes(Nodes, {removed, ExpiredNodes}) -> run_cleaning(ClusterName, Timestamp, Rows, State) -> #{expire_time := ExpireTime, node_name_to_insert := CurrentNode} = State, - ExpiredNodes = [DbNode || {DbNode, _Num, DbTS} <- Rows, + ExpiredNodes = [DbNode || {DbNode, _Num, _Addr, DbTS} <- Rows, is_expired(DbTS, Timestamp, ExpireTime), DbNode =/= CurrentNode], [delete_node_from_db(ClusterName, DbNode) || DbNode <- ExpiredNodes], @@ -110,30 +113,31 @@ prepare() -> mongoose_rdbms_timestamp:prepare(), mongoose_rdbms:prepare(cets_disco_select, T, [cluster_name], select()), mongoose_rdbms:prepare(cets_disco_insert_new, T, - [cluster_name, node_name, node_num, updated_timestamp], insert_new()), + [cluster_name, node_name, node_num, address, updated_timestamp], insert_new()), mongoose_rdbms:prepare(cets_disco_update_existing, T, - [updated_timestamp, cluster_name, node_name], update_existing()), + [updated_timestamp, address, cluster_name, node_name], update_existing()), mongoose_rdbms:prepare(cets_delete_node_from_db, T, [cluster_name, node_name], delete_node_from_db()). select() -> - <<"SELECT node_name, node_num, updated_timestamp FROM discovery_nodes WHERE cluster_name = ?">>. + <<"SELECT node_name, node_num, address, updated_timestamp FROM discovery_nodes WHERE cluster_name = ?">>. select(ClusterName) -> mongoose_rdbms:execute_successfully(global, cets_disco_select, [ClusterName]). insert_new() -> - <<"INSERT INTO discovery_nodes (cluster_name, node_name, node_num, updated_timestamp)" - " VALUES (?, ?, ?, ?)">>. + <<"INSERT INTO discovery_nodes (cluster_name, node_name, node_num, address, updated_timestamp)" + " VALUES (?, ?, ?, ?, ?)">>. -insert_new(ClusterName, Node, Timestamp, Num) -> - mongoose_rdbms:execute(global, cets_disco_insert_new, [ClusterName, Node, Num, Timestamp]). +insert_new(ClusterName, NodeName, NodeNum, Address, UpdatedTimestamp) -> + mongoose_rdbms:execute(global, cets_disco_insert_new, + [ClusterName, NodeName, NodeNum, Address, UpdatedTimestamp]). update_existing() -> - <<"UPDATE discovery_nodes SET updated_timestamp = ? WHERE cluster_name = ? AND node_name = ?">>. + <<"UPDATE discovery_nodes SET updated_timestamp = ?, address = ? WHERE cluster_name = ? AND node_name = ?">>. -update_existing(ClusterName, Node, Timestamp) -> - mongoose_rdbms:execute(global, cets_disco_update_existing, [Timestamp, ClusterName, Node]). +update_existing(ClusterName, NodeName, Address, UpdatedTimestamp) -> + mongoose_rdbms:execute(global, cets_disco_update_existing, [UpdatedTimestamp, Address, ClusterName, NodeName]). delete_node_from_db() -> <<"DELETE FROM discovery_nodes WHERE cluster_name = ? AND node_name = ?">>. From 94eb76c7c2948d57a26febcbbe2d1cd2e3387608 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Wed, 15 Nov 2023 23:41:55 +0100 Subject: [PATCH 03/15] Add mongoose_epmd module --- src/mongoose_epmd.erl | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 src/mongoose_epmd.erl diff --git a/src/mongoose_epmd.erl b/src/mongoose_epmd.erl new file mode 100644 index 00000000000..ce19e9301a9 --- /dev/null +++ b/src/mongoose_epmd.erl @@ -0,0 +1,37 @@ +-module(mongoose_epmd). +-export([ + start/0, + start_link/0, + stop/0, + port_please/2, port_please/3, + listen_port_please/2, + names/0, names/1, + register_node/2, register_node/3, + address_please/3, + open/0, open/1, open/2 +]). +-include_lib("kernel/include/logger.hrl"). + +start() -> erl_epmd:start(). +start_link() -> erl_epmd:start_link(). +stop() -> erl_epmd:stop(). +port_please(A, B) -> erl_epmd:port_please(A, B). +port_please(A, B, C) -> erl_epmd:port_please(A, B, C). +listen_port_please(A, B) -> erl_epmd:listen_port_please(A, B). +names() -> erl_epmd:names(). +names(A) -> erl_epmd:names(A). +register_node(A, B) -> erl_epmd:register_node(A, B). +register_node(A, B, C) -> erl_epmd:register_node(A, B, C). +open() -> erl_epmd:open(). +open(A) -> erl_epmd:open(A). +open(A, B) -> erl_epmd:open(A, B). + +address_please(Name, Host, AddressFamily) -> + Node = list_to_atom(Name ++ "@" ++ Host), + case mongoose_node_address:lookup(Node) of + {ok, _IP} = Res -> + Res; + _ -> + %% Fallback to the default behaviour + inet:getaddr(Host, AddressFamily) + end. From c865c54d420c91999f32f00020a595e0b41d3ab7 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Tue, 14 Nov 2023 19:00:34 +0100 Subject: [PATCH 04/15] Add lookup IP logic into mongoose_epmd Function address_please waits for get_nodes, gets backend_state, tries to use IP Add tests --- big_tests/tests/cets_disco_SUITE.erl | 65 ++++++++++++++++++- src/mongoose_cets_discovery.erl | 4 ++ src/mongoose_cets_discovery_rdbms.erl | 25 +++++--- src/mongoose_epmd.erl | 90 ++++++++++++++++++++++++++- 4 files changed, 172 insertions(+), 12 deletions(-) diff --git a/big_tests/tests/cets_disco_SUITE.erl b/big_tests/tests/cets_disco_SUITE.erl index b3f2a91e138..97081717b00 100644 --- a/big_tests/tests/cets_disco_SUITE.erl +++ b/big_tests/tests/cets_disco_SUITE.erl @@ -24,7 +24,12 @@ rdbms_cases() -> [rdbms_backend, rdbms_backend_supports_auto_cleaning, rdbms_backend_node_doesnt_remove_itself, - rdbms_backend_db_queries]. + rdbms_backend_db_queries, + rdbms_backend_publishes_node_ip, + no_record_for_node, + no_ip_in_db, + cannot_connect_to_epmd, + address_please]. suite() -> distributed_helper:require_rpc_nodes([mim, mim2]). @@ -34,6 +39,8 @@ suite() -> %%-------------------------------------------------------------------- init_per_group(rdbms, Config) -> + start_node_address_server(mim()), + start_node_address_server(mim2()), case not ct_helper:is_ct_running() orelse mongoose_helper:is_rdbms_enabled(domain_helper:host_type()) of false -> {skip, rdbms_or_ct_not_running}; @@ -42,6 +49,10 @@ init_per_group(rdbms, Config) -> init_per_group(_, Config) -> Config. +end_per_group(rdbms, Config) -> + stop_node_address_server(mim()), + stop_node_address_server(mim2()), + Config; end_per_group(_, Config) -> Config. @@ -163,6 +174,47 @@ rdbms_backend_db_queries(_Config) -> ?assertEqual({updated, 1}, delete_node_from_db(CN, <<"test1">>)), ?assertEqual({selected, [{<<"test2">>, 2, <<>>, TS}]}, select(CN)). +rdbms_backend_publishes_node_ip(_Config) -> + %% get_pairs would return only real available nodes, so use the real node names + Node1b = atom_to_binary(maps:get(node, mim())), + Node2b = atom_to_binary(maps:get(node, mim2())), + CN = random_cluster_name(?FUNCTION_NAME), + Opts1 = #{cluster_name => CN, node_name_to_insert => Node1b, + node_ip_binary => <<"127.0.0.1">>}, + Opts2 = #{cluster_name => CN, node_name_to_insert => Node2b, + node_ip_binary => <<"127.0.0.1">>}, + State1 = disco_init(mim(), Opts1), + State2 = disco_init(mim2(), Opts2), + {{ok, _Nodes1_2}, State1_2} = disco_get_nodes(mim(), State1), + {{ok, _Nodes2_2}, State2_2} = disco_get_nodes(mim2(), State2), + {{ok, _Nodes1_3}, State1_3} = disco_get_nodes(mim(), State1_2), + {{ok, _Nodes2_3}, State2_3} = disco_get_nodes(mim2(), State2_2), + {ok, {127, 0, 0, 1}} = match_node_name(mim2(), State2_3, Node1b), + {ok, {127, 0, 0, 1}} = match_node_name(mim(), State1_3, Node2b). + +no_record_for_node(_Config) -> + Node = <<"mongoose@badhost">>, + BackState = #{address_pairs => #{}}, + {error, {no_record_for_node, Node}} = match_node_name(mim(), BackState, Node), + ok. + +no_ip_in_db(_Config) -> + Node = <<"mongoose@noiphost">>, + BackState = #{address_pairs => #{Node => <<>>}}, + {error, {no_ip_in_db, Node}} = match_node_name(mim(), BackState, Node), + ok. + +cannot_connect_to_epmd(_Config) -> + Node = <<"mongoose@noepmdhost">>, + %% IP from a test range + BackState = #{address_pairs => #{Node => <<"192.0.2.1">>}}, + {error, {cannot_connect_to_epmd, Node, {192, 0, 2, 1}}} = match_node_name(mim(), BackState, Node), + ok. + +address_please(_Config) -> + {error, nxdomain} = + rpc(mim(), mongoose_epmd, address_please, ["mongooseim", "badbadhost", inet]). + %%-------------------------------------------------------------------- %% Helpers %%-------------------------------------------------------------------- @@ -177,6 +229,9 @@ disco_get_nodes(Node, State) -> log_disco_request(?FUNCTION_NAME, Node, State, NewState), NewState. +match_node_name(Node, SysInfo, NodeToLookup) -> + rpc(Node, mongoose_epmd, match_node_name, [SysInfo, NodeToLookup]). + log_disco_request(disco_init, Node, #{cluster_name := CN} = Opts, State) -> ct:log("[0] disco_init(~p,~n" ++ " ~p) =~n" ++ @@ -235,3 +290,11 @@ delete_node_from_db(CN, BinNode) -> Ret = rpc(mim(), mongoose_cets_discovery_rdbms, delete_node_from_db, [CN, BinNode]), ct:log("delete_node_from_db(~p, ~p) = ~p", [CN, BinNode, Ret]), Ret. + +start_node_address_server(Node) -> + MFA = {mongoose_node_address, start_link, []}, + ChildSpec = #{id => mongoose_node_address, start => MFA, restart => temporary}, + rpc(Node, supervisor, start_child, [ejabberd_sup, ChildSpec]). + +stop_node_address_server(Node) -> + rpc(Node, supervisor, terminate_child, [ejabberd_sup, mongoose_node_address]). diff --git a/src/mongoose_cets_discovery.erl b/src/mongoose_cets_discovery.erl index f3278a4e969..8a2407ade37 100644 --- a/src/mongoose_cets_discovery.erl +++ b/src/mongoose_cets_discovery.erl @@ -33,6 +33,7 @@ supervisor_specs(#{backend := DiscoBackend, cluster_name := ClusterName} = Opts) backend_module => disco_backend_to_module(DiscoBackend), cluster_name => atom_to_binary(ClusterName), node_name_to_insert => atom_to_binary(node(), latin1), + node_ip_binary => get_node_ip_binary(), name => mongoose_cets_discovery, disco_file => DiscoFile}, CetsDisco = {cets_discovery, @@ -42,3 +43,6 @@ supervisor_specs(#{backend := DiscoBackend, cluster_name := ClusterName} = Opts) disco_backend_to_module(rdbms) -> mongoose_cets_discovery_rdbms; disco_backend_to_module(file) -> cets_discovery_file. + +get_node_ip_binary() -> + list_to_binary(os:getenv("MIM_NODE_IP", "")). diff --git a/src/mongoose_cets_discovery_rdbms.erl b/src/mongoose_cets_discovery_rdbms.erl index 6a038328ba0..d5487bb287c 100644 --- a/src/mongoose_cets_discovery_rdbms.erl +++ b/src/mongoose_cets_discovery_rdbms.erl @@ -15,29 +15,34 @@ -type opts() :: #{cluster_name := binary(), node_name_to_insert := binary(), last_query_info => map(), expire_time => non_neg_integer(), + node_ip_binary => binary(), any() => any()}. -type state() :: #{cluster_name := binary(), node_name_to_insert := binary(), - last_query_info := map(), expire_time := non_neg_integer()}. + last_query_info := map(), expire_time := non_neg_integer(), + node_ip_binary := binary(), address_pairs := #{binary() => binary()}}. -spec init(opts()) -> state(). -init(Opts = #{cluster_name := _, node_name_to_insert := _}) -> - Keys = [cluster_name, node_name_to_insert, last_query_info, expire_time], +init(Opts = #{cluster_name := ClusterName, node_name_to_insert := Node}) + when is_binary(ClusterName), is_binary(Node) -> + Keys = [cluster_name, node_name_to_insert, last_query_info, expire_time, node_ip_binary], maps:with(Keys, maps:merge(defaults(), Opts)). defaults() -> #{expire_time => 60 * 60 * 1, %% 1 hour in seconds - last_query_info => #{}}. + last_query_info => #{}, + node_ip_binary => <<>>, + address_pairs => #{}}. -spec get_nodes(state()) -> {cets_discovery:get_nodes_result(), state()}. get_nodes(State = #{cluster_name := ClusterName, node_name_to_insert := Node}) -> case is_rdbms_running() of true -> try try_register(ClusterName, Node, State) of - {Num, Nodes, Info} -> + {Num, Nodes, Info, AddrPairs} -> mongoose_node_num:set_node_num(Num), {{ok, [binary_to_atom(N) || N <- Nodes]}, - State#{last_query_info => Info}} + State#{last_query_info => Info, address_pairs => AddrPairs}} catch Class:Reason:Stacktrace -> ?LOG_ERROR(#{what => discovery_failed_select, class => Class, reason => Reason, stacktrace => Stacktrace}), @@ -55,14 +60,16 @@ is_rdbms_running() -> false end. -try_register(ClusterName, Node, State) when is_binary(Node), is_binary(ClusterName) -> +try_register(ClusterName, Node, State = #{node_ip_binary := Address}) + when is_binary(Node), is_binary(ClusterName) -> prepare(), Timestamp = timestamp(), {selected, Rows} = select(ClusterName), Nodes = [element(1, Row) || Row <- Rows], Nums = [element(2, Row) || Row <- Rows], + Addresses = [element(3, Row) || Row <- Rows], + AddrPairs = maps:from_list(lists:zip(Nodes, Addresses)), AlreadyRegistered = lists:member(Node, Nodes), - Address = os:getenv("MIM_NODE_IP", ""), NodeNum = case AlreadyRegistered of true -> @@ -83,7 +90,7 @@ try_register(ClusterName, Node, State) when is_binary(Node), is_binary(ClusterNa Info = #{already_registered => AlreadyRegistered, timestamp => Timestamp, address => Address, node_num => Num, last_rows => Rows, run_cleaning_result => RunCleaningResult}, - {NodeNum, skip_expired_nodes(Nodes, RunCleaningResult), Info}. + {NodeNum, skip_expired_nodes(Nodes, RunCleaningResult), Info, AddrPairs}. skip_expired_nodes(Nodes, {removed, ExpiredNodes}) -> (Nodes -- ExpiredNodes). diff --git a/src/mongoose_epmd.erl b/src/mongoose_epmd.erl index ce19e9301a9..d68d1bcf53c 100644 --- a/src/mongoose_epmd.erl +++ b/src/mongoose_epmd.erl @@ -1,3 +1,5 @@ +%% EPMD implementation which redefines how name lookups work +%% There is no behaviour for erl_epmd -module(mongoose_epmd). -export([ start/0, @@ -12,6 +14,29 @@ ]). -include_lib("kernel/include/logger.hrl"). +%% For debugging +-export([lookup_ip/1]). +-export([match_node_name/2]). + +-type lookup_error() :: + {no_ip_in_db, node()} + | {cannot_connect_to_epmd, node(), inet:ip_address()} + | {no_record_for_node, node()} + | mongoose_node_address_ets_table_not_found. + +-ignore_xref([ + start/0, + start_link/0, + stop/0, + port_please/2, port_please/3, + listen_port_please/2, + names/0, names/1, + register_node/2, register_node/3, + address_please/3, + open/0, open/1, open/2, + lookup_ip/1, match_node_name/2 +]). + start() -> erl_epmd:start(). start_link() -> erl_epmd:start_link(). stop() -> erl_epmd:stop(). @@ -27,11 +52,72 @@ open(A) -> erl_epmd:open(A). open(A, B) -> erl_epmd:open(A, B). address_please(Name, Host, AddressFamily) -> - Node = list_to_atom(Name ++ "@" ++ Host), - case mongoose_node_address:lookup(Node) of + Node = list_to_binary(Name ++ "@" ++ Host), + case lookup_ip(Node) of {ok, _IP} = Res -> Res; _ -> %% Fallback to the default behaviour inet:getaddr(Host, AddressFamily) end. + +%% This function waits for the IP address to appear. +%% It only works well in case net_kernel does not contact a lot of +%% dead nodes. +%% Generally, we only expect calls for nodes that are alive +%% and the connect is issued by CETS (but CETS checks that node is +%% reachable first) or by connect_all feature in the global module of OTP. +-spec lookup_ip(binary()) -> {ok, inet:ip_address()} | {error, lookup_error()}. +lookup_ip(Node) -> + try + cets_discovery:wait_for_get_nodes(mongoose_cets_discovery, 3000), + cets_discovery:system_info(mongoose_cets_discovery) + of + #{backend_state := BackendState} -> + match_node_name(BackendState, Node) + catch _:_ -> + {error, failed_to_get_disco_state} + end. + +match_node_name(#{address_pairs := Pairs}, Node) -> + case Pairs of + #{Node := <<>>} -> + %% The caller should try DNS. + {error, {no_ip_in_db, Node}}; + #{Node := Bin} -> + {ok, IP} = inet:parse_address(binary_to_list(Bin)), + case can_connect(IP) of + true -> + {ok, IP}; + false -> + {error, {cannot_connect_to_epmd, Node, IP}} + end; + #{} -> + {error, {no_record_for_node, Node}} + end. + +-spec can_connect(inet:ip_address()) -> boolean(). +can_connect(IP) -> + Timeout = 1000, + case try_open(IP, Timeout) of + {ok, Socket} -> + gen_tcp:close(Socket), + true; + _ -> + false + end. + +-spec get_epmd_port() -> inet:port_number(). +get_epmd_port() -> + case init:get_argument(epmd_port) of + {ok, [[PortStr|_]|_]} when is_list(PortStr) -> + list_to_integer(PortStr); + error -> + 4369 + end. + +try_open({_, _, _, _} = IP, Timeout) -> + %% That would block + gen_tcp:connect(IP, get_epmd_port(), [inet], Timeout); +try_open({_, _, _, _, _, _, _, _} = IP, Timeout) -> + gen_tcp:connect(IP, get_epmd_port(), [inet6], Timeout). From 5039a805bde3c0523468ec8ece17100a9c3bbc19 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Tue, 28 Nov 2023 14:04:47 +0100 Subject: [PATCH 05/15] Add NOT NULL and empty default string for address field in schema --- priv/mssql2012.sql | 6 +++--- priv/mysql.sql | 6 +++--- priv/pg.sql | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/priv/mssql2012.sql b/priv/mssql2012.sql index a736a694f50..7712a00c172 100644 --- a/priv/mssql2012.sql +++ b/priv/mssql2012.sql @@ -754,10 +754,10 @@ CREATE TABLE domain_events ( CREATE INDEX i_domain_events_domain ON domain_events(domain); CREATE TABLE discovery_nodes ( - cluster_name varchar(250), - node_name varchar(250), + cluster_name varchar(250) NOT NULL, + node_name varchar(250) NOT NULL, node_num INT NOT NULL, - address varchar(250), + address varchar(250) NOT NULL DEFAULT '', -- empty means we should ask DNS updated_timestamp BIGINT NOT NULL, -- in seconds PRIMARY KEY (cluster_name, node_name) ); diff --git a/priv/mysql.sql b/priv/mysql.sql index 841ee188037..82953020727 100644 --- a/priv/mysql.sql +++ b/priv/mysql.sql @@ -546,10 +546,10 @@ CREATE TABLE domain_events ( CREATE INDEX i_domain_events_domain ON domain_events(domain); CREATE TABLE discovery_nodes ( - cluster_name varchar(250), - node_name varchar(250), + cluster_name varchar(250) NOT NULL, + node_name varchar(250) NOT NULL, node_num INT UNSIGNED NOT NULL, - address varchar(250), + address varchar(250) NOT NULL DEFAULT '', -- empty means we should ask DNS updated_timestamp BIGINT NOT NULL, -- in seconds PRIMARY KEY (cluster_name, node_name) ); diff --git a/priv/pg.sql b/priv/pg.sql index 401d5f26219..8b78cddffbf 100644 --- a/priv/pg.sql +++ b/priv/pg.sql @@ -506,10 +506,10 @@ CREATE TABLE domain_events ( CREATE INDEX i_domain_events_domain ON domain_events(domain); CREATE TABLE discovery_nodes ( - cluster_name varchar(250), - node_name varchar(250), + cluster_name varchar(250) NOT NULL, + node_name varchar(250) NOT NULL, node_num INT NOT NULL, - address varchar(250), + address varchar(250) NOT NULL DEFAULT '', -- empty means we should ask DNS updated_timestamp BIGINT NOT NULL, -- in seconds PRIMARY KEY (cluster_name, node_name) ); From aa686855541102a9536792593d25fe119019e80c Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Fri, 24 Nov 2023 13:26:07 +0100 Subject: [PATCH 06/15] Use latest version of CETS --- rebar.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.lock b/rebar.lock index 4108d969445..fed1ff8c9dc 100644 --- a/rebar.lock +++ b/rebar.lock @@ -8,7 +8,7 @@ {<<"certifi">>,{pkg,<<"certifi">>,<<"2.9.0">>},1}, {<<"cets">>, {git,"https://github.com/esl/cets.git", - {ref,"2ca31058bf616392ed91e4eb8642cc5af872902e"}}, + {ref,"4ef4a347f8dd9b5e798499705e290b8e1cce5523"}}, 0}, {<<"cowboy">>,{pkg,<<"cowboy">>,<<"2.9.0">>},0}, {<<"cowboy_swagger">>,{pkg,<<"cowboy_swagger">>,<<"2.5.1">>},0}, From fc3500fe89a89b2af11c958b481401cfb6803579 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Tue, 28 Nov 2023 18:28:49 +0100 Subject: [PATCH 07/15] Test address_please with IP --- big_tests/tests/cets_disco_SUITE.erl | 52 +++++++++++++++++++++++++++- rebar.lock | 2 +- src/mongoose_cets_discovery.erl | 11 +++--- src/mongoose_epmd.erl | 13 +++++-- 4 files changed, 70 insertions(+), 8 deletions(-) diff --git a/big_tests/tests/cets_disco_SUITE.erl b/big_tests/tests/cets_disco_SUITE.erl index 97081717b00..6a720e23df3 100644 --- a/big_tests/tests/cets_disco_SUITE.erl +++ b/big_tests/tests/cets_disco_SUITE.erl @@ -29,7 +29,8 @@ rdbms_cases() -> no_record_for_node, no_ip_in_db, cannot_connect_to_epmd, - address_please]. + address_please, + address_please_returns_ip]. suite() -> distributed_helper:require_rpc_nodes([mim, mim2]). @@ -56,12 +57,46 @@ end_per_group(rdbms, Config) -> end_per_group(_, Config) -> Config. +init_per_testcase(address_please_returns_ip, Config) -> + case rpc(mim(), erlang, whereis, [mongoose_cets_discovery]) of + undefined -> + mock_epmd(mim()), + {ok, _} = rpc(mim(), supervisor, start_child, [ejabberd_sup, cets_disco_spec(<<"testmim1@localhost">>, <<"127.0.0.2">>)]), + {ok, _} = rpc(mim2(), supervisor, start_child, [ejabberd_sup, cets_disco_spec(<<"testmim2@localhost">>, <<"127.0.0.5">>)]), + %% Force nodes to see each other + rpc(mim2(), erlang, send, [mongoose_cets_discovery, check]), + ok = rpc(mim2(), cets_discovery, wait_for_get_nodes, [mongoose_cets_discovery, 5000]), + rpc(mim(), erlang, send, [mongoose_cets_discovery, check]), + Config; + _ -> + {skip, cets_disco_already_running} + end; init_per_testcase(_CaseName, Config) -> Config. +end_per_testcase(address_please_returns_ip, Config) -> + ok = rpc(mim(), supervisor, terminate_child, [ejabberd_sup, cets_discovery]), + ok = rpc(mim2(), supervisor, terminate_child, [ejabberd_sup, cets_discovery]), + unmock_epmd(mim()), + Config; end_per_testcase(_CaseName, Config) -> unmock(mim()), unmock(mim2()). +cets_disco_spec(Node, IP) -> + DiscoOpts = #{ + backend_module => mongoose_cets_discovery_rdbms, + cluster_name => <<"mim">>, + node_name_to_insert => Node, + node_ip_binary => IP, + name => mongoose_cets_discovery}, + #{ + id => cets_discovery, + start => {mongoose_cets_discovery, start_link, [DiscoOpts]}, + restart => temporary, + type => worker, + shutdown => infinity, + modules => [cets_discovery]}. + %%-------------------------------------------------------------------- %% Test cases %%-------------------------------------------------------------------- @@ -215,6 +250,12 @@ address_please(_Config) -> {error, nxdomain} = rpc(mim(), mongoose_epmd, address_please, ["mongooseim", "badbadhost", inet]). +address_please_returns_ip(Config) -> + Res = rpc(mim(), mongoose_epmd, address_please, ["testmim2", "localhost", inet]), + Info = rpc(mim(), cets_discovery, system_info, [mongoose_cets_discovery]), + ct:log("system_info ~p", [Info]), + {ok, {127, 0, 0, 5}} = Res. + %%-------------------------------------------------------------------- %% Helpers %%-------------------------------------------------------------------- @@ -264,6 +305,15 @@ mock_timestamp(Node, Timestamp) -> unmock_timestamp(Node) -> ok = rpc(Node, meck, unload, [mongoose_rdbms_timestamp]). +mock_epmd(Node) -> + ok = rpc(Node, meck, new, [mongoose_epmd, [passthrough, no_link]]), + ok = rpc(Node, meck, expect, [mongoose_epmd, can_connect, 1, true]), + %% Ensure that we mock + true = rpc(Node, mongoose_epmd, can_connect, [{192, 168, 0, 100}]). + +unmock_epmd(Node) -> + ok = rpc(Node, meck, unload, [mongoose_epmd]). + unmock(Node) -> rpc(Node, meck, unload, []). diff --git a/rebar.lock b/rebar.lock index fed1ff8c9dc..ad628194fbb 100644 --- a/rebar.lock +++ b/rebar.lock @@ -8,7 +8,7 @@ {<<"certifi">>,{pkg,<<"certifi">>,<<"2.9.0">>},1}, {<<"cets">>, {git,"https://github.com/esl/cets.git", - {ref,"4ef4a347f8dd9b5e798499705e290b8e1cce5523"}}, + {ref,"ce92a1bf821b5cf7d4241f2d49f656171875cafe"}}, 0}, {<<"cowboy">>,{pkg,<<"cowboy">>,<<"2.9.0">>},0}, {<<"cowboy_swagger">>,{pkg,<<"cowboy_swagger">>,<<"2.5.1">>},0}, diff --git a/src/mongoose_cets_discovery.erl b/src/mongoose_cets_discovery.erl index 8a2407ade37..99ae0c2c6ec 100644 --- a/src/mongoose_cets_discovery.erl +++ b/src/mongoose_cets_discovery.erl @@ -35,10 +35,13 @@ supervisor_specs(#{backend := DiscoBackend, cluster_name := ClusterName} = Opts) node_name_to_insert => atom_to_binary(node(), latin1), node_ip_binary => get_node_ip_binary(), name => mongoose_cets_discovery, disco_file => DiscoFile}, - CetsDisco = - {cets_discovery, - {?MODULE, start_link, [DiscoOpts]}, - permanent, infinity, supervisor, [cets_discovery]}, + CetsDisco = #{ + id => cets_discovery, + start => {?MODULE, start_link, [DiscoOpts]}, + restart => permanent, + type => worker, + shutdown => infinity, + modules => [cets_discovery]}, [CetsDisco]. disco_backend_to_module(rdbms) -> mongoose_cets_discovery_rdbms; diff --git a/src/mongoose_epmd.erl b/src/mongoose_epmd.erl index d68d1bcf53c..6a3cc1b4dad 100644 --- a/src/mongoose_epmd.erl +++ b/src/mongoose_epmd.erl @@ -18,6 +18,9 @@ -export([lookup_ip/1]). -export([match_node_name/2]). +%% Make it mockable +-export([can_connect/1]). + -type lookup_error() :: {no_ip_in_db, node()} | {cannot_connect_to_epmd, node(), inet:ip_address()} @@ -86,7 +89,7 @@ match_node_name(#{address_pairs := Pairs}, Node) -> {error, {no_ip_in_db, Node}}; #{Node := Bin} -> {ok, IP} = inet:parse_address(binary_to_list(Bin)), - case can_connect(IP) of + case ?MODULE:can_connect(IP) of true -> {ok, IP}; false -> @@ -94,7 +97,13 @@ match_node_name(#{address_pairs := Pairs}, Node) -> end; #{} -> {error, {no_record_for_node, Node}} - end. + end; +match_node_name(BackendState, Node) -> + %% Could happen if CETS backend is not mongoose_cets_discovery_rdbms + ?LOG_ERROR(#{what => cets_no_address_pairs_set, + backend_state => BackendState, + remote_node => Node}), + {error, no_address_pairs_set}. -spec can_connect(inet:ip_address()) -> boolean(). can_connect(IP) -> From 4c2a9ff6060382847b7f2e5b1c2ffb3c168e898a Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Tue, 28 Nov 2023 18:48:38 +0100 Subject: [PATCH 08/15] Test that mongoose_epmd could be used --- rel/files/vm.args | 2 ++ rel/mim3.vars-toml.config | 1 + 2 files changed, 3 insertions(+) diff --git a/rel/files/vm.args b/rel/files/vm.args index 0b24338fab2..194e061d7e3 100644 --- a/rel/files/vm.args +++ b/rel/files/vm.args @@ -20,3 +20,5 @@ ## Tweak GC to run more often -env ERL_FULLSWEEP_AFTER 2 + +{{epmd_vm_args}} diff --git a/rel/mim3.vars-toml.config b/rel/mim3.vars-toml.config index 7583766b781..fe761e84931 100644 --- a/rel/mim3.vars-toml.config +++ b/rel/mim3.vars-toml.config @@ -1,5 +1,6 @@ %% vm.args {node_name, "mongooseim3@localhost"}. +{epmd_vm_args, "-epmd_module mongoose_epmd"}. %% mongooseim.toml {c2s_port, 5262}. From f8452ee0245ac5a9459e0c8153d2971b3624e136 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Tue, 28 Nov 2023 19:37:14 +0100 Subject: [PATCH 09/15] Add address_please_returns_ip_fallbacks_to_resolve testcase --- big_tests/tests/cets_disco_SUITE.erl | 87 ++++++++++++++++++---------- 1 file changed, 55 insertions(+), 32 deletions(-) diff --git a/big_tests/tests/cets_disco_SUITE.erl b/big_tests/tests/cets_disco_SUITE.erl index 6a720e23df3..7646571d25d 100644 --- a/big_tests/tests/cets_disco_SUITE.erl +++ b/big_tests/tests/cets_disco_SUITE.erl @@ -30,7 +30,8 @@ rdbms_cases() -> no_ip_in_db, cannot_connect_to_epmd, address_please, - address_please_returns_ip]. + address_please_returns_ip, + address_please_returns_ip_fallbacks_to_resolve]. suite() -> distributed_helper:require_rpc_nodes([mim, mim2]). @@ -58,45 +59,22 @@ end_per_group(_, Config) -> Config. init_per_testcase(address_please_returns_ip, Config) -> - case rpc(mim(), erlang, whereis, [mongoose_cets_discovery]) of - undefined -> - mock_epmd(mim()), - {ok, _} = rpc(mim(), supervisor, start_child, [ejabberd_sup, cets_disco_spec(<<"testmim1@localhost">>, <<"127.0.0.2">>)]), - {ok, _} = rpc(mim2(), supervisor, start_child, [ejabberd_sup, cets_disco_spec(<<"testmim2@localhost">>, <<"127.0.0.5">>)]), - %% Force nodes to see each other - rpc(mim2(), erlang, send, [mongoose_cets_discovery, check]), - ok = rpc(mim2(), cets_discovery, wait_for_get_nodes, [mongoose_cets_discovery, 5000]), - rpc(mim(), erlang, send, [mongoose_cets_discovery, check]), - Config; - _ -> - {skip, cets_disco_already_running} - end; + start_cets_discovery(Config, true); +init_per_testcase(address_please_returns_ip_fallbacks_to_resolve, Config) -> + start_cets_discovery(Config, false); init_per_testcase(_CaseName, Config) -> Config. end_per_testcase(address_please_returns_ip, Config) -> - ok = rpc(mim(), supervisor, terminate_child, [ejabberd_sup, cets_discovery]), - ok = rpc(mim2(), supervisor, terminate_child, [ejabberd_sup, cets_discovery]), + stop_cets_discovery(), unmock_epmd(mim()), Config; +end_per_testcase(address_please_returns_ip_fallbacks_to_resolve, Config) -> + stop_cets_discovery(), + Config; end_per_testcase(_CaseName, Config) -> unmock(mim()), unmock(mim2()). -cets_disco_spec(Node, IP) -> - DiscoOpts = #{ - backend_module => mongoose_cets_discovery_rdbms, - cluster_name => <<"mim">>, - node_name_to_insert => Node, - node_ip_binary => IP, - name => mongoose_cets_discovery}, - #{ - id => cets_discovery, - start => {mongoose_cets_discovery, start_link, [DiscoOpts]}, - restart => temporary, - type => worker, - shutdown => infinity, - modules => [cets_discovery]}. - %%-------------------------------------------------------------------- %% Test cases %%-------------------------------------------------------------------- @@ -254,7 +232,13 @@ address_please_returns_ip(Config) -> Res = rpc(mim(), mongoose_epmd, address_please, ["testmim2", "localhost", inet]), Info = rpc(mim(), cets_discovery, system_info, [mongoose_cets_discovery]), ct:log("system_info ~p", [Info]), - {ok, {127, 0, 0, 5}} = Res. + {ok, {192, 168, 115, 112}} = Res. + +address_please_returns_ip_fallbacks_to_resolve(Config) -> + Res = rpc(mim(), mongoose_epmd, address_please, ["testmim2", "localhost", inet]), + Info = rpc(mim(), cets_discovery, system_info, [mongoose_cets_discovery]), + ct:log("system_info ~p", [Info]), + {ok, {127, 0, 0, 1}} = Res. %%-------------------------------------------------------------------- %% Helpers @@ -348,3 +332,42 @@ start_node_address_server(Node) -> stop_node_address_server(Node) -> rpc(Node, supervisor, terminate_child, [ejabberd_sup, mongoose_node_address]). + +start_cets_discovery(Config, MockEpmd) -> + case rpc(mim(), erlang, whereis, [mongoose_cets_discovery]) of + undefined -> + case MockEpmd of + true -> + mock_epmd(mim()); + false -> + ok + end, + {ok, _} = rpc(mim(), supervisor, start_child, [ejabberd_sup, cets_disco_spec(<<"testmim1@localhost">>, <<"192.168.115.111">>)]), + {ok, _} = rpc(mim2(), supervisor, start_child, [ejabberd_sup, cets_disco_spec(<<"testmim2@localhost">>, <<"192.168.115.112">>)]), + %% Force nodes to see each other + rpc(mim2(), erlang, send, [mongoose_cets_discovery, check]), + ok = rpc(mim2(), cets_discovery, wait_for_get_nodes, [mongoose_cets_discovery, 5000]), + rpc(mim(), erlang, send, [mongoose_cets_discovery, check]), + Config; + _ -> + {skip, cets_disco_already_running} + end. + +stop_cets_discovery() -> + ok = rpc(mim(), supervisor, terminate_child, [ejabberd_sup, cets_discovery]), + ok = rpc(mim2(), supervisor, terminate_child, [ejabberd_sup, cets_discovery]). + +cets_disco_spec(Node, IP) -> + DiscoOpts = #{ + backend_module => mongoose_cets_discovery_rdbms, + cluster_name => <<"mim">>, + node_name_to_insert => Node, + node_ip_binary => IP, + name => mongoose_cets_discovery}, + #{ + id => cets_discovery, + start => {mongoose_cets_discovery, start_link, [DiscoOpts]}, + restart => temporary, + type => worker, + shutdown => infinity, + modules => [cets_discovery]}. From b8ec9b32b6f9a1ac48826f9aae149935f4f3e519 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Wed, 29 Nov 2023 10:49:29 +0100 Subject: [PATCH 10/15] Improve code coverage --- big_tests/tests/cets_disco_SUITE.erl | 62 +++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 2 deletions(-) diff --git a/big_tests/tests/cets_disco_SUITE.erl b/big_tests/tests/cets_disco_SUITE.erl index 7646571d25d..9a641725a0e 100644 --- a/big_tests/tests/cets_disco_SUITE.erl +++ b/big_tests/tests/cets_disco_SUITE.erl @@ -31,7 +31,9 @@ rdbms_cases() -> cannot_connect_to_epmd, address_please, address_please_returns_ip, - address_please_returns_ip_fallbacks_to_resolve]. + address_please_returns_ip_fallbacks_to_resolve, + address_please_returns_ip_fallbacks_to_resolve_with_file_backend, + address_please_returns_ip_127_0_0_1_from_db]. suite() -> distributed_helper:require_rpc_nodes([mim, mim2]). @@ -62,13 +64,19 @@ init_per_testcase(address_please_returns_ip, Config) -> start_cets_discovery(Config, true); init_per_testcase(address_please_returns_ip_fallbacks_to_resolve, Config) -> start_cets_discovery(Config, false); +init_per_testcase(address_please_returns_ip_fallbacks_to_resolve_with_file_backend, Config) -> + start_cets_discovery_with_file_backnend(Config); +init_per_testcase(address_please_returns_ip_127_0_0_1_from_db, Config) -> + start_cets_discovery_with_real_ips(Config); init_per_testcase(_CaseName, Config) -> Config. end_per_testcase(address_please_returns_ip, Config) -> stop_cets_discovery(), unmock_epmd(mim()), Config; -end_per_testcase(address_please_returns_ip_fallbacks_to_resolve, Config) -> +end_per_testcase(Name, Config) when Name == address_please_returns_ip_fallbacks_to_resolve; + Name == address_please_returns_ip_fallbacks_to_resolve_with_file_backend; + Name == address_please_returns_ip_127_0_0_1_from_db -> stop_cets_discovery(), Config; end_per_testcase(_CaseName, Config) -> @@ -240,6 +248,20 @@ address_please_returns_ip_fallbacks_to_resolve(Config) -> ct:log("system_info ~p", [Info]), {ok, {127, 0, 0, 1}} = Res. +address_please_returns_ip_fallbacks_to_resolve_with_file_backend(Config) -> + Res = rpc(mim2(), mongoose_epmd, address_please, ["testmim1", "localhost", inet]), + Info = rpc(mim2(), cets_discovery, system_info, [mongoose_cets_discovery]), + ct:log("system_info ~p", [Info]), + {ok, {127, 0, 0, 1}} = Res. + +address_please_returns_ip_127_0_0_1_from_db(Config) -> + %% We use mim2() because it has no meck. We want to improve code coverage + %% covering case when mongoose_epmd:lookup_ip returns IP. + Res = rpc(mim2(), mongoose_epmd, address_please, ["node1", "localhost", inet]), + Info = rpc(mim2(), cets_discovery, system_info, [mongoose_cets_discovery]), + ct:log("system_info ~p", [Info]), + {ok, {127, 0, 0, 1}} = Res. + %%-------------------------------------------------------------------- %% Helpers %%-------------------------------------------------------------------- @@ -353,6 +375,32 @@ start_cets_discovery(Config, MockEpmd) -> {skip, cets_disco_already_running} end. +start_cets_discovery_with_real_ips(Config) -> + case rpc(mim(), erlang, whereis, [mongoose_cets_discovery]) of + undefined -> + {ok, _} = rpc(mim(), supervisor, start_child, [ejabberd_sup, cets_disco_spec(<<"node1@localhost">>, <<"127.0.0.1">>)]), + {ok, _} = rpc(mim2(), supervisor, start_child, [ejabberd_sup, cets_disco_spec(<<"node2@localhost">>, <<"127.0.0.1">>)]), + %% Force nodes to see each other + rpc(mim2(), erlang, send, [mongoose_cets_discovery, check]), + ok = rpc(mim2(), cets_discovery, wait_for_get_nodes, [mongoose_cets_discovery, 5000]), + rpc(mim(), erlang, send, [mongoose_cets_discovery, check]), + ok = rpc(mim(), cets_discovery, wait_for_get_nodes, [mongoose_cets_discovery, 5000]), + rpc(mim2(), erlang, send, [mongoose_cets_discovery, check]), + Config; + _ -> + {skip, cets_disco_already_running} + end. + +start_cets_discovery_with_file_backnend(Config) -> + case rpc(mim(), erlang, whereis, [mongoose_cets_discovery]) of + undefined -> + {ok, _} = rpc(mim(), supervisor, start_child, [ejabberd_sup, cets_disco_spec_for_file_backend()]), + {ok, _} = rpc(mim2(), supervisor, start_child, [ejabberd_sup, cets_disco_spec_for_file_backend()]), + Config; + _ -> + {skip, cets_disco_already_running} + end. + stop_cets_discovery() -> ok = rpc(mim(), supervisor, terminate_child, [ejabberd_sup, cets_discovery]), ok = rpc(mim2(), supervisor, terminate_child, [ejabberd_sup, cets_discovery]). @@ -364,6 +412,16 @@ cets_disco_spec(Node, IP) -> node_name_to_insert => Node, node_ip_binary => IP, name => mongoose_cets_discovery}, + cets_disco_spec(DiscoOpts). + +cets_disco_spec_for_file_backend() -> + DiscoOpts = #{ + backend_module => cets_discovery_file, + disco_file => "/tmp/does_not_exist", + name => mongoose_cets_discovery}, + cets_disco_spec(DiscoOpts). + +cets_disco_spec(DiscoOpts) -> #{ id => cets_discovery, start => {mongoose_cets_discovery, start_link, [DiscoOpts]}, From 39a074259eb707ca3fb5fcdbdd01230ef5af128a Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Wed, 29 Nov 2023 11:04:56 +0100 Subject: [PATCH 11/15] Remove can_connect logic from mongoose_epmd --- big_tests/tests/cets_disco_SUITE.erl | 41 ++++------------------------ src/mongoose_epmd.erl | 37 +------------------------ 2 files changed, 7 insertions(+), 71 deletions(-) diff --git a/big_tests/tests/cets_disco_SUITE.erl b/big_tests/tests/cets_disco_SUITE.erl index 9a641725a0e..c8b5aaf87d6 100644 --- a/big_tests/tests/cets_disco_SUITE.erl +++ b/big_tests/tests/cets_disco_SUITE.erl @@ -28,10 +28,9 @@ rdbms_cases() -> rdbms_backend_publishes_node_ip, no_record_for_node, no_ip_in_db, - cannot_connect_to_epmd, + epmd_just_returns_ip_from_db, address_please, address_please_returns_ip, - address_please_returns_ip_fallbacks_to_resolve, address_please_returns_ip_fallbacks_to_resolve_with_file_backend, address_please_returns_ip_127_0_0_1_from_db]. @@ -61,9 +60,7 @@ end_per_group(_, Config) -> Config. init_per_testcase(address_please_returns_ip, Config) -> - start_cets_discovery(Config, true); -init_per_testcase(address_please_returns_ip_fallbacks_to_resolve, Config) -> - start_cets_discovery(Config, false); + start_cets_discovery(Config); init_per_testcase(address_please_returns_ip_fallbacks_to_resolve_with_file_backend, Config) -> start_cets_discovery_with_file_backnend(Config); init_per_testcase(address_please_returns_ip_127_0_0_1_from_db, Config) -> @@ -72,10 +69,8 @@ init_per_testcase(_CaseName, Config) -> Config. end_per_testcase(address_please_returns_ip, Config) -> stop_cets_discovery(), - unmock_epmd(mim()), Config; -end_per_testcase(Name, Config) when Name == address_please_returns_ip_fallbacks_to_resolve; - Name == address_please_returns_ip_fallbacks_to_resolve_with_file_backend; +end_per_testcase(Name, Config) when Name == address_please_returns_ip_fallbacks_to_resolve_with_file_backend; Name == address_please_returns_ip_127_0_0_1_from_db -> stop_cets_discovery(), Config; @@ -225,12 +220,11 @@ no_ip_in_db(_Config) -> {error, {no_ip_in_db, Node}} = match_node_name(mim(), BackState, Node), ok. -cannot_connect_to_epmd(_Config) -> +epmd_just_returns_ip_from_db(_Config) -> Node = <<"mongoose@noepmdhost">>, %% IP from a test range BackState = #{address_pairs => #{Node => <<"192.0.2.1">>}}, - {error, {cannot_connect_to_epmd, Node, {192, 0, 2, 1}}} = match_node_name(mim(), BackState, Node), - ok. + {ok, {192, 0, 2, 1}} = match_node_name(mim(), BackState, Node). address_please(_Config) -> {error, nxdomain} = @@ -242,12 +236,6 @@ address_please_returns_ip(Config) -> ct:log("system_info ~p", [Info]), {ok, {192, 168, 115, 112}} = Res. -address_please_returns_ip_fallbacks_to_resolve(Config) -> - Res = rpc(mim(), mongoose_epmd, address_please, ["testmim2", "localhost", inet]), - Info = rpc(mim(), cets_discovery, system_info, [mongoose_cets_discovery]), - ct:log("system_info ~p", [Info]), - {ok, {127, 0, 0, 1}} = Res. - address_please_returns_ip_fallbacks_to_resolve_with_file_backend(Config) -> Res = rpc(mim2(), mongoose_epmd, address_please, ["testmim1", "localhost", inet]), Info = rpc(mim2(), cets_discovery, system_info, [mongoose_cets_discovery]), @@ -255,8 +243,6 @@ address_please_returns_ip_fallbacks_to_resolve_with_file_backend(Config) -> {ok, {127, 0, 0, 1}} = Res. address_please_returns_ip_127_0_0_1_from_db(Config) -> - %% We use mim2() because it has no meck. We want to improve code coverage - %% covering case when mongoose_epmd:lookup_ip returns IP. Res = rpc(mim2(), mongoose_epmd, address_please, ["node1", "localhost", inet]), Info = rpc(mim2(), cets_discovery, system_info, [mongoose_cets_discovery]), ct:log("system_info ~p", [Info]), @@ -311,15 +297,6 @@ mock_timestamp(Node, Timestamp) -> unmock_timestamp(Node) -> ok = rpc(Node, meck, unload, [mongoose_rdbms_timestamp]). -mock_epmd(Node) -> - ok = rpc(Node, meck, new, [mongoose_epmd, [passthrough, no_link]]), - ok = rpc(Node, meck, expect, [mongoose_epmd, can_connect, 1, true]), - %% Ensure that we mock - true = rpc(Node, mongoose_epmd, can_connect, [{192, 168, 0, 100}]). - -unmock_epmd(Node) -> - ok = rpc(Node, meck, unload, [mongoose_epmd]). - unmock(Node) -> rpc(Node, meck, unload, []). @@ -355,15 +332,9 @@ start_node_address_server(Node) -> stop_node_address_server(Node) -> rpc(Node, supervisor, terminate_child, [ejabberd_sup, mongoose_node_address]). -start_cets_discovery(Config, MockEpmd) -> +start_cets_discovery(Config) -> case rpc(mim(), erlang, whereis, [mongoose_cets_discovery]) of undefined -> - case MockEpmd of - true -> - mock_epmd(mim()); - false -> - ok - end, {ok, _} = rpc(mim(), supervisor, start_child, [ejabberd_sup, cets_disco_spec(<<"testmim1@localhost">>, <<"192.168.115.111">>)]), {ok, _} = rpc(mim2(), supervisor, start_child, [ejabberd_sup, cets_disco_spec(<<"testmim2@localhost">>, <<"192.168.115.112">>)]), %% Force nodes to see each other diff --git a/src/mongoose_epmd.erl b/src/mongoose_epmd.erl index 6a3cc1b4dad..ccfd006251b 100644 --- a/src/mongoose_epmd.erl +++ b/src/mongoose_epmd.erl @@ -18,9 +18,6 @@ -export([lookup_ip/1]). -export([match_node_name/2]). -%% Make it mockable --export([can_connect/1]). - -type lookup_error() :: {no_ip_in_db, node()} | {cannot_connect_to_epmd, node(), inet:ip_address()} @@ -88,13 +85,7 @@ match_node_name(#{address_pairs := Pairs}, Node) -> %% The caller should try DNS. {error, {no_ip_in_db, Node}}; #{Node := Bin} -> - {ok, IP} = inet:parse_address(binary_to_list(Bin)), - case ?MODULE:can_connect(IP) of - true -> - {ok, IP}; - false -> - {error, {cannot_connect_to_epmd, Node, IP}} - end; + inet:parse_address(binary_to_list(Bin)); #{} -> {error, {no_record_for_node, Node}} end; @@ -104,29 +95,3 @@ match_node_name(BackendState, Node) -> backend_state => BackendState, remote_node => Node}), {error, no_address_pairs_set}. - --spec can_connect(inet:ip_address()) -> boolean(). -can_connect(IP) -> - Timeout = 1000, - case try_open(IP, Timeout) of - {ok, Socket} -> - gen_tcp:close(Socket), - true; - _ -> - false - end. - --spec get_epmd_port() -> inet:port_number(). -get_epmd_port() -> - case init:get_argument(epmd_port) of - {ok, [[PortStr|_]|_]} when is_list(PortStr) -> - list_to_integer(PortStr); - error -> - 4369 - end. - -try_open({_, _, _, _} = IP, Timeout) -> - %% That would block - gen_tcp:connect(IP, get_epmd_port(), [inet], Timeout); -try_open({_, _, _, _, _, _, _, _} = IP, Timeout) -> - gen_tcp:connect(IP, get_epmd_port(), [inet6], Timeout). From e7e923055698df0f04a085273df971b329efd65b Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Wed, 29 Nov 2023 11:26:27 +0100 Subject: [PATCH 12/15] Remove start_node_address_server from cets_disco_SUITE Simplify start_cets_discovery code --- big_tests/tests/cets_disco_SUITE.erl | 61 ++++++++++++---------------- 1 file changed, 27 insertions(+), 34 deletions(-) diff --git a/big_tests/tests/cets_disco_SUITE.erl b/big_tests/tests/cets_disco_SUITE.erl index c8b5aaf87d6..18a8f0dea16 100644 --- a/big_tests/tests/cets_disco_SUITE.erl +++ b/big_tests/tests/cets_disco_SUITE.erl @@ -42,8 +42,6 @@ suite() -> %%-------------------------------------------------------------------- init_per_group(rdbms, Config) -> - start_node_address_server(mim()), - start_node_address_server(mim2()), case not ct_helper:is_ct_running() orelse mongoose_helper:is_rdbms_enabled(domain_helper:host_type()) of false -> {skip, rdbms_or_ct_not_running}; @@ -52,10 +50,6 @@ init_per_group(rdbms, Config) -> init_per_group(_, Config) -> Config. -end_per_group(rdbms, Config) -> - stop_node_address_server(mim()), - stop_node_address_server(mim2()), - Config; end_per_group(_, Config) -> Config. @@ -67,10 +61,8 @@ init_per_testcase(address_please_returns_ip_127_0_0_1_from_db, Config) -> start_cets_discovery_with_real_ips(Config); init_per_testcase(_CaseName, Config) -> Config. -end_per_testcase(address_please_returns_ip, Config) -> - stop_cets_discovery(), - Config; -end_per_testcase(Name, Config) when Name == address_please_returns_ip_fallbacks_to_resolve_with_file_backend; +end_per_testcase(Name, Config) when Name == address_please_returns_ip; + Name == address_please_returns_ip_fallbacks_to_resolve_with_file_backend; Name == address_please_returns_ip_127_0_0_1_from_db -> stop_cets_discovery(), Config; @@ -324,23 +316,12 @@ delete_node_from_db(CN, BinNode) -> ct:log("delete_node_from_db(~p, ~p) = ~p", [CN, BinNode, Ret]), Ret. -start_node_address_server(Node) -> - MFA = {mongoose_node_address, start_link, []}, - ChildSpec = #{id => mongoose_node_address, start => MFA, restart => temporary}, - rpc(Node, supervisor, start_child, [ejabberd_sup, ChildSpec]). - -stop_node_address_server(Node) -> - rpc(Node, supervisor, terminate_child, [ejabberd_sup, mongoose_node_address]). - start_cets_discovery(Config) -> case rpc(mim(), erlang, whereis, [mongoose_cets_discovery]) of undefined -> - {ok, _} = rpc(mim(), supervisor, start_child, [ejabberd_sup, cets_disco_spec(<<"testmim1@localhost">>, <<"192.168.115.111">>)]), - {ok, _} = rpc(mim2(), supervisor, start_child, [ejabberd_sup, cets_disco_spec(<<"testmim2@localhost">>, <<"192.168.115.112">>)]), - %% Force nodes to see each other - rpc(mim2(), erlang, send, [mongoose_cets_discovery, check]), - ok = rpc(mim2(), cets_discovery, wait_for_get_nodes, [mongoose_cets_discovery, 5000]), - rpc(mim(), erlang, send, [mongoose_cets_discovery, check]), + start_disco(mim(), cets_disco_spec(<<"testmim1@localhost">>, <<"192.168.115.111">>)), + start_disco(mim2(), cets_disco_spec(<<"testmim2@localhost">>, <<"192.168.115.112">>)), + force_nodes_to_see_each_other(mim(), mim2()), Config; _ -> {skip, cets_disco_already_running} @@ -349,14 +330,9 @@ start_cets_discovery(Config) -> start_cets_discovery_with_real_ips(Config) -> case rpc(mim(), erlang, whereis, [mongoose_cets_discovery]) of undefined -> - {ok, _} = rpc(mim(), supervisor, start_child, [ejabberd_sup, cets_disco_spec(<<"node1@localhost">>, <<"127.0.0.1">>)]), - {ok, _} = rpc(mim2(), supervisor, start_child, [ejabberd_sup, cets_disco_spec(<<"node2@localhost">>, <<"127.0.0.1">>)]), - %% Force nodes to see each other - rpc(mim2(), erlang, send, [mongoose_cets_discovery, check]), - ok = rpc(mim2(), cets_discovery, wait_for_get_nodes, [mongoose_cets_discovery, 5000]), - rpc(mim(), erlang, send, [mongoose_cets_discovery, check]), - ok = rpc(mim(), cets_discovery, wait_for_get_nodes, [mongoose_cets_discovery, 5000]), - rpc(mim2(), erlang, send, [mongoose_cets_discovery, check]), + start_disco(mim(), cets_disco_spec(<<"node1@localhost">>, <<"127.0.0.1">>)), + start_disco(mim2(), cets_disco_spec(<<"node2@localhost">>, <<"127.0.0.1">>)), + force_nodes_to_see_each_other(mim(), mim2()), Config; _ -> {skip, cets_disco_already_running} @@ -365,8 +341,8 @@ start_cets_discovery_with_real_ips(Config) -> start_cets_discovery_with_file_backnend(Config) -> case rpc(mim(), erlang, whereis, [mongoose_cets_discovery]) of undefined -> - {ok, _} = rpc(mim(), supervisor, start_child, [ejabberd_sup, cets_disco_spec_for_file_backend()]), - {ok, _} = rpc(mim2(), supervisor, start_child, [ejabberd_sup, cets_disco_spec_for_file_backend()]), + start_disco(mim(), cets_disco_spec_for_file_backend()), + start_disco(mim2(), cets_disco_spec_for_file_backend()), Config; _ -> {skip, cets_disco_already_running} @@ -400,3 +376,20 @@ cets_disco_spec(DiscoOpts) -> type => worker, shutdown => infinity, modules => [cets_discovery]}. + +send_check(Node) -> + rpc(Node, erlang, send, [mongoose_cets_discovery, check]). + +wait_for_get_nodes(Node) -> + ok = rpc(Node, cets_discovery, wait_for_get_nodes, [mongoose_cets_discovery, 5000]). + +start_disco(Node, Spec) -> + {ok, _} = rpc(Node, supervisor, start_child, [ejabberd_sup, Spec]). + +force_nodes_to_see_each_other(Node1, Node2) -> + send_check(Node2), + wait_for_get_nodes(Node2), + send_check(Node1), + wait_for_get_nodes(Node1), + send_check(Node2), + wait_for_get_nodes(Node2). From 408b3a4aa51ad6e26af74572464448b69dd15152 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Wed, 29 Nov 2023 11:39:59 +0100 Subject: [PATCH 13/15] Allow to restore CETS discovery process in pgsql_cets preset Or just remove test disco process in case we are not in CETS preset but still want to test RDBMS logic with all DB backends --- big_tests/tests/cets_disco_SUITE.erl | 67 +++++++++++++++++----------- 1 file changed, 41 insertions(+), 26 deletions(-) diff --git a/big_tests/tests/cets_disco_SUITE.erl b/big_tests/tests/cets_disco_SUITE.erl index 18a8f0dea16..6f690691b4c 100644 --- a/big_tests/tests/cets_disco_SUITE.erl +++ b/big_tests/tests/cets_disco_SUITE.erl @@ -45,11 +45,16 @@ init_per_group(rdbms, Config) -> case not ct_helper:is_ct_running() orelse mongoose_helper:is_rdbms_enabled(domain_helper:host_type()) of false -> {skip, rdbms_or_ct_not_running}; - true -> Config + true -> + stop_and_delete_cets_discovery_if_running(), + Config end; init_per_group(_, Config) -> Config. +end_per_group(rdbms, Config) -> + restore_default_cets_discovery(), + Config; end_per_group(_, Config) -> Config. @@ -317,40 +322,50 @@ delete_node_from_db(CN, BinNode) -> Ret. start_cets_discovery(Config) -> - case rpc(mim(), erlang, whereis, [mongoose_cets_discovery]) of - undefined -> - start_disco(mim(), cets_disco_spec(<<"testmim1@localhost">>, <<"192.168.115.111">>)), - start_disco(mim2(), cets_disco_spec(<<"testmim2@localhost">>, <<"192.168.115.112">>)), - force_nodes_to_see_each_other(mim(), mim2()), - Config; - _ -> - {skip, cets_disco_already_running} - end. + start_disco(mim(), cets_disco_spec(<<"testmim1@localhost">>, <<"192.168.115.111">>)), + start_disco(mim2(), cets_disco_spec(<<"testmim2@localhost">>, <<"192.168.115.112">>)), + force_nodes_to_see_each_other(mim(), mim2()), + Config. start_cets_discovery_with_real_ips(Config) -> - case rpc(mim(), erlang, whereis, [mongoose_cets_discovery]) of - undefined -> - start_disco(mim(), cets_disco_spec(<<"node1@localhost">>, <<"127.0.0.1">>)), - start_disco(mim2(), cets_disco_spec(<<"node2@localhost">>, <<"127.0.0.1">>)), - force_nodes_to_see_each_other(mim(), mim2()), - Config; - _ -> - {skip, cets_disco_already_running} - end. + start_disco(mim(), cets_disco_spec(<<"node1@localhost">>, <<"127.0.0.1">>)), + start_disco(mim2(), cets_disco_spec(<<"node2@localhost">>, <<"127.0.0.1">>)), + force_nodes_to_see_each_other(mim(), mim2()), + Config. start_cets_discovery_with_file_backnend(Config) -> + start_disco(mim(), cets_disco_spec_for_file_backend()), + start_disco(mim2(), cets_disco_spec_for_file_backend()), + Config. + +stop_cets_discovery() -> + ok = rpc(mim(), supervisor, terminate_child, [ejabberd_sup, cets_discovery]), + ok = rpc(mim2(), supervisor, terminate_child, [ejabberd_sup, cets_discovery]). + +stop_and_delete_cets_discovery() -> + stop_cets_discovery(), + ok = rpc(mim(), supervisor, delete_child, [ejabberd_sup, cets_discovery]), + ok = rpc(mim2(), supervisor, delete_child, [ejabberd_sup, cets_discovery]). + +stop_and_delete_cets_discovery_if_running() -> case rpc(mim(), erlang, whereis, [mongoose_cets_discovery]) of undefined -> - start_disco(mim(), cets_disco_spec_for_file_backend()), - start_disco(mim2(), cets_disco_spec_for_file_backend()), - Config; + ok; _ -> - {skip, cets_disco_already_running} + stop_and_delete_cets_discovery() end. -stop_cets_discovery() -> - ok = rpc(mim(), supervisor, terminate_child, [ejabberd_sup, cets_discovery]), - ok = rpc(mim2(), supervisor, terminate_child, [ejabberd_sup, cets_discovery]). +restore_default_cets_discovery() -> + restore_default_cets_discovery(mim()), + restore_default_cets_discovery(mim2()). + +restore_default_cets_discovery(Node) -> + case rpc(Node, mongoose_cets_discovery, supervisor_specs, []) of + [] -> + ok; + [Spec] -> + start_disco(Node, Spec) + end. cets_disco_spec(Node, IP) -> DiscoOpts = #{ From dad1fbf6aa17beb13f6375fafd28663cbde85bc0 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Wed, 29 Nov 2023 11:47:23 +0100 Subject: [PATCH 14/15] Document epmd_module option --- doc/configuration/release-options.md | 11 +++++++++++ rel/files/vm.args | 4 +++- rel/mim3.vars-toml.config | 2 +- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/doc/configuration/release-options.md b/doc/configuration/release-options.md index 22937368cda..281cfa71101 100644 --- a/doc/configuration/release-options.md +++ b/doc/configuration/release-options.md @@ -33,6 +33,17 @@ These options are inserted into the `rel/files/vm.args` template. * **Syntax:** command-line arguments * **Example:** `{highload_vm_args, "+P 10000000 -env ERL_MAX_PORTS 250000"}.` +### epmd_module + +Allows to set EPMD module to `mongoose_epmd` in case CETS is used with RDBMS backend +to enable getting IP addresses of the remote nodes using RDBMS instead of the default +resolver. + +* **Type:** parameter +* **Option:** value of `-epmd_module` in [vm.args](configuration-files.md#vmargs) +* **Syntax:** Erlang module name: `mongoose_epmd` +* **Example:** `{epmd_module, "mongoose_epmd"}.` + ## TOML Options These options are inserted into the `rel/files/mongooseim.toml` template. diff --git a/rel/files/vm.args b/rel/files/vm.args index 194e061d7e3..f93acc72646 100644 --- a/rel/files/vm.args +++ b/rel/files/vm.args @@ -21,4 +21,6 @@ ## Tweak GC to run more often -env ERL_FULLSWEEP_AFTER 2 -{{epmd_vm_args}} +{{#epmd_module}} +-epmd_module {{{epmd_module}}} +{{/epmd_module}} diff --git a/rel/mim3.vars-toml.config b/rel/mim3.vars-toml.config index fe761e84931..9d00c273364 100644 --- a/rel/mim3.vars-toml.config +++ b/rel/mim3.vars-toml.config @@ -1,6 +1,6 @@ %% vm.args {node_name, "mongooseim3@localhost"}. -{epmd_vm_args, "-epmd_module mongoose_epmd"}. +{epmd_module, "mongoose_epmd"}. %% mongooseim.toml {c2s_port, 5262}. From 162799455b767c3334300ea26299464c8fae6d77 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Thu, 30 Nov 2023 11:14:16 +0100 Subject: [PATCH 15/15] Update CETS with ping fixes --- rebar.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.lock b/rebar.lock index ad628194fbb..b9dc1d71321 100644 --- a/rebar.lock +++ b/rebar.lock @@ -8,7 +8,7 @@ {<<"certifi">>,{pkg,<<"certifi">>,<<"2.9.0">>},1}, {<<"cets">>, {git,"https://github.com/esl/cets.git", - {ref,"ce92a1bf821b5cf7d4241f2d49f656171875cafe"}}, + {ref,"9c5b2c9269daeb4ee6830f7cd1538e0a4218e281"}}, 0}, {<<"cowboy">>,{pkg,<<"cowboy">>,<<"2.9.0">>},0}, {<<"cowboy_swagger">>,{pkg,<<"cowboy_swagger">>,<<"2.5.1">>},0},